io.github.seniortesting.grpc.protoc.GrpcTools Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of sparrow-grpc Show documentation
Show all versions of sparrow-grpc Show documentation
Wrapped grpc Client for Junit5
The newest version!
package io.github.seniortesting.grpc.protoc;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.stream.Collectors;
import org.apache.commons.lang3.concurrent.BasicThreadFactory;
import org.apache.commons.lang3.time.StopWatch;
import org.w3c.dom.NodeList;
import io.github.seniortesting.core.exception.SparrowException;
import io.github.seniortesting.core.util.CommandLines;
import io.github.seniortesting.core.util.FilesExt;
import io.github.seniortesting.core.util.StringsExt;
import io.github.seniortesting.core.util.Xmls;
import lombok.extern.slf4j.Slf4j;
/**
* A simple wrapped grpc tool to compile the .proto files to java source code using protoc
*
* @author Walter Hu
* Limit: Not support for grpc custom plugin
*/
@Slf4j
public class GrpcTools {
private static GrpcTools grpcTools = null;
private static final String DEFAULT_PROTO_ROOT_FOLDER =
System.getProperty("user.home") + File.separator + ".grpc-proto";
private static final String DEFAULT_PROTOC_ROOT_FOLDER =
System.getProperty("user.home") + File.separator + ".grpc-proto";
// protoc, protoc-gen-grpc-java files for protoc compile
private String DEFAULT_PROTOC_VERSION = "LATEST";
private String DEFAULT_GRPC_VERSION = "LATEST";
// NOTE: DEFAULT_PROTO_COMMON_VERSION bind with the GRPC version
private String DEFAULT_PROTO_COMMON_VERSION = "LATEST";
private static final String LATEST_VERSION_FLAG = "LATEST";
private static final String DEFAULT_SOURCE_FOLDER = "proto-dependencies";
private static final String DEFAULT_PROTO_SOURCE_FOLDER =
DEFAULT_PROTO_ROOT_FOLDER + File.separator + DEFAULT_SOURCE_FOLDER;
private static final String DEFAULT_GIT_BRANCH = "develop";
private static final String DEFAULT_PROTO_SOURCE_ROOT = "src/main/proto";
private static final String PROTO_PROTO_FILTER = ".proto";
private static final String DEFAULT_PROTOC_PLUGIN_FOLDER =
DEFAULT_PROTOC_ROOT_FOLDER + File.separator + "protoc-plugins";
private static final String DEFAULT_PROTOC_DEPENDENCIES_FOLDER =
DEFAULT_PROTOC_ROOT_FOLDER + File.separator + "protoc-dependencies";
private static final String DEFAULT_FILE_DESCRIPTOR_SET_FOLDER =
DEFAULT_PROTOC_ROOT_FOLDER + File.separator + "proto-descriptor-sets";
// this path should be the same as {@link io.github.grpc.core.constants.DescriptorFile }
private static final String DEFAULT_FILE_DESCRIPTOR_SET =
DEFAULT_FILE_DESCRIPTOR_SET_FOLDER + File.separator + "grpc-proto.pb";
// download urls:
// for china user, please use this maven mirror url: https://maven.aliyun.com/repository/public
private static final String DEFAULT_MAVEN_MIRROR = "https://maven.aliyun.com/repository/public";
private static final String URL_SEPARATOR = "/";
private static final String MAVEN_METADATA = "maven-metadata.xml";
private static final String RELEASE_VERSION_NODE_XPATH = "//metadata/versioning/release";
private static final String PROTOC_URL =
"%s/com/google/protobuf/protoc/";
private static final String PROTOC_EXECUTABLE = "protoc-%s-%s.exe";
private static final String PROTOC_PLUGIN_URL =
"%s/io/grpc/protoc-gen-grpc-java/";
private static final String PROTOC_PLUGIN_EXECUTABLE = "protoc-gen-grpc-java-%s-%s.exe";
private static final String PROTOBUF_URL =
"%s/com/google/protobuf/protobuf-java/";
private static final String PROTOBUF_EXECUTABLE = "protobuf-java-%s.jar";
private static final String PROTO_COMMON_URL =
"%s/com/google/api/grpc/proto-google-common-protos/";
private static final String PROTO_COMMON_EXECUTABLE = "proto-google-common-protos-%s.jar";
private static final String DEFAULT_INCLUDES = "**/*.proto*";
private int corePoolSize = 2;
private int maxPoolSize = 10;
private int keepAliveTime = 60;
private int queueCapacity = 200;
private String threadNamePrefix = "protoc-thread-%d";
private ExecutorService executorService;
public static GrpcTools getInstance() {
if (null == grpcTools) {
grpcTools = new GrpcTools();
}
grpcTools.initThreadPoolExecutor();
return grpcTools;
}
private void initThreadPoolExecutor() {
final BasicThreadFactory threadFactory = new BasicThreadFactory.Builder()
.namingPattern(threadNamePrefix)
.daemon(true)
.priority(Thread.MAX_PRIORITY).build();
ThreadPoolExecutor threadPoolExecutor = new ThreadPoolExecutor(
corePoolSize,
maxPoolSize,
keepAliveTime,
TimeUnit.SECONDS,
new LinkedBlockingQueue<>(queueCapacity),
threadFactory);
threadPoolExecutor.setRejectedExecutionHandler(new ThreadPoolExecutor.CallerRunsPolicy());
this.executorService = threadPoolExecutor;
}
private Path cloneProtoRepository(String gitRepoUrl) {
return cloneProtoRepository(gitRepoUrl, DEFAULT_GIT_BRANCH);
}
private Path cloneProtoRepository(String gitRepoUrl, String branch) {
Path defaultProtoFolder = Path.of(DEFAULT_PROTO_SOURCE_FOLDER);
return cloneProtoRepository(gitRepoUrl, branch, defaultProtoFolder);
}
private Path cloneProtoRepository(String gitRepoUrl, Path outputPath) {
return cloneProtoRepository(gitRepoUrl, DEFAULT_GIT_BRANCH, outputPath);
}
/**
* git clone the proto files repository firstly
*
* @param gitRepoUrl
*/
private Path cloneProtoRepository(String gitRepoUrl, String branch, Path outputPath) {
Path tempCloneDirectory = null;
try {
tempCloneDirectory = Files.createTempDirectory(DEFAULT_SOURCE_FOLDER);
// first delete the existing files, always download source files
final StopWatch stopWatch = StopWatch.createStarted();
String tempFileAbsolutePath = tempCloneDirectory.normalize().toAbsolutePath().toString();
// clone protoc git code
final List cloneCommandList = List.of("git", "clone", "-b", branch, gitRepoUrl,
tempFileAbsolutePath);
try {
final String commandStr = cloneCommandList.stream().map(String::toString)
.collect(Collectors.joining(" "));
LOGGER.info(commandStr);
final Process cloneProcess = CommandLines.exec(cloneCommandList);
final int errCode = cloneProcess.waitFor();
if (errCode != 0) {
final String execResult = CommandLines.getExecErrorResult(cloneProcess);
LOGGER.error("command execution exit code return {}", execResult);
throw new SparrowException(commandStr + " git clone command exit error");
} else {
// copy clone files into current specified location
Path sourceRootPath = Path.of(tempFileAbsolutePath, DEFAULT_PROTO_SOURCE_ROOT);
FilesExt.copyDir(sourceRootPath, outputPath);
}
LOGGER.info("Git clone proto completed, time takes: {} seconds", stopWatch.getTime(TimeUnit.SECONDS));
} catch (InterruptedException e) {
LOGGER.error("Clone command exception:", e);
throw new SparrowException(e);
}
} catch (IOException e) {
LOGGER.error("Clone command IO exception:", e);
throw new SparrowException(e);
} finally {
// delete the temp git files
FilesExt.del(tempCloneDirectory.toFile());
}
return outputPath;
}
private Path downloadProtoc(ProtocType protocType) {
return downloadProtoc(DEFAULT_MAVEN_MIRROR, protocType, Path.of(DEFAULT_PROTOC_PLUGIN_FOLDER));
}
private Path downloadProtoc(String mavenMirrorUrl, ProtocType protocType, Path outputDirectory) {
if (protocType.equals(ProtocType.PROTOC)) {
return downloadProtoc(
protocType,
mavenMirrorUrl,
PROTOC_URL,
DEFAULT_PROTOC_VERSION,
PROTOC_EXECUTABLE,
outputDirectory);
} else if (protocType.equals(ProtocType.PROTOC_PLUGIN)) {
return downloadProtoc(
protocType,
mavenMirrorUrl,
PROTOC_PLUGIN_URL,
DEFAULT_GRPC_VERSION,
PROTOC_PLUGIN_EXECUTABLE,
outputDirectory);
} else if (protocType.equals(ProtocType.PROTOBUF)) {
return downloadProtoc(
protocType,
mavenMirrorUrl,
PROTOBUF_URL,
DEFAULT_PROTOC_VERSION,
PROTOBUF_EXECUTABLE,
outputDirectory);
} else if (protocType.equals(ProtocType.PROTO_COMMON)) {
return downloadProtoc(
protocType,
mavenMirrorUrl,
PROTO_COMMON_URL,
DEFAULT_PROTO_COMMON_VERSION,
PROTO_COMMON_EXECUTABLE,
outputDirectory);
} else {
LOGGER.error("unsupported download type: {}", protocType.toString());
}
return null;
}
private Path downloadProtoc(
ProtocType protocType,
String mavenMirrorUrl,
String protocDownloadUrl,
String protocVersion,
String protocExecutable,
Path outputDirectory) {
final String outputDirectoryPath = outputDirectory.normalize().toAbsolutePath().toString();
String protocUrl = null;
try {
if (Files.notExists(outputDirectory)) {
Files.createDirectories(outputDirectory);
}
if (!Files.isDirectory(outputDirectory)) {
LOGGER.warn("download directory {} is not a directory ", outputDirectory);
return null;
}
// if protocVersion not pass, will try to get the latest version
if (protocVersion == null || protocVersion.equals("") || protocVersion.equals(LATEST_VERSION_FLAG)) {
String metaDataUrl = String.format(protocDownloadUrl, mavenMirrorUrl) + MAVEN_METADATA;
LOGGER.info("Version is {}, will try to build the latest protoc executable", protocVersion);
final byte[] downloadBytes = FilesExt.UrlDownloader.downloadWithOkhttpSync(metaDataUrl);
// metadata/versioning/release
final NodeList nodeList = Xmls.parse(downloadBytes, RELEASE_VERSION_NODE_XPATH);
if (Optional.ofNullable(nodeList).isPresent()) {
protocVersion = nodeList.item(0).getTextContent();
} else {
LOGGER.error("Not found the node path: {} from uri: {}", RELEASE_VERSION_NODE_XPATH, metaDataUrl);
throw new SparrowException("Version fetch exception");
}
}
//set the version fields
if (protocType.equals(ProtocType.PROTOC)) {
DEFAULT_PROTOC_VERSION = protocVersion;
} else if (protocType.equals(ProtocType.PROTOC_PLUGIN)) {
DEFAULT_GRPC_VERSION = protocVersion;
} else if (protocType.equals(ProtocType.PROTO_COMMON)) {
DEFAULT_PROTO_COMMON_VERSION = protocVersion;
}
Detector.detect();
String DETECTED_CLASSIFIER = System.getProperty(Detector.DETECTED_CLASSIFIER);
String executableName = String.format(protocExecutable, protocVersion, DETECTED_CLASSIFIER);
Path protocExecutablePath = Paths.get(outputDirectoryPath, executableName);
String saveAsExecutablePath = protocExecutablePath.normalize().toAbsolutePath().toString();
if (Files.exists(protocExecutablePath)) {
LOGGER.warn("Proto file exists in: {}", saveAsExecutablePath);
makeFileExecutable(protocExecutablePath);
return protocExecutablePath;
}
protocUrl =
String.format(protocDownloadUrl, mavenMirrorUrl)
+ protocVersion + URL_SEPARATOR + executableName;
LOGGER.info("Downloading file: {}", protocUrl);
final StopWatch stopWatch = StopWatch.createStarted();
FilesExt.UrlDownloader.downloadFileWithResume(protocUrl, saveAsExecutablePath);
makeFileExecutable(protocExecutablePath);
LOGGER.info("Download proto {} completed, time takes: {} seconds", executableName,
stopWatch.getTime(TimeUnit.SECONDS));
return protocExecutablePath;
} catch (Exception e) {
LOGGER.error("Download proto exception, proto url {} ", protocUrl, e);
}
return null;
}
private void makeFileExecutable(Path file) {
if (!Detector.isFamilyWindows()) {
file.toFile().setReadable(true);
file.toFile().setExecutable(true);
}
}
private Path makeProtoPathFromJars(ProtobufType protobufType) {
return makeProtoPathFromJars(protobufType, Path.of(DEFAULT_PROTO_SOURCE_FOLDER), true);
}
private Path makeProtoPathFromJars(ProtobufType protobufType, Path extractDirectory, boolean extractSamePath) {
// two main jars need to add as protoc import path:
// 1. https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/3.15.1/protobuf-java-3.15.1.jar
// 2. https://repo1.maven.org/maven2/com/google/api/grpc/proto-google-common-protos/2.0
// download the jar file
Path downloadJarPath = null;
if (protobufType.equals(ProtobufType.PROTOBUF)) {
downloadJarPath = downloadProtoc(ProtocType.PROTOBUF);
} else if (protobufType.equals(ProtobufType.PROTO_COMMON)) {
downloadJarPath = downloadProtoc(ProtocType.PROTO_COMMON);
}
if (Optional.ofNullable(downloadJarPath).isPresent()) {
// extract the jar file for proto files
// for some reason under IAM, we receive poms as dependent files
// I am excluding .xml rather than including .jar as there may be other extensions in use (sar, har, zip)
File classpathElementFile = downloadJarPath.toFile();
if (classpathElementFile.isFile() && classpathElementFile.canRead() &&
!classpathElementFile.getName().endsWith(".xml")) {
// create the jar file. the constructor validates.
try (final JarFile classpathJar = new JarFile(classpathElementFile)) {
final Enumeration jarEntries = classpathJar.entries();
File jarDirectory;
if (extractSamePath) {
jarDirectory = new File(extractDirectory.normalize().toAbsolutePath().toString());
} else {
jarDirectory = new File(extractDirectory.normalize().toAbsolutePath().toString(),
truncatePath(classpathJar.getName()));
// clean the temporary directory to ensure that stale files aren't used
FilesExt.del(jarDirectory);
}
while (jarEntries.hasMoreElements()) {
final JarEntry jarEntry = jarEntries.nextElement();
final String jarEntryName = jarEntry.getName();
if (!jarEntry.isDirectory() && SelectorUtils
.matchPath(DEFAULT_INCLUDES, jarEntryName, "/", true)) {
try {
// Check for Zip Slip vulnerability
// https://snyk.io/research/zip-slip-vulnerability
final String canonicalJarDirectoryPath = jarDirectory.getCanonicalPath();
final File uncompressedCopy = new File(jarDirectory, jarEntryName);
final String canonicalUncompressedCopyPath = uncompressedCopy.getCanonicalPath();
if (!canonicalUncompressedCopyPath
.startsWith(canonicalJarDirectoryPath + File.separator)) {
throw new SparrowException(
"ZIP SLIP: Entry " + jarEntry.getName() +
" in " + classpathJar.getName() + " is outside of the target dir");
}
Files.createDirectories(uncompressedCopy.getParentFile().toPath());
Files.copy(classpathJar.getInputStream(jarEntry), uncompressedCopy.toPath(),
StandardCopyOption.REPLACE_EXISTING);
} catch (final IOException e) {
throw new SparrowException("Unable to unpack proto files", e);
}
}
}
return jarDirectory.toPath();
} catch (final IOException e) {
throw new SparrowException(
"Not a readable JAR artifact: " + classpathElementFile.getAbsolutePath(), e);
}
}
}
return null;
}
/**
* Truncates the path of jar files so that they are relative to the local repository.
*
* @param jarPath the full path of a jar file.
* @return the truncated path relative to the local repository or root of the drive.
*/
private String truncatePath(final String jarPath) {
return StringsExt.md5(jarPath);
}
private boolean protocCompiler(
Path protoPath,
List protoPathElements,
Path protocExecutablePath,
Path protocGenGrpcJavaExecutablePath) {
// clear old descriptorsFile
final Path descriptorFile = Path.of(DEFAULT_FILE_DESCRIPTOR_SET);
try {
if (Files.exists(descriptorFile)) {
Files.deleteIfExists(descriptorFile);
}
Files.createDirectories(descriptorFile.getParent());
} catch (IOException e) {
}
final List compileCommand = new ArrayList<>();
compileCommand.add(protocExecutablePath.normalize().toAbsolutePath().toString());
try {
final String protoRootFilePath = protoPath.normalize().toAbsolutePath().toString();
// add "--proto_path=" + protoPathElement
compileCommand.add("--proto_path=" + protoRootFilePath);
for (final Path protoPathElement : protoPathElements) {
compileCommand.add("--proto_path=" + protoPathElement.normalize().toAbsolutePath().toString());
}
// add --java_out=
compileCommand.add("--java_out=" + protoRootFilePath);
// command.add("--plugin=protoc-gen-grpc-java=' + pluginExecutable);
// command.add("--grpc-java_out=" + javaOutputDirectory);
if (Optional.ofNullable(protocGenGrpcJavaExecutablePath).isPresent()) {
compileCommand.add("--plugin=protoc-gen-grpc-java=" + protocGenGrpcJavaExecutablePath);
compileCommand.add("--grpc-java_out=" + protoRootFilePath);
}
// add proto path one by one
final List protoFiles = FilesExt
.listFiles(protoPath, path -> path.toFile().getName().endsWith(PROTO_PROTO_FILTER));
for (final String protoFile : protoFiles) {
compileCommand.add(protoFile);
}
// command.add("--descriptor_set_out=" + descriptorSetFile);
compileCommand.add("--descriptor_set_out=" + DEFAULT_FILE_DESCRIPTOR_SET);
// command.add("--include_imports");
// command.add("--include_source_info");
final StopWatch stopWatch = StopWatch.createStarted();
final Process commandProcess = CommandLines.exec(compileCommand);
final int errCode = commandProcess.waitFor();
LOGGER.info("protoc compile completed, time takes: {} seconds", stopWatch.getTime(TimeUnit.SECONDS));
boolean commandSuccess = (errCode == 0);
if (!commandSuccess) {
final String execErrorResult = CommandLines.getExecErrorResult(commandProcess);
LOGGER.error("command execution exit code return {}", execErrorResult);
}
return commandSuccess;
} catch (InterruptedException e) {
LOGGER.error("Clone command exception:", e);
throw new SparrowException(e);
}
}
private void addClassPaths(Path protoPath)
throws NoSuchMethodException, InvocationTargetException, IllegalAccessException, MalformedURLException {
final URL url = protoPath.toUri().toURL();
ClassLoader classLoader = ClassLoader.getSystemClassLoader();
URLClassLoader urlClassLoader = null;
if (classLoader instanceof URLClassLoader) {
urlClassLoader = (URLClassLoader) classLoader;
} else {
}
if (Optional.ofNullable(urlClassLoader).isPresent()) {
Method method = URLClassLoader.class.getDeclaredMethod("addURL", new Class[] { URL.class });
method.setAccessible(true);
method.invoke(urlClassLoader, new Object[] { url });
}
}
public void compile(String protoGitRepoUrl) {
compile(protoGitRepoUrl, DEFAULT_GIT_BRANCH);
}
public void compile(String protoGitRepoUrl, String branch) {
final List protocPaths = new ArrayList<>();
final StopWatch stopWatch = StopWatch.createStarted();
Callable cloneRepoThread = () -> cloneProtoRepository(protoGitRepoUrl, branch);
Callable protocDownloadThread = () -> downloadProtoc(ProtocType.PROTOC);
Callable protocPluginDownloadThread = () -> downloadProtoc(ProtocType.PROTOC_PLUGIN);
Callable protobufDownloadThread = () -> makeProtoPathFromJars(ProtobufType.PROTOBUF);
Callable protoCommonDownloadThread = () -> makeProtoPathFromJars(ProtobufType.PROTO_COMMON);
final List> downloadThreadList = List.of(
cloneRepoThread,
protocDownloadThread,
protocPluginDownloadThread,
protobufDownloadThread,
protoCommonDownloadThread);
try {
final List> futureList = grpcTools.executorService.invokeAll(downloadThreadList);
for (final Future future : futureList) {
final Path path = future.get();
protocPaths.add(path);
}
} catch (InterruptedException e) {
LOGGER.error("timeout download: ", e);
} catch (ExecutionException e) {
LOGGER.error("execution exception: ", e);
}
// compile protoc
final boolean allMatch = protocPaths.stream().findAny().stream().allMatch(path -> Objects.nonNull(path));
if (protocPaths.size() == downloadThreadList.size() && allMatch) {
Detector.detect();
String DETECTED_CLASSIFIER = System.getProperty(Detector.DETECTED_CLASSIFIER);
String protocExecutableName = String.format(PROTOC_EXECUTABLE, DEFAULT_PROTOC_VERSION,
DETECTED_CLASSIFIER);
String protocPluginExecutableName = String
.format(PROTOC_PLUGIN_EXECUTABLE, DEFAULT_GRPC_VERSION,
DETECTED_CLASSIFIER);
Path cloneProtoPath =
protocPaths.stream()
.filter(path -> path.equals(Path.of(DEFAULT_PROTO_SOURCE_FOLDER)))
.findFirst().orElse(null);
Path downloadProtocPath =
protocPaths.stream()
.filter(path -> path.normalize().toAbsolutePath().endsWith(protocExecutableName))
.findFirst().orElse(null);
Path downloadProtocPluginPath =
protocPaths.stream()
.filter(path -> path.normalize().toAbsolutePath().endsWith(protocPluginExecutableName))
.findFirst().orElse(null);
Set protoPathElements =
protocPaths.stream()
.filter(path -> path.normalize().toAbsolutePath().toString()
.contains(DEFAULT_PROTO_SOURCE_FOLDER))
.collect(Collectors.toSet());
if (Optional.ofNullable(downloadProtocPath).isPresent()
&& Optional.ofNullable(downloadProtocPluginPath).isPresent()
&& protoPathElements.size() == 1) {
final boolean protocCompilerResult =
protocCompiler(cloneProtoPath,
protoPathElements.stream().collect(Collectors.toList()),
downloadProtocPath,
downloadProtocPluginPath);
LOGGER.info("Compile Info: \n - Proto Source Path: {}", cloneProtoPath);
LOGGER.info("Compile success result is: {}, time takes {} seconds", protocCompilerResult,
stopWatch.getTime(TimeUnit.SECONDS));
} else {
LOGGER.error("Compile check executable file not found!");
}
}
}
public String getFileDescriptorSet() {
return DEFAULT_FILE_DESCRIPTOR_SET;
}
/**
* The protoc type
*
* @author Walter Hu
*/
public enum ProtocType {
/**
* inspect the download type
*/
PROTOC,
PROTOC_PLUGIN,
PROTOBUF,
PROTO_COMMON;
}
/**
* The download protobuf common libraries type
*
* @author Walter Hu
*/
public enum ProtobufType {
/**
* inspect the download type
*/
PROTOBUF,
PROTO_COMMON;
}
} © 2015 - 2025 Weber Informatics LLC | Privacy Policy