org.openrewrite.maven.ResourceParser Maven / Gradle / Ivy
/*
* Copyright 2020 the original author or authors.
*
* Licensed under the Moderne Source Available License (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://docs.moderne.io/licensing/moderne-source-available-license
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.openrewrite.maven;
import org.apache.maven.plugin.logging.Log;
import org.openrewrite.ExecutionContext;
import org.openrewrite.SourceFile;
import org.openrewrite.groovy.GroovyParser;
import org.openrewrite.hcl.HclParser;
import org.openrewrite.java.JavaParser;
import org.openrewrite.json.JsonParser;
import org.openrewrite.kotlin.KotlinParser;
import org.openrewrite.properties.PropertiesParser;
import org.openrewrite.protobuf.ProtoParser;
import org.openrewrite.quark.QuarkParser;
import org.openrewrite.text.PlainTextParser;
import org.openrewrite.xml.XmlParser;
import org.openrewrite.yaml.YamlParser;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
public class ResourceParser {
private static final Set DEFAULT_ACCEPTED_DIRECTORIES = new HashSet<>(Collections.singleton("src"));
private static final Set DEFAULT_IGNORED_DIRECTORIES = new HashSet<>(Arrays.asList(
"build", "target", "out",
".sonar", ".gradle", ".idea", ".project", "node_modules", ".git", ".metadata", ".DS_Store"));
private final Path baseDir;
private final Log logger;
private final Collection exclusions;
private final int sizeThresholdMb;
private final Collection excludedDirectories;
private final Collection plainTextMasks;
/**
* Sometimes java files will exist in the src/main/resources directory. For example, Drools:
*/
private final JavaParser.Builder javaParserBuilder;
private final KotlinParser.Builder kotlinParserBuilder;
private final ExecutionContext ctx;
public ResourceParser(Path baseDir, Log logger, Collection exclusions, Collection plainTextMasks, int sizeThresholdMb, Collection excludedDirectories,
JavaParser.Builder javaParserBuilder, KotlinParser.Builder kotlinParserBuilder, ExecutionContext ctx) {
this.baseDir = baseDir;
this.logger = logger;
this.javaParserBuilder = javaParserBuilder;
this.kotlinParserBuilder = kotlinParserBuilder;
this.exclusions = pathMatchers(baseDir, exclusions);
this.sizeThresholdMb = sizeThresholdMb;
this.excludedDirectories = excludedDirectories;
this.plainTextMasks = pathMatchers(baseDir, plainTextMasks);
this.ctx = ctx;
}
private Collection pathMatchers(Path basePath, Collection pathExpressions) {
return pathExpressions.stream()
.map(o -> basePath.getFileSystem().getPathMatcher("glob:" + o))
.collect(Collectors.toList());
}
public Stream parse(Path searchDir, Collection alreadyParsed) {
Stream sourceFiles = Stream.empty();
if (!searchDir.toFile().exists()) {
return sourceFiles;
}
try {
sourceFiles = Stream.concat(sourceFiles, parseSourceFiles(searchDir, alreadyParsed, ctx));
} catch (IOException e) {
logger.error(e.getMessage(), e);
throw new UncheckedIOException(e);
}
return sourceFiles;
}
@SuppressWarnings({"DuplicatedCode", "unchecked"})
public Stream parseSourceFiles(
Path searchDir,
Collection alreadyParsed,
ExecutionContext ctx) throws IOException {
List resources = new ArrayList<>();
List quarkPaths = new ArrayList<>();
List plainTextPaths = new ArrayList<>();
Files.walkFileTree(searchDir, Collections.emptySet(), 16, new SimpleFileVisitor() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) {
if (isExcluded(dir) || isIgnoredDirectory(searchDir, dir) || excludedDirectories.contains(dir)) {
return FileVisitResult.SKIP_SUBTREE;
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
if (!attrs.isOther() && !attrs.isSymbolicLink() &&
!alreadyParsed.contains(file) && !isExcluded(file)) {
if (isOverSizeThreshold(attrs.size())) {
logger.info("Not parsing quark " + file + " as its size " + attrs.size() / (1024L * 1024L) +
" MB exceeds size threshold " + sizeThresholdMb + " MB");
quarkPaths.add(file);
} else if (isParsedAsPlainText(file)) {
plainTextPaths.add(file);
} else {
resources.add(file);
}
}
return FileVisitResult.CONTINUE;
}
});
Stream sourceFiles = Stream.empty();
JavaParser javaParser = javaParserBuilder.build();
List javaPaths = new ArrayList<>();
JsonParser jsonParser = new JsonParser();
List jsonPaths = new ArrayList<>();
XmlParser xmlParser = new XmlParser();
List xmlPaths = new ArrayList<>();
YamlParser yamlParser = new YamlParser();
List yamlPaths = new ArrayList<>();
PropertiesParser propertiesParser = new PropertiesParser();
List propertiesPaths = new ArrayList<>();
ProtoParser protoParser = new ProtoParser();
List protoPaths = new ArrayList<>();
KotlinParser kotlinParser = kotlinParserBuilder.build();
List kotlinPaths = new ArrayList<>();
GroovyParser groovyParser = GroovyParser.builder().build();
List groovyPaths = new ArrayList<>();
HclParser hclParser = HclParser.builder().build();
List hclPaths = new ArrayList<>();
PlainTextParser plainTextParser = new PlainTextParser();
QuarkParser quarkParser = new QuarkParser();
resources.forEach(path -> {
// See https://github.com/quarkusio/quarkus/blob/main/devtools/project-core-extension-codestarts/src/main/resources/codestarts/quarkus/extension-codestarts/resteasy-reactive-codestart/java/src/main/java/org/acme/%7Bresource.class-name%7D.tpl.qute.java
// for an example of why we don't want qute files be parsed as java
if (javaParser.accept(path) && !path.endsWith(".qute.java")) {
javaPaths.add(path);
} else if (jsonParser.accept(path)) {
jsonPaths.add(path);
} else if (xmlParser.accept(path)) {
xmlPaths.add(path);
} else if (yamlParser.accept(path)) {
yamlPaths.add(path);
} else if (propertiesParser.accept(path)) {
propertiesPaths.add(path);
} else if (protoParser.accept(path)) {
protoPaths.add(path);
} else if (kotlinParser.accept(path)) {
kotlinPaths.add(path);
} else if (groovyParser.accept(path)) {
groovyPaths.add(path);
} else if (hclParser.accept(path)) {
hclPaths.add(path);
} else if (quarkParser.accept(path)) {
quarkPaths.add(path);
}
});
if (!javaPaths.isEmpty()) {
sourceFiles = Stream.concat(sourceFiles, (Stream) javaParser.parse(javaPaths, baseDir, ctx));
alreadyParsed.addAll(javaPaths);
}
if (!jsonPaths.isEmpty()) {
sourceFiles = Stream.concat(sourceFiles, (Stream) jsonParser.parse(jsonPaths, baseDir, ctx));
alreadyParsed.addAll(jsonPaths);
}
if (!xmlPaths.isEmpty()) {
sourceFiles = Stream.concat(sourceFiles, (Stream) xmlParser.parse(xmlPaths, baseDir, ctx));
alreadyParsed.addAll(xmlPaths);
}
if (!yamlPaths.isEmpty()) {
sourceFiles = Stream.concat(sourceFiles, (Stream) yamlParser.parse(yamlPaths, baseDir, ctx));
alreadyParsed.addAll(yamlPaths);
}
if (!propertiesPaths.isEmpty()) {
sourceFiles = Stream.concat(sourceFiles, (Stream) propertiesParser.parse(propertiesPaths, baseDir, ctx));
alreadyParsed.addAll(propertiesPaths);
}
if (!protoPaths.isEmpty()) {
sourceFiles = Stream.concat(sourceFiles, (Stream) protoParser.parse(protoPaths, baseDir, ctx));
alreadyParsed.addAll(protoPaths);
}
if (!kotlinPaths.isEmpty()) {
sourceFiles = Stream.concat(sourceFiles, (Stream) kotlinParser.parse(kotlinPaths, baseDir, ctx));
alreadyParsed.addAll(kotlinPaths);
}
if (!groovyPaths.isEmpty()) {
sourceFiles = Stream.concat(sourceFiles, (Stream) groovyParser.parse(groovyPaths, baseDir, ctx));
alreadyParsed.addAll(groovyPaths);
}
if (!hclPaths.isEmpty()) {
sourceFiles = Stream.concat(sourceFiles, (Stream) hclParser.parse(hclPaths, baseDir, ctx));
alreadyParsed.addAll(hclPaths);
}
if (!plainTextPaths.isEmpty()) {
sourceFiles = Stream.concat(sourceFiles, (Stream) plainTextParser.parse(plainTextPaths, baseDir, ctx));
alreadyParsed.addAll(plainTextPaths);
}
if (!quarkPaths.isEmpty()) {
sourceFiles = Stream.concat(sourceFiles, (Stream) quarkParser.parse(quarkPaths, baseDir, ctx));
alreadyParsed.addAll(quarkPaths);
}
return sourceFiles;
}
private boolean isOverSizeThreshold(long fileSize) {
return sizeThresholdMb > 0 && fileSize > sizeThresholdMb * 1024L * 1024L;
}
private boolean isExcluded(Path path) {
for (PathMatcher excluded : exclusions) {
if (excluded.matches(path)) {
return true;
}
}
// PathMather will not evaluate the path "pom.xml" to be matched by the pattern "**/pom.xml"
// This is counter-intuitive for most users and would otherwise require separate exclusions for files at the root and files in subdirectories
if(!path.isAbsolute() && !path.startsWith(File.separator)) {
return isExcluded(Paths.get("/" + path));
}
return false;
}
private boolean isParsedAsPlainText(Path path) {
if (!plainTextMasks.isEmpty()) {
Path computed = baseDir.relativize(path);
if (!computed.startsWith("/")) {
computed = Paths.get("/").resolve(computed);
}
for (PathMatcher matcher : plainTextMasks) {
if (matcher.matches(computed)) {
return true;
}
}
}
return false;
}
private boolean isIgnoredDirectory(Path searchDir, Path path) {
for (Path pathSegment : searchDir.relativize(path)) {
if (DEFAULT_ACCEPTED_DIRECTORIES.contains(pathSegment.toString())){
return false;
}
if (DEFAULT_IGNORED_DIRECTORIES.contains(pathSegment.toString())) {
return true;
}
}
return false;
}
}