diff --git a/spring-rewrite-commons-launcher/src/main/java/org/springframework/rewrite/parsers/ModuleParser.java b/spring-rewrite-commons-launcher/src/main/java/org/springframework/rewrite/parsers/ModuleParser.java deleted file mode 100644 index 39e5d0b..0000000 --- a/spring-rewrite-commons-launcher/src/main/java/org/springframework/rewrite/parsers/ModuleParser.java +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Copyright 2021 - 2023 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.rewrite.parsers; - -import org.jetbrains.annotations.NotNull; -import org.openrewrite.ExecutionContext; -import org.openrewrite.FileAttributes; -import org.openrewrite.Parser; -import org.openrewrite.SourceFile; -import org.openrewrite.internal.lang.Nullable; -import org.openrewrite.java.JavaParser; -import org.openrewrite.java.internal.JavaTypeCache; -import org.openrewrite.java.marker.JavaSourceSet; -import org.openrewrite.java.tree.J; -import org.openrewrite.java.tree.JavaType; -import org.openrewrite.marker.Generated; -import org.openrewrite.marker.Marker; -import org.openrewrite.marker.Markers; -import org.openrewrite.xml.tree.Xml; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.core.io.Resource; -import org.springframework.rewrite.utils.ResourceUtil; - -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; -import java.util.function.Predicate; -import java.util.function.Supplier; -import java.util.function.UnaryOperator; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.openrewrite.Tree.randomId; - -/** - * @author Fabian Krüger - */ -public class ModuleParser { - - private static final Logger LOGGER = LoggerFactory.getLogger(ModuleParser.class); - - /** - * Add {@link Marker}s to {@link SourceFile}. - */ - public UnaryOperator addProvenance(Path baseDir, List provenance, - @Nullable Collection generatedSources) { - return s -> { - Markers markers = s.getMarkers(); - for (Marker marker : provenance) { - markers = markers.addIfAbsent(marker); - } - if (generatedSources != null && generatedSources.contains(baseDir.resolve(s.getSourcePath()))) { - markers = markers.addIfAbsent(new Generated(randomId())); - } - return s.withMarkers(markers); - }; - } - - /** - * Parse Java sources and resources under {@code src/main} of current module. - */ - public SourceSetParsingResult processMainSources(Path baseDir, List resources, - Xml.Document moduleBuildFile, JavaParser.Builder javaParserBuilder, - RewriteResourceParser rp, List provenanceMarkers, Set alreadyParsed, - ExecutionContext executionContext, MavenProject currentProject) { - LOGGER.info("Processing main sources in module '%s'".formatted(currentProject.getProjectId())); - // FIXME: 945 - // Some annotation processors output generated sources to the /target directory. - // These are added for parsing but - // should be filtered out of the final SourceFile list. - - List mainJavaSources = new ArrayList<>(); - List javaSourcesInTarget = currentProject.getJavaSourcesInTarget(); // listJavaSources(resources, - // currentProject.getBasedir().resolve(currentProject.getBuildDirectory())); - List javaSourcesInMain = currentProject.getMainJavaSources(); // listJavaSources(resources, - // currentProject.getBasedir().resolve(currentProject.getSourceDirectory())); - mainJavaSources.addAll(javaSourcesInTarget); - mainJavaSources.addAll(javaSourcesInMain); - - LOGGER.info("[%s] Parsing source files".formatted(currentProject)); - - // FIXME 945 classpath - // - Resolve dependencies to non-reactor projects from Maven repository - // - Resolve dependencies to reactor projects by providing the sources - // javaParserBuilder.classpath(byte[]) - - // we're processing a module here. The classpath of the module consists of all - // declared dependencies and their transitive dependencies too. - // For dependencies to projects that belong to the current rector... - // They'd either need to be built with Maven before to guarantee that the jars are - // installed to local Maven repo. - // Or, the classpath must be created from the sources of the project. - - List dependencies = currentProject.getCompileClasspathElements(); - - javaParserBuilder.classpath(dependencies); - - JavaTypeCache typeCache = new JavaTypeCache(); - javaParserBuilder.typeCache(typeCache); - - Iterable inputs = mainJavaSources.stream().map(r -> { - FileAttributes fileAttributes = null; - Path path = ResourceUtil.getPath(r); - boolean isSynthetic = Files.exists(path); - Supplier inputStreamSupplier = () -> ResourceUtil.getInputStream(r); - Parser.Input input = new Parser.Input(path, fileAttributes, inputStreamSupplier, isSynthetic); - return input; - }).toList(); - - Set localClassesCp = new HashSet<>(); - JavaSourceSet javaSourceSet = sourceSet("main", dependencies, typeCache); - List cus = javaParserBuilder.build() - .parseInputs(inputs, baseDir, executionContext) - .peek(s -> { - ((J.CompilationUnit) s).getClasses() - .stream() - .map(J.ClassDeclaration::getType) - .forEach(localClassesCp::add); - - alreadyParsed.add(baseDir.resolve(s.getSourcePath())); - }) - .toList(); - - // TODO: This is a hack: - // Parsed java sources are not themselves on the classpath (here). - // The actual parsing happens when the stream is terminated (toList), - // therefore the toList() must be called before the parsed compilation units can - // be added to the classpath - List mainProjectProvenance = new ArrayList<>(provenanceMarkers); - javaSourceSet = appendToClasspath(localClassesCp, javaSourceSet); - mainProjectProvenance.add(javaSourceSet); - - List parsedJavaPaths = javaSourcesInTarget.stream().map(ResourceUtil::getPath).toList(); - Stream parsedJava = cus.stream() - .map(addProvenance(baseDir, mainProjectProvenance, parsedJavaPaths)); - LOGGER.debug( - "[%s] Scanned %d java source files in main scope.".formatted(currentProject, mainJavaSources.size())); - - // Filter out any generated source files from the returned list, as we do not want - // to apply the recipe to the - // generated files. - Path buildDirectory = Paths.get(currentProject.getBuildDirectory()); - List sourceFiles = parsedJava.filter(s -> !s.getSourcePath().startsWith(buildDirectory)) - .collect(Collectors.toCollection(ArrayList::new)); - - int sourcesParsedBefore = alreadyParsed.size(); - alreadyParsed.addAll(parsedJavaPaths); - List parsedResourceFiles = rp - .parse(currentProject.getModulePath().resolve("src/main/resources"), resources, alreadyParsed) - .map(addProvenance(baseDir, mainProjectProvenance, null)) - .toList(); - - LOGGER.debug("[%s] Scanned %d resource files in main scope.".formatted(currentProject, - (alreadyParsed.size() - sourcesParsedBefore))); - // Any resources parsed from "main/resources" should also have the main source set - // added to them. - sourceFiles.addAll(parsedResourceFiles); - return new SourceSetParsingResult(sourceFiles, javaSourceSet.getClasspath()); - } - - /** - * Add entries that don't exist in the classpath of {@code javaSourceSet} from - * {@code appendingClasspath}. - */ - @NotNull - private static JavaSourceSet appendToClasspath(Set appendingClasspath, - JavaSourceSet javaSourceSet) { - List curCp = javaSourceSet.getClasspath(); - appendingClasspath.forEach(f -> { - if (!curCp.contains(f)) { - curCp.add(f); - } - }); - javaSourceSet = javaSourceSet.withClasspath(new ArrayList<>(curCp)); - return javaSourceSet; - } - - @NotNull - private static JavaSourceSet sourceSet(String name, List dependencies, JavaTypeCache typeCache) { - return JavaSourceSet.build(name, dependencies, typeCache, false); - } - - /** - * Parse Java sources and resource files under {@code src/test}. - */ - public SourceSetParsingResult processTestSources(Path baseDir, Xml.Document moduleBuildFile, - JavaParser.Builder javaParserBuilder, RewriteResourceParser rp, - List provenanceMarkers, Set alreadyParsed, ExecutionContext executionContext, - MavenProject currentProject, List resources, List classpath) { - LOGGER.info("Processing test sources in module '%s'".formatted(currentProject.getProjectId())); - - List testDependencies = currentProject.getTestClasspathElements(); - - javaParserBuilder.classpath(testDependencies); - JavaTypeCache typeCache = new JavaTypeCache(); - javaParserBuilder.typeCache(typeCache); - - List testJavaSources = listJavaSources(resources, - currentProject.getBasedir().resolve(currentProject.getTestSourceDirectory())); - alreadyParsed.addAll(testJavaSources.stream().map(ResourceUtil::getPath).toList()); - - Iterable inputs = testJavaSources.stream() - .map(r -> new Parser.Input(ResourceUtil.getPath(r), () -> ResourceUtil.getInputStream(r))) - .toList(); - - final List localClassesCp = new ArrayList<>(); - List cus = javaParserBuilder.build() - .parseInputs(inputs, baseDir, executionContext) - .peek(s -> { - ((J.CompilationUnit) s).getClasses() - .stream() - .map(J.ClassDeclaration::getType) - .forEach(localClassesCp::add); - alreadyParsed.add(baseDir.resolve(s.getSourcePath())); - }) - .toList(); - - List markers = new ArrayList<>(provenanceMarkers); - - JavaSourceSet javaSourceSet = sourceSet("test", testDependencies, typeCache); - Set curClasspath = Stream.concat(classpath.stream(), localClassesCp.stream()) - .collect(Collectors.toSet()); - javaSourceSet = appendToClasspath(curClasspath, javaSourceSet); - markers.add(javaSourceSet); - Stream parsedJava = cus.stream().map(addProvenance(baseDir, markers, null)); - - LOGGER.debug( - "[%s] Scanned %d java source files in test scope.".formatted(currentProject, testJavaSources.size())); - Stream sourceFiles = parsedJava; - - // Any resources parsed from "test/resources" should also have the test source set - // added to them. - int sourcesParsedBefore = alreadyParsed.size(); - Stream parsedResourceFiles = rp - .parse(currentProject.getBasedir().resolve("src/test/resources"), resources, alreadyParsed) - .map(addProvenance(baseDir, markers, null)); - LOGGER.debug("[%s] Scanned %d resource files in test scope.".formatted(currentProject, - (alreadyParsed.size() - sourcesParsedBefore))); - sourceFiles = Stream.concat(sourceFiles, parsedResourceFiles); - List result = sourceFiles.toList(); - return new SourceSetParsingResult(result, javaSourceSet.getClasspath()); - } - - // FIXME: 945 take Java sources from resources - private static List listJavaSources(List resources, Path sourceDirectory) { - return resources.stream().filter(whenIn(sourceDirectory)).filter(whenFileNameEndsWithJava()).toList(); - } - - @NotNull - private static Predicate whenFileNameEndsWithJava() { - return p -> ResourceUtil.getPath(p).getFileName().toString().endsWith(".java"); - } - - @NotNull - private static Predicate whenIn(Path sourceDirectory) { - return r -> ResourceUtil.getPath(r).toString().startsWith(sourceDirectory.toString()); - } - -} diff --git a/spring-rewrite-commons-launcher/src/main/java/org/springframework/rewrite/parsers/RewriteParserConfiguration.java b/spring-rewrite-commons-launcher/src/main/java/org/springframework/rewrite/parsers/RewriteParserConfiguration.java index 8cad8b0..90cbc46 100644 --- a/spring-rewrite-commons-launcher/src/main/java/org/springframework/rewrite/parsers/RewriteParserConfiguration.java +++ b/spring-rewrite-commons-launcher/src/main/java/org/springframework/rewrite/parsers/RewriteParserConfiguration.java @@ -80,13 +80,8 @@ public class RewriteParserConfiguration { } @Bean - ModuleParser moduleParser() { - return new ModuleParser(); - } - - @Bean - MavenModuleParser mavenModuleParser(SpringRewriteProperties springRewriteProperties, ModuleParser moduleParser) { - return new MavenModuleParser(springRewriteProperties, moduleParser); + MavenModuleParser mavenModuleParser(SpringRewriteProperties springRewriteProperties) { + return new MavenModuleParser(springRewriteProperties); } @Bean diff --git a/spring-rewrite-commons-launcher/src/main/java/org/springframework/rewrite/parsers/maven/MavenModuleParser.java b/spring-rewrite-commons-launcher/src/main/java/org/springframework/rewrite/parsers/maven/MavenModuleParser.java index b2d2cf7..72bc7b1 100644 --- a/spring-rewrite-commons-launcher/src/main/java/org/springframework/rewrite/parsers/maven/MavenModuleParser.java +++ b/spring-rewrite-commons-launcher/src/main/java/org/springframework/rewrite/parsers/maven/MavenModuleParser.java @@ -15,11 +15,20 @@ */ package org.springframework.rewrite.parsers.maven; +import org.jetbrains.annotations.NotNull; import org.openrewrite.ExecutionContext; +import org.openrewrite.FileAttributes; +import org.openrewrite.Parser; import org.openrewrite.SourceFile; +import org.openrewrite.internal.lang.Nullable; import org.openrewrite.java.JavaParser; +import org.openrewrite.java.internal.JavaTypeCache; +import org.openrewrite.java.marker.JavaSourceSet; +import org.openrewrite.java.tree.J; import org.openrewrite.java.tree.JavaType; +import org.openrewrite.marker.Generated; import org.openrewrite.marker.Marker; +import org.openrewrite.marker.Markers; import org.openrewrite.style.NamedStyles; import org.openrewrite.tree.ParsingExecutionContextView; import org.openrewrite.xml.tree.Xml; @@ -27,134 +36,355 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.core.io.Resource; import org.springframework.rewrite.parsers.*; +import org.springframework.rewrite.utils.ResourceUtil; +import java.io.InputStream; import java.nio.charset.Charset; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.PathMatcher; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.nio.file.Paths; +import java.util.*; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.function.UnaryOperator; import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.openrewrite.Tree.randomId; + /** * @author Fabian Krüger */ public class MavenModuleParser { - private static final Logger LOGGER = LoggerFactory.getLogger(MavenProvenanceMarkerFactory.class); + private static final Logger LOGGER = LoggerFactory.getLogger(MavenProvenanceMarkerFactory.class); - private final SpringRewriteProperties springRewriteProperties; + private final SpringRewriteProperties springRewriteProperties; - private final ModuleParser mavenMojoProjectParserPrivateMethods; + public MavenModuleParser(SpringRewriteProperties springRewriteProperties) { + this.springRewriteProperties = springRewriteProperties; + } - public MavenModuleParser(SpringRewriteProperties springRewriteProperties, - ModuleParser mavenMojoProjectParserPrivateMethods) { - this.springRewriteProperties = springRewriteProperties; - this.mavenMojoProjectParserPrivateMethods = mavenMojoProjectParserPrivateMethods; - } + public List parseModuleSourceFiles(List resources, MavenProject currentProject, + Xml.Document moduleBuildFile, List provenanceMarkers, List styles, + ExecutionContext executionContext, Path baseDir) { - public List parseModuleSourceFiles(List resources, MavenProject currentProject, - Xml.Document moduleBuildFile, List provenanceMarkers, List styles, - ExecutionContext executionContext, Path baseDir) { + List sourceFiles = new ArrayList<>(); + // 146:149: get source encoding from maven + // TDOD: + // String s = + // moduleBuildFile.getMarkers().findFirst(MavenResolutionResult.class).get().getPom().getProperties().get("project.build.sourceEncoding"); + // if (mavenSourceEncoding != null) { + // ParsingExecutionContextView.view(ctx).setCharset(Charset.forName(mavenSourceEncoding.toString())); + // } + Object mavenSourceEncoding = currentProject.getProjectEncoding(); + if (mavenSourceEncoding != null) { + ParsingExecutionContextView.view(executionContext) + .setCharset(Charset.forName(mavenSourceEncoding.toString())); + } - List sourceFiles = new ArrayList<>(); - // 146:149: get source encoding from maven - // TDOD: - // String s = - // moduleBuildFile.getMarkers().findFirst(MavenResolutionResult.class).get().getPom().getProperties().get("project.build.sourceEncoding"); - // if (mavenSourceEncoding != null) { - // ParsingExecutionContextView.view(ctx).setCharset(Charset.forName(mavenSourceEncoding.toString())); - // } - Object mavenSourceEncoding = currentProject.getProjectEncoding(); - if (mavenSourceEncoding != null) { - ParsingExecutionContextView.view(executionContext) - .setCharset(Charset.forName(mavenSourceEncoding.toString())); - } + JavaParser.Builder javaParserBuilder = JavaParser.fromJavaVersion() + .styles(styles) + .logCompilationWarningsAndErrors(false); - JavaParser.Builder javaParserBuilder = JavaParser.fromJavaVersion() - .styles(styles) - .logCompilationWarningsAndErrors(false); + Path buildFilePath = currentProject.getBasedir().resolve(moduleBuildFile.getSourcePath()); + LOGGER.info("Parsing module " + buildFilePath); + // these paths will be ignored by ResourceParser + Set skipResourceScanDirs = pathsToOtherMavenProjects(currentProject, buildFilePath); + RewriteResourceParser rp = new RewriteResourceParser(baseDir, springRewriteProperties.getIgnoredPathPatterns(), + springRewriteProperties.getPlainTextMasks(), springRewriteProperties.getSizeThresholdMb(), + skipResourceScanDirs, javaParserBuilder.clone(), executionContext); - Path buildFilePath = currentProject.getBasedir().resolve(moduleBuildFile.getSourcePath()); - LOGGER.info("Parsing module " + buildFilePath); - // these paths will be ignored by ResourceParser - Set skipResourceScanDirs = pathsToOtherMavenProjects(currentProject, buildFilePath); - RewriteResourceParser rp = new RewriteResourceParser(baseDir, springRewriteProperties.getIgnoredPathPatterns(), - springRewriteProperties.getPlainTextMasks(), springRewriteProperties.getSizeThresholdMb(), - skipResourceScanDirs, javaParserBuilder.clone(), executionContext); + Set alreadyParsed = new HashSet<>(); + Path moduleBuildFilePath = baseDir.resolve(moduleBuildFile.getSourcePath()); + alreadyParsed.add(moduleBuildFilePath); + alreadyParsed.addAll(skipResourceScanDirs); + SourceSetParsingResult mainSourcesParsingResult = parseMainSources(baseDir, currentProject, moduleBuildFile, + resources, javaParserBuilder.clone(), rp, provenanceMarkers, alreadyParsed, executionContext); + SourceSetParsingResult testSourcesParsingResult = parseTestSources(baseDir, currentProject, moduleBuildFile, + javaParserBuilder.clone(), rp, provenanceMarkers, alreadyParsed, executionContext, resources, + mainSourcesParsingResult.classpath()); + // Collect the dirs of modules parsed in previous steps - Set alreadyParsed = new HashSet<>(); - Path moduleBuildFilePath = baseDir.resolve(moduleBuildFile.getSourcePath()); - alreadyParsed.add(moduleBuildFilePath); - alreadyParsed.addAll(skipResourceScanDirs); - SourceSetParsingResult mainSourcesParsingResult = parseMainSources(baseDir, currentProject, moduleBuildFile, - resources, javaParserBuilder.clone(), rp, provenanceMarkers, alreadyParsed, executionContext); - SourceSetParsingResult testSourcesParsingResult = parseTestSources(baseDir, currentProject, moduleBuildFile, - javaParserBuilder.clone(), rp, provenanceMarkers, alreadyParsed, executionContext, resources, - mainSourcesParsingResult.classpath()); - // Collect the dirs of modules parsed in previous steps + // parse other project resources + Stream parsedResourceFiles = rp.parse(moduleBuildFilePath.getParent(), resources, alreadyParsed) + // FIXME: handle generated sources + .map(addProvenance(baseDir, provenanceMarkers, null)); - // parse other project resources - Stream parsedResourceFiles = rp.parse(moduleBuildFilePath.getParent(), resources, alreadyParsed) - // FIXME: handle generated sources - .map(mavenMojoProjectParserPrivateMethods.addProvenance(baseDir, provenanceMarkers, null)); + List mainAndTestSources = mergeAndFilterExcluded(baseDir, + springRewriteProperties.getIgnoredPathPatterns(), mainSourcesParsingResult.sourceFiles(), + testSourcesParsingResult.sourceFiles()); + List resourceFilesList = parsedResourceFiles.toList(); + sourceFiles.addAll(mainAndTestSources); + sourceFiles.addAll(resourceFilesList); - List mainAndTestSources = mergeAndFilterExcluded(baseDir, - springRewriteProperties.getIgnoredPathPatterns(), mainSourcesParsingResult.sourceFiles(), - testSourcesParsingResult.sourceFiles()); - List resourceFilesList = parsedResourceFiles.toList(); - sourceFiles.addAll(mainAndTestSources); - sourceFiles.addAll(resourceFilesList); + return sourceFiles; + } - return sourceFiles; - } - private List mergeAndFilterExcluded(Path baseDir, Set exclusions, List mainSources, - List testSources) { - List pathMatchers = exclusions.stream() - .map(pattern -> baseDir.getFileSystem().getPathMatcher("glob:" + pattern)) - .toList(); - if (pathMatchers.isEmpty()) { - return Stream.concat(mainSources.stream(), testSources.stream()).toList(); - } - return new ArrayList<>(Stream.concat(mainSources.stream(), testSources.stream()) - .filter(s -> isNotExcluded(baseDir, pathMatchers, s)) - .toList()); - } + /** + * Add {@link Marker}s to {@link SourceFile}. + */ + public UnaryOperator addProvenance(Path baseDir, List provenance, + @Nullable Collection generatedSources) { + return s -> { + Markers markers = s.getMarkers(); + for (Marker marker : provenance) { + markers = markers.addIfAbsent(marker); + } + if (generatedSources != null && generatedSources.contains(baseDir.resolve(s.getSourcePath()))) { + markers = markers.addIfAbsent(new Generated(randomId())); + } + return s.withMarkers(markers); + }; + } - private static boolean isNotExcluded(Path baseDir, List exclusions, SourceFile s) { - return exclusions.stream() - .noneMatch(pm -> pm.matches(baseDir.resolve(s.getSourcePath()).toAbsolutePath().normalize())); - } + private List mergeAndFilterExcluded(Path baseDir, Set exclusions, List mainSources, + List testSources) { + List pathMatchers = exclusions.stream() + .map(pattern -> baseDir.getFileSystem().getPathMatcher("glob:" + pattern)) + .toList(); + if (pathMatchers.isEmpty()) { + return Stream.concat(mainSources.stream(), testSources.stream()).toList(); + } + return new ArrayList<>(Stream.concat(mainSources.stream(), testSources.stream()) + .filter(s -> isNotExcluded(baseDir, pathMatchers, s)) + .toList()); + } - private SourceSetParsingResult parseTestSources(Path baseDir, MavenProject mavenProject, - Xml.Document moduleBuildFile, JavaParser.Builder javaParserBuilder, - RewriteResourceParser rp, List provenanceMarkers, Set alreadyParsed, - ExecutionContext executionContext, List resources, List classpath) { - return mavenMojoProjectParserPrivateMethods.processTestSources(baseDir, moduleBuildFile, javaParserBuilder, rp, - provenanceMarkers, alreadyParsed, executionContext, mavenProject, resources, classpath); - } + private static boolean isNotExcluded(Path baseDir, List exclusions, SourceFile s) { + return exclusions.stream() + .noneMatch(pm -> pm.matches(baseDir.resolve(s.getSourcePath()).toAbsolutePath().normalize())); + } - /** - * - */ - private SourceSetParsingResult parseMainSources(Path baseDir, MavenProject mavenProject, - Xml.Document moduleBuildFile, List resources, - JavaParser.Builder javaParserBuilder, RewriteResourceParser rp, - List provenanceMarkers, Set alreadyParsed, ExecutionContext executionContext) { + private SourceSetParsingResult parseTestSources(Path baseDir, MavenProject mavenProject, + Xml.Document moduleBuildFile, JavaParser.Builder javaParserBuilder, + RewriteResourceParser rp, List provenanceMarkers, Set alreadyParsed, + ExecutionContext executionContext, List resources, List classpath) { + return processTestSources(baseDir, moduleBuildFile, javaParserBuilder, rp, + provenanceMarkers, alreadyParsed, executionContext, mavenProject, resources, classpath); + } - return mavenMojoProjectParserPrivateMethods.processMainSources(baseDir, resources, moduleBuildFile, - javaParserBuilder, rp, provenanceMarkers, alreadyParsed, executionContext, mavenProject); - } + /** + * + */ + private SourceSetParsingResult parseMainSources(Path baseDir, MavenProject mavenProject, + Xml.Document moduleBuildFile, List resources, + JavaParser.Builder javaParserBuilder, RewriteResourceParser rp, + List provenanceMarkers, Set alreadyParsed, ExecutionContext executionContext) { - private Set pathsToOtherMavenProjects(MavenProject mavenProject, Path moduleBuildFile) { - return mavenProject.getCollectedProjects() - .stream() - .filter(p -> !p.getFile().toPath().toString().equals(moduleBuildFile.toString())) - .map(p -> p.getFile().toPath().getParent()) - .collect(Collectors.toSet()); - } + return processMainSources(baseDir, resources, moduleBuildFile, + javaParserBuilder, rp, provenanceMarkers, alreadyParsed, executionContext, mavenProject); + } + + private Set pathsToOtherMavenProjects(MavenProject mavenProject, Path moduleBuildFile) { + return mavenProject.getCollectedProjects() + .stream() + .filter(p -> !p.getFile().toPath().toString().equals(moduleBuildFile.toString())) + .map(p -> p.getFile().toPath().getParent()) + .collect(Collectors.toSet()); + } + + + /** + * Parse Java sources and resources under {@code src/main} of current module. + */ + public SourceSetParsingResult processMainSources(Path baseDir, List resources, + Xml.Document moduleBuildFile, JavaParser.Builder javaParserBuilder, + RewriteResourceParser rp, List provenanceMarkers, Set alreadyParsed, + ExecutionContext executionContext, MavenProject currentProject) { + LOGGER.info("Processing main sources in module '%s'".formatted(currentProject.getProjectId())); + // FIXME: 945 + // Some annotation processors output generated sources to the /target directory. + // These are added for parsing but + // should be filtered out of the final SourceFile list. + + List mainJavaSources = new ArrayList<>(); + List javaSourcesInTarget = currentProject.getJavaSourcesInTarget(); // listJavaSources(resources, + // currentProject.getBasedir().resolve(currentProject.getBuildDirectory())); + List javaSourcesInMain = currentProject.getMainJavaSources(); // listJavaSources(resources, + // currentProject.getBasedir().resolve(currentProject.getSourceDirectory())); + mainJavaSources.addAll(javaSourcesInTarget); + mainJavaSources.addAll(javaSourcesInMain); + + LOGGER.info("[%s] Parsing source files".formatted(currentProject)); + + // FIXME 945 classpath + // - Resolve dependencies to non-reactor projects from Maven repository + // - Resolve dependencies to reactor projects by providing the sources + // javaParserBuilder.classpath(byte[]) + + // we're processing a module here. The classpath of the module consists of all + // declared dependencies and their transitive dependencies too. + // For dependencies to projects that belong to the current rector... + // They'd either need to be built with Maven before to guarantee that the jars are + // installed to local Maven repo. + // Or, the classpath must be created from the sources of the project. + + List dependencies = currentProject.getCompileClasspathElements(); + + javaParserBuilder.classpath(dependencies); + + JavaTypeCache typeCache = new JavaTypeCache(); + javaParserBuilder.typeCache(typeCache); + + Iterable inputs = mainJavaSources.stream().map(r -> { + FileAttributes fileAttributes = null; + Path path = ResourceUtil.getPath(r); + boolean isSynthetic = Files.exists(path); + Supplier inputStreamSupplier = () -> ResourceUtil.getInputStream(r); + Parser.Input input = new Parser.Input(path, fileAttributes, inputStreamSupplier, isSynthetic); + return input; + }).toList(); + + Set localClassesCp = new HashSet<>(); + JavaSourceSet javaSourceSet = sourceSet("main", dependencies, typeCache); + List cus = javaParserBuilder.build() + .parseInputs(inputs, baseDir, executionContext) + .peek(s -> { + ((J.CompilationUnit) s).getClasses() + .stream() + .map(J.ClassDeclaration::getType) + .forEach(localClassesCp::add); + + alreadyParsed.add(baseDir.resolve(s.getSourcePath())); + }) + .toList(); + + // TODO: This is a hack: + // Parsed java sources are not themselves on the classpath (here). + // The actual parsing happens when the stream is terminated (toList), + // therefore the toList() must be called before the parsed compilation units can + // be added to the classpath + List mainProjectProvenance = new ArrayList<>(provenanceMarkers); + javaSourceSet = appendToClasspath(localClassesCp, javaSourceSet); + mainProjectProvenance.add(javaSourceSet); + + List parsedJavaPaths = javaSourcesInTarget.stream().map(ResourceUtil::getPath).toList(); + Stream parsedJava = cus.stream() + .map(addProvenance(baseDir, mainProjectProvenance, parsedJavaPaths)); + LOGGER.debug( + "[%s] Scanned %d java source files in main scope.".formatted(currentProject, mainJavaSources.size())); + + // Filter out any generated source files from the returned list, as we do not want + // to apply the recipe to the + // generated files. + Path buildDirectory = Paths.get(currentProject.getBuildDirectory()); + List sourceFiles = parsedJava.filter(s -> !s.getSourcePath().startsWith(buildDirectory)) + .collect(Collectors.toCollection(ArrayList::new)); + + int sourcesParsedBefore = alreadyParsed.size(); + alreadyParsed.addAll(parsedJavaPaths); + List parsedResourceFiles = rp + .parse(currentProject.getModulePath().resolve("src/main/resources"), resources, alreadyParsed) + .map(addProvenance(baseDir, mainProjectProvenance, null)) + .toList(); + + LOGGER.debug("[%s] Scanned %d resource files in main scope.".formatted(currentProject, + (alreadyParsed.size() - sourcesParsedBefore))); + // Any resources parsed from "main/resources" should also have the main source set + // added to them. + sourceFiles.addAll(parsedResourceFiles); + return new SourceSetParsingResult(sourceFiles, javaSourceSet.getClasspath()); + } + + /** + * Add entries that don't exist in the classpath of {@code javaSourceSet} from + * {@code appendingClasspath}. + */ + @NotNull + private static JavaSourceSet appendToClasspath(Set appendingClasspath, + JavaSourceSet javaSourceSet) { + List curCp = javaSourceSet.getClasspath(); + appendingClasspath.forEach(f -> { + if (!curCp.contains(f)) { + curCp.add(f); + } + }); + javaSourceSet = javaSourceSet.withClasspath(new ArrayList<>(curCp)); + return javaSourceSet; + } + + @NotNull + private static JavaSourceSet sourceSet(String name, List dependencies, JavaTypeCache typeCache) { + return JavaSourceSet.build(name, dependencies, typeCache, false); + } + + /** + * Parse Java sources and resource files under {@code src/test}. + */ + public SourceSetParsingResult processTestSources(Path baseDir, Xml.Document moduleBuildFile, + JavaParser.Builder javaParserBuilder, RewriteResourceParser rp, + List provenanceMarkers, Set alreadyParsed, ExecutionContext executionContext, + MavenProject currentProject, List resources, List classpath) { + LOGGER.info("Processing test sources in module '%s'".formatted(currentProject.getProjectId())); + + List testDependencies = currentProject.getTestClasspathElements(); + + javaParserBuilder.classpath(testDependencies); + JavaTypeCache typeCache = new JavaTypeCache(); + javaParserBuilder.typeCache(typeCache); + + List testJavaSources = listJavaSources(resources, + currentProject.getBasedir().resolve(currentProject.getTestSourceDirectory())); + alreadyParsed.addAll(testJavaSources.stream().map(ResourceUtil::getPath).toList()); + + Iterable inputs = testJavaSources.stream() + .map(r -> new Parser.Input(ResourceUtil.getPath(r), () -> ResourceUtil.getInputStream(r))) + .toList(); + + final List localClassesCp = new ArrayList<>(); + List cus = javaParserBuilder.build() + .parseInputs(inputs, baseDir, executionContext) + .peek(s -> { + ((J.CompilationUnit) s).getClasses() + .stream() + .map(J.ClassDeclaration::getType) + .forEach(localClassesCp::add); + alreadyParsed.add(baseDir.resolve(s.getSourcePath())); + }) + .toList(); + + List markers = new ArrayList<>(provenanceMarkers); + + JavaSourceSet javaSourceSet = sourceSet("test", testDependencies, typeCache); + Set curClasspath = Stream.concat(classpath.stream(), localClassesCp.stream()) + .collect(Collectors.toSet()); + javaSourceSet = appendToClasspath(curClasspath, javaSourceSet); + markers.add(javaSourceSet); + Stream parsedJava = cus.stream().map(addProvenance(baseDir, markers, null)); + + LOGGER.debug( + "[%s] Scanned %d java source files in test scope.".formatted(currentProject, testJavaSources.size())); + Stream sourceFiles = parsedJava; + + // Any resources parsed from "test/resources" should also have the test source set + // added to them. + int sourcesParsedBefore = alreadyParsed.size(); + Stream parsedResourceFiles = rp + .parse(currentProject.getBasedir().resolve("src/test/resources"), resources, alreadyParsed) + .map(addProvenance(baseDir, markers, null)); + LOGGER.debug("[%s] Scanned %d resource files in test scope.".formatted(currentProject, + (alreadyParsed.size() - sourcesParsedBefore))); + sourceFiles = Stream.concat(sourceFiles, parsedResourceFiles); + List result = sourceFiles.toList(); + return new SourceSetParsingResult(result, javaSourceSet.getClasspath()); + } + + // FIXME: 945 take Java sources from resources + private static List listJavaSources(List resources, Path sourceDirectory) { + return resources.stream().filter(whenIn(sourceDirectory)).filter(whenFileNameEndsWithJava()).toList(); + } + + @NotNull + private static Predicate whenFileNameEndsWithJava() { + return p -> ResourceUtil.getPath(p).getFileName().toString().endsWith(".java"); + } + + @NotNull + private static Predicate whenIn(Path sourceDirectory) { + return r -> ResourceUtil.getPath(r).toString().startsWith(sourceDirectory.toString()); + } } diff --git a/spring-rewrite-commons-launcher/src/test/java/org/springframework/rewrite/parsers/RewriteProjectParserTest.java b/spring-rewrite-commons-launcher/src/test/java/org/springframework/rewrite/parsers/RewriteProjectParserTest.java index 824f950..51f44c0 100644 --- a/spring-rewrite-commons-launcher/src/test/java/org/springframework/rewrite/parsers/RewriteProjectParserTest.java +++ b/spring-rewrite-commons-launcher/src/test/java/org/springframework/rewrite/parsers/RewriteProjectParserTest.java @@ -91,12 +91,10 @@ class RewriteProjectParserTest { void parseSimpleMavenProject(@TempDir Path tempDir) throws PlexusCipherException { Path basePath = tempDir; SpringRewriteProperties springRewriteProperties = new SpringRewriteProperties(); - ModuleParser mavenMojoParserPrivateMethods = new ModuleParser(); ExecutionContext executionContext = new InMemoryExecutionContext(t -> { throw new RuntimeException(t); }); - MavenModuleParser mavenModuleParser = new MavenModuleParser(springRewriteProperties, - mavenMojoParserPrivateMethods); + MavenModuleParser mavenModuleParser = new MavenModuleParser(springRewriteProperties); ProjectMetadata projectMetadata = new ProjectMetadata(); MavenSettingsInitializer mavenSettingsInitializer = new MavenSettingsInitializer(executionContext, projectMetadata);