From 9017c020aa1efc3d9c62cb8429fb7945485ebaad Mon Sep 17 00:00:00 2001 From: Croway Date: Mon, 18 Mar 2024 17:35:55 +0100 Subject: [PATCH] Validate routes in transitive dependencies Add docs and logs Add docs --- .../main/docs/camel-report-maven-plugin.adoc | 34 +++++ .../org/apache/camel/maven/ValidateMojo.java | 144 ++++++++++++++++-- 2 files changed, 166 insertions(+), 12 deletions(-) diff --git a/catalog/camel-report-maven-plugin/src/main/docs/camel-report-maven-plugin.adoc b/catalog/camel-report-maven-plugin/src/main/docs/camel-report-maven-plugin.adoc index 9da9525fcf2b7..c0cf076b7c230 100644 --- a/catalog/camel-report-maven-plugin/src/main/docs/camel-report-maven-plugin.adoc +++ b/catalog/camel-report-maven-plugin/src/main/docs/camel-report-maven-plugin.adoc @@ -151,6 +151,9 @@ The maven plugin *validate* goal supports the following options which can be con | directOrSedaPairCheck | true |Whether to validate direct/seda endpoints sending to non existing consumers. | configurationFiles | application.properties | Location of configuration files to validate. The default is application.properties. Multiple values can be separated by comma and use wildcard pattern matching. | showAll | false | Whether to show all endpoints and simple expressions (both invalid and valid). +| downloadTransitiveArtifacts | false | When sourcesArtifacts are declared, this flag can be used to download transitive dependencies, carefully enable this flag since it will try to download the whole dependency tree. +| sourcesArtifacts | | List of sources transitive dependencies that contain camel routes, this option can be used to download extra dependencies that contain camel route that your project may depend on. +| extraMavenRepositories | | List of extra maven repositories. |=== For example to turn on ignoring usage of deprecated options from the command line, you can run: @@ -171,6 +174,37 @@ $cd myproject $mvn org.apache.camel:camel-report-maven-plugin:3.0.0:validate -Dcamel.includeTest=true ---- +=== Validate Apache Camel routes in transitive dependencies + +If your routes use `direct` or `seda` endpoints that are not present in the current project, but the routes are declared into a dependency of your project, you can edit the plugin configuration accordingly so that these routes can be taken into account by the Camel validate plugin. +In particular, in order to use the validate plugin with transitive dependencies, *sources jars are needed*, for example: + +* Given the following Camel route `from("direct:in").to("direct:out")` defined in the current project +* The route `from("direct:out")` is declared into a dependency of your project, for example `my.company:routes-dependency:1.0` +* If `routes-dependency` sources are released into a maven repository, the following plugin configuration can be used: + +```xml + + org.apache.camel + camel-report-maven-plugin + + + package + + validate + + + + my.company:routes-dependency:jar:sources:1.0 + + + http://internal.repo:8080/maven + + + + + +``` == camel-report:route-coverage diff --git a/catalog/camel-report-maven-plugin/src/main/java/org/apache/camel/maven/ValidateMojo.java b/catalog/camel-report-maven-plugin/src/main/java/org/apache/camel/maven/ValidateMojo.java index 0a7fc18c66d36..0e7436ff43187 100644 --- a/catalog/camel-report-maven-plugin/src/main/java/org/apache/camel/maven/ValidateMojo.java +++ b/catalog/camel-report-maven-plugin/src/main/java/org/apache/camel/maven/ValidateMojo.java @@ -16,23 +16,21 @@ */ package org.apache.camel.maven; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileReader; -import java.io.InputStream; -import java.io.LineNumberReader; -import java.util.ArrayList; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Properties; -import java.util.Set; +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; import java.util.stream.Collectors; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; import org.apache.camel.catalog.CamelCatalog; import org.apache.camel.catalog.ConfigurationPropertiesValidationResult; import org.apache.camel.catalog.DefaultCamelCatalog; import org.apache.camel.catalog.EndpointValidationResult; import org.apache.camel.catalog.LanguageValidationResult; +import org.apache.camel.catalog.common.FileUtil; import org.apache.camel.catalog.lucene.LuceneSuggestionStrategy; import org.apache.camel.catalog.maven.MavenVersionManager; import org.apache.camel.parser.RouteBuilderParser; @@ -41,8 +39,12 @@ import org.apache.camel.parser.model.CamelRouteDetails; import org.apache.camel.parser.model.CamelSimpleExpressionDetails; import org.apache.camel.support.PatternHelper; +import org.apache.camel.tooling.maven.MavenArtifact; +import org.apache.camel.tooling.maven.MavenDownloaderImpl; +import org.apache.camel.tooling.maven.MavenResolutionException; import org.apache.camel.util.OrderedProperties; import org.apache.camel.util.StringHelper; +import org.apache.commons.io.IOUtils; import org.apache.maven.model.Dependency; import org.apache.maven.model.Resource; import org.apache.maven.plugin.MojoExecutionException; @@ -95,6 +97,21 @@ public class ValidateMojo extends AbstractExecMojo { @Parameter(property = "camel.includeJava", defaultValue = "true") private boolean includeJava; + /** + * List of extra maven repositories + */ + @Parameter(property = "camel.extraRepositories") + private String[] extraMavenRepositories; + + /** + * List of sources transitive dependencies that contains camel routes + */ + @Parameter(property = "camel.sourcesArtifacts") + private String[] sourcesArtifacts; + + @Parameter(defaultValue = "${project.build.directory}") + private String projectBuildDir; + /** * Whether to include XML files to be validated for invalid Camel endpoints */ @@ -173,6 +190,13 @@ public class ValidateMojo extends AbstractExecMojo { @Parameter(property = "camel.directOrSedaPairCheck", defaultValue = "true") private boolean directOrSedaPairCheck; + /** + * When sourcesArtifacts are declared, choose to download transitive artifacts or not carefully enable this flag + * since it will try to download the whole dependency tree + */ + @Parameter(property = "camel.downloadTransitiveArtifacts", defaultValue = "false") + private boolean downloadTransitiveArtifacts; + /** * Location of configuration files to validate. The default is application.properties Multiple values can be * separated by comma and use wildcard pattern matching. @@ -186,8 +210,87 @@ public class ValidateMojo extends AbstractExecMojo { @Parameter(defaultValue = "${repositorySystemSession}", readonly = true) private RepositorySystemSession repositorySystemSession; + /** + * javaFiles in memory cache, useful for multi modules maven project + */ + private static Set javaFiles = new LinkedHashSet<>(); + /** + * xmlFiles in memory cache, useful for multi modules maven project + */ + private static Set xmlFiles = new LinkedHashSet<>(); + + private static Set downloadedArtifacts = new LinkedHashSet<>(); + @Override public void execute() throws MojoExecutionException { + // Download extra sources only if artifacts sources are defined and the current project is not a parent project + if (!"pom".equals(project.getPackaging()) && sourcesArtifacts != null && sourcesArtifacts.length > 0) { + // setup MavenDownloader, it will be used to download and locate artifacts declared via sourcesArtifacts + + List artifacts = Arrays.asList(sourcesArtifacts); + + artifacts + .parallelStream() + .forEach(artifact -> { + if (!artifact.contains(":sources:")) { + getLog().warn("The artifact " + artifact + + " does not contain sources classifier, and may be excluded in future releases"); + } + }); + + try (MavenDownloaderImpl downloader + = new MavenDownloaderImpl(repositorySystem, repositorySystemSession, getSession().getSettings())) { + downloader.init(); + Set repositorySet = Arrays.stream(extraMavenRepositories) + .collect(Collectors.toSet()); + List artifactList = new ArrayList<>(artifacts); + + // Remove already downloaded Artifacts + artifactList.removeAll(downloadedArtifacts); + + if (!artifactList.isEmpty()) { + getLog().info("Downloading the following artifacts: " + artifactList); + List mavenSourcesArtifacts + = downloader.resolveArtifacts(artifactList, repositorySet, downloadTransitiveArtifacts, false); + + // Create folder into the target folder that will be used to unzip + // the downloaded artifacts + Path extraSourcesPath = Paths.get(projectBuildDir, "camel-validate-sources"); + if (!Files.exists(extraSourcesPath)) { + Files.createDirectories(extraSourcesPath); + } + + // Unzip all the downloaded artifacts and add javas and xmls files into the cache + for (MavenArtifact artifact : mavenSourcesArtifacts) { + StringBuilder sb = new StringBuilder(); + sb.append(artifact.getGav().getGroupId()).append(":") + .append(artifact.getGav().getArtifactId()).append(":") + .append(artifact.getGav().getPackaging()).append(":") + .append(artifact.getGav().getClassifier()).append(":") + .append(artifact.getGav().getVersion()); + // Avoid downloading the same dependency multiple times + downloadedArtifacts.add(sb.toString()); + + Path target = extraSourcesPath.resolve(artifact.getGav().getArtifactId()); + getLog().info("Unzipping the artifact: " + artifact + " to " + target); + if (Files.exists(target)) { + continue; + } + + unzipArtifact(artifact, target); + + FileUtil.findJavaFiles(target.toFile(), javaFiles); + FileUtil.findXmlFiles(target.toFile(), xmlFiles); + } + } + } catch (IOException e) { + throw new MojoExecutionException(e); + } catch (MavenResolutionException e) { + // missing artifact, log and proceed + getLog().warn(e.getMessage()); + } + } + CamelCatalog catalog = new DefaultCamelCatalog(); // add activemq as known component catalog.addComponent("activemq", "org.apache.activemq.camel.component.ActiveMQComponent"); @@ -229,6 +332,25 @@ public void execute() throws MojoExecutionException { doExecuteConfigurationFiles(catalog); } + private static void unzipArtifact(MavenArtifact artifact, Path target) throws IOException { + try (ZipFile zipFile = new ZipFile(artifact.getFile().toPath().toFile())) { + Enumeration entries = zipFile.entries(); + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + File entryDestination = new File(target.toString(), entry.getName()); + if (entry.isDirectory()) { + entryDestination.mkdirs(); + } else { + entryDestination.getParentFile().mkdirs(); + try (InputStream in = zipFile.getInputStream(entry); + OutputStream out = new FileOutputStream(entryDestination)) { + IOUtils.copy(in, out); + } + } + } + } + } + protected void doExecuteConfigurationFiles(CamelCatalog catalog) throws MojoExecutionException { Set propertiesFiles = new LinkedHashSet<>(); for (Resource dir : project.getResources()) { @@ -390,8 +512,6 @@ protected void doExecuteRoutes(CamelCatalog catalog) throws MojoExecutionExcepti List endpoints = new ArrayList<>(); List simpleExpressions = new ArrayList<>(); List routeIds = new ArrayList<>(); - Set javaFiles = new LinkedHashSet<>(); - Set xmlFiles = new LinkedHashSet<>(); // find all java route builder classes findJavaRouteBuilderClasses(javaFiles, includeJava, includeTest, project);