Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Validate routes in transitive dependencies #13570

Merged
merged 1 commit into from Mar 21, 2024
Merged

Validate routes in transitive dependencies #13570

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
34 changes: 34 additions & 0 deletions catalog/camel-report-maven-plugin/src/main/docs/camel-report-maven-plugin.adoc
View file
Open in desktop
Expand Up @@ -151,6 +151,9 @@ The maven plugin *validate* goal supports the following options which can be con
| directOrSedaPairCheck | true |Whether to validate direct/seda endpoints sending to non existing consumers.
| configurationFiles | application.properties | Location of configuration files to validate. The default is application.properties. Multiple values can be separated by comma and use wildcard pattern matching.
| showAll | false | Whether to show all endpoints and simple expressions (both invalid and valid).
| downloadTransitiveArtifacts | false | When sourcesArtifacts are declared, this flag can be used to download transitive dependencies, carefully enable this flag since it will try to download the whole dependency tree.
| sourcesArtifacts | | List of sources transitive dependencies that contain camel routes, this option can be used to download extra dependencies that contain camel route that your project may depend on.
| extraMavenRepositories | | List of extra maven repositories.
|===

For example to turn on ignoring usage of deprecated options from the command line, you can run:
Expand All @@ -171,6 +174,37 @@ $cd myproject
$mvn org.apache.camel:camel-report-maven-plugin:3.0.0:validate -Dcamel.includeTest=true
----

=== Validate Apache Camel routes in transitive dependencies

If your routes use `direct` or `seda` endpoints that are not present in the current project, but the routes are declared into a dependency of your project, you can edit the plugin configuration accordingly so that these routes can be taken into account by the Camel validate plugin.
In particular, in order to use the validate plugin with transitive dependencies, *sources jars are needed*, for example:

* Given the following Camel route `from("direct:in").to("direct:out")` defined in the current project
* The route `from("direct:out")` is declared into a dependency of your project, for example `my.company:routes-dependency:1.0`
* If `routes-dependency` sources are released into a maven repository, the following plugin configuration can be used:

```xml
<plugin>
<groupId>org.apache.camel</groupId>
<artifactId>camel-report-maven-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>validate</goal>
</goals>
<configuration>
<sourcesArtifacts>
<sourcesArtifact>my.company:routes-dependency:jar:sources:1.0</sourcesArtifact>
</sourcesArtifacts>
<extraMavenRepositories>
<extraMavenRepository>http://internal.repo:8080/maven</extraMavenRepository>
</extraMavenRepositories>
</configuration>
</execution>
</executions>
</plugin>
```

== camel-report:route-coverage

Expand Down
144 changes: 132 additions & 12 deletions catalog/camel-report-maven-plugin/src/main/java/org/apache/camel/maven/ValidateMojo.java
View file
Open in desktop
Expand Up @@ -16,23 +16,21 @@
*/
package org.apache.camel.maven;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.InputStream;
import java.io.LineNumberReader;
import java.util.ArrayList;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

import org.apache.camel.catalog.CamelCatalog;
import org.apache.camel.catalog.ConfigurationPropertiesValidationResult;
import org.apache.camel.catalog.DefaultCamelCatalog;
import org.apache.camel.catalog.EndpointValidationResult;
import org.apache.camel.catalog.LanguageValidationResult;
import org.apache.camel.catalog.common.FileUtil;
import org.apache.camel.catalog.lucene.LuceneSuggestionStrategy;
import org.apache.camel.catalog.maven.MavenVersionManager;
import org.apache.camel.parser.RouteBuilderParser;
Expand All @@ -41,8 +39,12 @@
import org.apache.camel.parser.model.CamelRouteDetails;
import org.apache.camel.parser.model.CamelSimpleExpressionDetails;
import org.apache.camel.support.PatternHelper;
import org.apache.camel.tooling.maven.MavenArtifact;
import org.apache.camel.tooling.maven.MavenDownloaderImpl;
import org.apache.camel.tooling.maven.MavenResolutionException;
import org.apache.camel.util.OrderedProperties;
import org.apache.camel.util.StringHelper;
import org.apache.commons.io.IOUtils;
import org.apache.maven.model.Dependency;
import org.apache.maven.model.Resource;
import org.apache.maven.plugin.MojoExecutionException;
Expand Down Expand Up @@ -95,6 +97,21 @@ public class ValidateMojo extends AbstractExecMojo {
@Parameter(property = "camel.includeJava", defaultValue = "true")
private boolean includeJava;

/**
* List of extra maven repositories
*/
@Parameter(property = "camel.extraRepositories")
private String[] extraMavenRepositories;

/**
* List of sources transitive dependencies that contains camel routes
*/
@Parameter(property = "camel.sourcesArtifacts")
private String[] sourcesArtifacts;

@Parameter(defaultValue = "${project.build.directory}")
private String projectBuildDir;

/**
* Whether to include XML files to be validated for invalid Camel endpoints
*/
Expand Down Expand Up @@ -173,6 +190,13 @@ public class ValidateMojo extends AbstractExecMojo {
@Parameter(property = "camel.directOrSedaPairCheck", defaultValue = "true")
private boolean directOrSedaPairCheck;

/**
* When sourcesArtifacts are declared, choose to download transitive artifacts or not carefully enable this flag
* since it will try to download the whole dependency tree
*/
@Parameter(property = "camel.downloadTransitiveArtifacts", defaultValue = "false")
private boolean downloadTransitiveArtifacts;

/**
* Location of configuration files to validate. The default is application.properties Multiple values can be
* separated by comma and use wildcard pattern matching.
Expand All @@ -186,8 +210,87 @@ public class ValidateMojo extends AbstractExecMojo {
@Parameter(defaultValue = "${repositorySystemSession}", readonly = true)
private RepositorySystemSession repositorySystemSession;

/**
* javaFiles in memory cache, useful for multi modules maven project
*/
private static Set<File> javaFiles = new LinkedHashSet<>();
/**
* xmlFiles in memory cache, useful for multi modules maven project
*/
private static Set<File> xmlFiles = new LinkedHashSet<>();

private static Set<String> downloadedArtifacts = new LinkedHashSet<>();

@Override
public void execute() throws MojoExecutionException {
// Download extra sources only if artifacts sources are defined and the current project is not a parent project
if (!"pom".equals(project.getPackaging()) && sourcesArtifacts != null && sourcesArtifacts.length > 0) {
// setup MavenDownloader, it will be used to download and locate artifacts declared via sourcesArtifacts

List<String> artifacts = Arrays.asList(sourcesArtifacts);

artifacts
.parallelStream()
.forEach(artifact -> {
if (!artifact.contains(":sources:")) {
getLog().warn("The artifact " + artifact
+ " does not contain sources classifier, and may be excluded in future releases");
}
});

try (MavenDownloaderImpl downloader
= new MavenDownloaderImpl(repositorySystem, repositorySystemSession, getSession().getSettings())) {
downloader.init();
Set<String> repositorySet = Arrays.stream(extraMavenRepositories)
.collect(Collectors.toSet());
List<String> artifactList = new ArrayList<>(artifacts);

// Remove already downloaded Artifacts
artifactList.removeAll(downloadedArtifacts);

if (!artifactList.isEmpty()) {
getLog().info("Downloading the following artifacts: " + artifactList);
List<MavenArtifact> mavenSourcesArtifacts
= downloader.resolveArtifacts(artifactList, repositorySet, downloadTransitiveArtifacts, false);

// Create folder into the target folder that will be used to unzip
// the downloaded artifacts
Path extraSourcesPath = Paths.get(projectBuildDir, "camel-validate-sources");
if (!Files.exists(extraSourcesPath)) {
Files.createDirectories(extraSourcesPath);
}

// Unzip all the downloaded artifacts and add javas and xmls files into the cache
for (MavenArtifact artifact : mavenSourcesArtifacts) {
StringBuilder sb = new StringBuilder();
sb.append(artifact.getGav().getGroupId()).append(":")
.append(artifact.getGav().getArtifactId()).append(":")
.append(artifact.getGav().getPackaging()).append(":")
.append(artifact.getGav().getClassifier()).append(":")
.append(artifact.getGav().getVersion());
// Avoid downloading the same dependency multiple times
downloadedArtifacts.add(sb.toString());

Path target = extraSourcesPath.resolve(artifact.getGav().getArtifactId());
getLog().info("Unzipping the artifact: " + artifact + " to " + target);
if (Files.exists(target)) {
continue;
}

unzipArtifact(artifact, target);

FileUtil.findJavaFiles(target.toFile(), javaFiles);
FileUtil.findXmlFiles(target.toFile(), xmlFiles);
}
}
} catch (IOException e) {
throw new MojoExecutionException(e);
} catch (MavenResolutionException e) {
// missing artifact, log and proceed
getLog().warn(e.getMessage());
}
}

CamelCatalog catalog = new DefaultCamelCatalog();
// add activemq as known component
catalog.addComponent("activemq", "org.apache.activemq.camel.component.ActiveMQComponent");
Expand Down Expand Up @@ -229,6 +332,25 @@ public void execute() throws MojoExecutionException {
doExecuteConfigurationFiles(catalog);
}

private static void unzipArtifact(MavenArtifact artifact, Path target) throws IOException {
try (ZipFile zipFile = new ZipFile(artifact.getFile().toPath().toFile())) {
Enumeration<? extends ZipEntry> entries = zipFile.entries();
while (entries.hasMoreElements()) {
ZipEntry entry = entries.nextElement();
File entryDestination = new File(target.toString(), entry.getName());
if (entry.isDirectory()) {
entryDestination.mkdirs();
} else {
entryDestination.getParentFile().mkdirs();
try (InputStream in = zipFile.getInputStream(entry);
OutputStream out = new FileOutputStream(entryDestination)) {
IOUtils.copy(in, out);
}
}
}
}
}

protected void doExecuteConfigurationFiles(CamelCatalog catalog) throws MojoExecutionException {
Set<File> propertiesFiles = new LinkedHashSet<>();
for (Resource dir : project.getResources()) {
Expand Down Expand Up @@ -390,8 +512,6 @@ protected void doExecuteRoutes(CamelCatalog catalog) throws MojoExecutionExcepti
List<CamelEndpointDetails> endpoints = new ArrayList<>();
List<CamelSimpleExpressionDetails> simpleExpressions = new ArrayList<>();
List<CamelRouteDetails> routeIds = new ArrayList<>();
Set<File> javaFiles = new LinkedHashSet<>();
Set<File> xmlFiles = new LinkedHashSet<>();

// find all java route builder classes
findJavaRouteBuilderClasses(javaFiles, includeJava, includeTest, project);
Expand Down