diff --git a/build.gradle b/build.gradle index 64c09ee3..fff79f90 100644 --- a/build.gradle +++ b/build.gradle @@ -73,8 +73,8 @@ repositories { } def buildInfoVersion = '2.43.9' -def idePluginsCommonVersion = '2.4.5' -def jacksonVersion = '2.18.6' +def idePluginsCommonVersion = '2.5.0' +def jacksonVersion = '2.21.7' dependencies { implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: jacksonVersion @@ -99,6 +99,12 @@ dependencies { testImplementation group: 'org.mockito', name: 'mockito-core', version: '4.2.0' } +tasks.withType(Test).configureEach { + // The IDE's app.jar bundles an older Jackson; tests load the plugin's Jackson first, as the IDE does. + def jacksonJars = classpath.filter { it.name.startsWith('jackson-') } + classpath = jacksonJars + (classpath - jacksonJars) +} + test { scanForTestClasses false include "**/*Test.class" diff --git a/src/main/java/com/jfrog/ide/idea/scan/ScannerBase.java b/src/main/java/com/jfrog/ide/idea/scan/ScannerBase.java index 8f3c2c81..6d4de065 100644 --- a/src/main/java/com/jfrog/ide/idea/scan/ScannerBase.java +++ b/src/main/java/com/jfrog/ide/idea/scan/ScannerBase.java @@ -18,7 +18,6 @@ import com.intellij.psi.PsiFile; import com.jfrog.ide.common.configuration.ServerConfig; import com.jfrog.ide.common.deptree.DepTree; -import com.jfrog.ide.common.deptree.DepTreeNode; import com.jfrog.ide.common.log.ProgressIndicator; import com.jfrog.ide.common.nodes.DependencyNode; import com.jfrog.ide.common.nodes.FileTreeNode; @@ -187,7 +186,7 @@ private void scanAndUpdate(ProgressIndicator indicator) { protected List buildImpactGraph(Map vulnerableDependencies, DepTree depTree) throws IOException { Map> parents = getParents(depTree); - ImpactTreeBuilder.populateImpactTrees(vulnerableDependencies, parents, depTree.rootId()); + ImpactTreeBuilder.populateImpactTrees(vulnerableDependencies, depTree); return groupDependenciesToDescriptorNodes(vulnerableDependencies.values(), depTree, parents); } @@ -199,15 +198,7 @@ protected List buildImpactGraph(Map vulner * @return a map of nodes from the {@link DepTree} amd each one's parents */ static Map> getParents(DepTree depTree) { - Map> parents = new HashMap<>(); - for (Map.Entry node : depTree.nodes().entrySet()) { - String parentId = node.getKey(); - for (String childId : node.getValue().getChildren()) { - parents.putIfAbsent(childId, new HashSet<>()); - parents.get(childId).add(parentId); - } - } - return parents; + return ImpactTreeBuilder.getParents(depTree.nodes()); } /** diff --git a/src/main/java/com/jfrog/ide/idea/scan/utils/ImpactTreeBuilder.java b/src/main/java/com/jfrog/ide/idea/scan/utils/ImpactTreeBuilder.java index 2f3494ad..50fa408e 100644 --- a/src/main/java/com/jfrog/ide/idea/scan/utils/ImpactTreeBuilder.java +++ b/src/main/java/com/jfrog/ide/idea/scan/utils/ImpactTreeBuilder.java @@ -1,5 +1,8 @@ package com.jfrog.ide.idea.scan.utils; +import com.jfrog.ide.common.deptree.DepTree; +import com.jfrog.ide.common.deptree.DepTreeModule; +import com.jfrog.ide.common.deptree.DepTreeNode; import com.jfrog.ide.common.nodes.DependencyNode; import com.jfrog.ide.common.nodes.DescriptorFileTreeNode; import com.jfrog.ide.common.nodes.FileTreeNode; @@ -9,6 +12,42 @@ import java.util.*; public class ImpactTreeBuilder { + /** + * Builds impact paths for {@link DependencyNode} objects, walking each module's own tree when the project has modules. + * + * @param vulnerableDependencies a map of component IDs and the {@link DependencyNode} object matching each of them + * @param depTree the project's dependency tree + */ + public static void populateImpactTrees(Map vulnerableDependencies, DepTree depTree) { + if (depTree.modules().isEmpty()) { + populateImpactTrees(vulnerableDependencies, getParents(depTree.nodes()), depTree.rootId()); + } else { + for (DepTreeModule module : depTree.modules()) { + String projectRootId = module.rootId().equals(depTree.rootId()) ? null : depTree.rootId(); + populateImpactTrees(vulnerableDependencies, getParents(module.nodes()), module.rootId(), projectRootId); + } + } + addMissingImpactTrees(vulnerableDependencies, depTree.rootId()); + } + + /** + * Find the parents of each node. Nodes without parents (the root) don't appear in the returned map. + * + * @param nodes a map of component IDs and their {@link DepTreeNode} + * @return a map of the nodes and each one's parents + */ + public static Map> getParents(Map nodes) { + Map> parents = new HashMap<>(); + for (Map.Entry node : nodes.entrySet()) { + String parentId = node.getKey(); + for (String childId : node.getValue().getChildren()) { + parents.putIfAbsent(childId, new HashSet<>()); + parents.get(childId).add(parentId); + } + } + return parents; + } + /** * Builds impact paths for {@link DependencyNode} objects. * @@ -18,8 +57,20 @@ public class ImpactTreeBuilder { * @param rootId the project's root component ID */ public static void populateImpactTrees(Map vulnerableDependencies, Map> parents, String rootId) { + populateImpactTrees(vulnerableDependencies, parents, rootId, null); + } + + /** + * Builds impact paths for {@link DependencyNode} objects within a single module of the project. + * + * @param vulnerableDependencies a map of component IDs and the {@link DependencyNode} object matching each of them + * @param parents a map of the module's dependencies and their parents + * @param rootId the module's root component ID, where every impact path ends + * @param projectRootId the project's root component ID to prepend to each path, or null + */ + private static void populateImpactTrees(Map vulnerableDependencies, Map> parents, String rootId, String projectRootId) { for (DependencyNode vulnDep : vulnerableDependencies.values()) { - walkParents(vulnDep, parents, rootId, Collections.singletonList(vulnDep.getComponentIdWithoutPrefix())); + walkParents(vulnDep, parents, rootId, Collections.singletonList(vulnDep.getComponentIdWithoutPrefix()), projectRootId); } } @@ -28,29 +79,50 @@ public static void populateImpactTrees(Map vulnerableDep * * @param depNode a vulnerable dependency * @param parents a map of all dependencies and their parents - * @param rootId the project's root component ID + * @param rootId the module's root component ID * @param path a path of nodes (represented by their component IDs) from the current parent to the current node + * @param projectRootId the project's root component ID to prepend to each path, or null */ - private static void walkParents(DependencyNode depNode, Map> parents, String rootId, List path) { + private static void walkParents(DependencyNode depNode, Map> parents, String rootId, List path, String projectRootId) { String currParentId = path.get(0); if (depNode.getImpactTree() != null && depNode.getImpactTree().getImpactPathsCount() >= ImpactTree.IMPACT_PATHS_LIMIT) { return; } // If we arrived at the root, add the path to the impact tree if (currParentId.equals(rootId)) { - addImpactPathToDependencyNode(depNode, path); - } else { - for (String grandparentId : parents.get(currParentId)) { - if (path.contains(grandparentId)) { - continue; - } - List pathToGrandparent = new ArrayList<>(path); - pathToGrandparent.add(0, grandparentId); - walkParents(depNode, parents, rootId, pathToGrandparent); + addImpactPathToDependencyNode(depNode, prependProjectRoot(path, projectRootId)); + return; + } + for (String grandparentId : parents.getOrDefault(currParentId, Collections.emptySet())) { + if (path.contains(grandparentId)) { + continue; + } + List pathToGrandparent = new ArrayList<>(path); + pathToGrandparent.add(0, grandparentId); + walkParents(depNode, parents, rootId, pathToGrandparent, projectRootId); + } + } + + /** + * Gives unreachable libraries a fallback direct path, since code reading impact trees crashes on a missing one. + */ + private static void addMissingImpactTrees(Map vulnerableDependencies, String rootId) { + for (DependencyNode dependency : vulnerableDependencies.values()) { + if (dependency.getImpactTree() == null) { + addImpactPathToDependencyNode(dependency, List.of(rootId, dependency.getComponentIdWithoutPrefix())); } } } + private static List prependProjectRoot(List path, String projectRootId) { + if (projectRootId == null) { + return path; + } + List fullPath = new ArrayList<>(path); + fullPath.add(0, projectRootId); + return fullPath; + } + public static void addImpactPathToDependencyNode(DependencyNode dependencyNode, List path) { if (dependencyNode.getImpactTree() == null) { dependencyNode.setImpactTree(new ImpactTree(new ImpactTreeNode(path.get(0)))); diff --git a/src/test/java/com/jfrog/ide/idea/scan/GradleModuleImpactPathsTest.java b/src/test/java/com/jfrog/ide/idea/scan/GradleModuleImpactPathsTest.java new file mode 100644 index 00000000..bf99c571 --- /dev/null +++ b/src/test/java/com/jfrog/ide/idea/scan/GradleModuleImpactPathsTest.java @@ -0,0 +1,86 @@ +package com.jfrog.ide.idea.scan; + +import com.jfrog.ide.idea.scan.utils.ImpactTreeBuilder; + +import com.jfrog.ide.common.deptree.DepTree; +import com.jfrog.ide.common.gradle.GradleTreeBuilder; +import com.jfrog.ide.common.nodes.DependencyNode; +import com.jfrog.ide.common.nodes.subentities.ImpactTreeNode; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.jfrog.build.api.util.NullLog; +import org.junit.Assert; +import org.junit.Test; + +import java.net.URI; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * A Gradle project where 'moda' gets commons-lang3 through commons-text and 'modb' excludes it. + */ +public class GradleModuleImpactPathsTest { + private static final String EXCLUDED_BY_MODB_COMP_ID = "org.apache.commons:commons-lang3:3.11"; + private static final String PATH_THROUGH_MODA = "com.example:moda:1.0 -> org.apache.commons:commons-text:1.9 -> " + EXCLUDED_BY_MODB_COMP_ID; + private static final String GRADLE_DEP_TREE_CLASS = "com.jfrog.GradleDependencyNode"; + + @Test + public void testImpactPathsNameOnlyTheModuleThatResolvesTheDependency() throws Exception { + Path projectDir = Files.createTempDirectory("moduleImpactPaths"); + try { + DepTree depTree = buildDependencyTree(projectDir); + Assert.assertTrue("commons-lang3 is expected in the dependency tree", depTree.nodes().containsKey(EXCLUDED_BY_MODB_COMP_ID)); + + DependencyNode vulnerableDependency = new DependencyNode().componentId("gav://" + EXCLUDED_BY_MODB_COMP_ID); + ImpactTreeBuilder.populateImpactTrees(Map.of(EXCLUDED_BY_MODB_COMP_ID, vulnerableDependency), depTree); + + Assert.assertNotNull("an impact path should have been built for commons-lang3", vulnerableDependency.getImpactTree()); + List paths = new ArrayList<>(); + collectPaths(vulnerableDependency.getImpactTree().getRoot(), "", paths); + Assert.assertEquals("commons-lang3 is only reachable through moda: " + paths, 1, paths.size()); + Assert.assertTrue("the only impact path must go through moda and commons-text: " + paths, paths.get(0).endsWith(PATH_THROUGH_MODA)); + Assert.assertFalse("modb excludes commons-lang3: " + paths, paths.get(0).contains(":modb:")); + } finally { + FileUtils.deleteQuietly(projectDir.toFile()); + } + } + + private void collectPaths(ImpactTreeNode node, String prefix, List paths) { + String path = prefix.isEmpty() ? node.getName() : prefix + " -> " + node.getName(); + if (node.getChildren().isEmpty()) { + paths.add(path); + return; + } + node.getChildren().forEach(child -> collectPaths(child, path, paths)); + } + + private DepTree buildDependencyTree(Path projectDir) throws Exception { + Path source = Paths.get("src", "test", "resources", "gradle", "moduleImpactPaths").toAbsolutePath().normalize(); + FileUtils.copyDirectory(source.toFile(), projectDir.toFile()); + Map env = new HashMap<>(System.getenv()); + env.put("pluginLibDir", copyDepTreePlugin(projectDir).toString()); + GradleTreeBuilder treeBuilder = new GradleTreeBuilder(projectDir, projectDir.resolve("build.gradle").toString(), env, ""); + return treeBuilder.buildTree(new NullLog()); + } + + private Path copyDepTreePlugin(Path projectDir) throws Exception { + Path libDir = Files.createDirectories(projectDir.resolve("gradle-dep-tree-lib")); + FileUtils.copyFileToDirectory(gradleDepTreeJar().toFile(), libDir.toFile()); + return libDir; + } + + private Path gradleDepTreeJar() throws Exception { + String resource = "/" + GRADLE_DEP_TREE_CLASS.replace('.', '/') + ".class"; + URL location = Class.forName(GRADLE_DEP_TREE_CLASS).getResource(resource); + Assert.assertTrue("'" + GRADLE_DEP_TREE_CLASS + "' is expected to come from a jar, but was loaded from " + location, + StringUtils.startsWith(location.toString(), "jar:") && StringUtils.contains(location.toString(), "!")); + String jarUrl = StringUtils.substringBefore(StringUtils.removeStart(location.toString(), "jar:"), "!"); + return Path.of(URI.create(jarUrl)); + } +} diff --git a/src/test/java/com/jfrog/ide/idea/scan/ModuleImpactTreesTest.java b/src/test/java/com/jfrog/ide/idea/scan/ModuleImpactTreesTest.java new file mode 100644 index 00000000..270e4034 --- /dev/null +++ b/src/test/java/com/jfrog/ide/idea/scan/ModuleImpactTreesTest.java @@ -0,0 +1,123 @@ +package com.jfrog.ide.idea.scan; + +import com.jfrog.ide.idea.scan.utils.ImpactTreeBuilder; + +import com.jfrog.ide.common.deptree.DepTree; +import com.jfrog.ide.common.deptree.DepTreeModule; +import com.jfrog.ide.common.deptree.DepTreeNode; +import com.jfrog.ide.common.nodes.DependencyNode; +import com.jfrog.ide.common.nodes.subentities.ImpactTreeNode; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Two modules where 'includes' gets commons-lang3 through commons-text and 'excludes' excludes it. + */ +public class ModuleImpactTreesTest { + private static final String PROJECT_ROOT_ID = "multi-project"; + private static final String INCLUDES_COMP_ID = "org.jfrog.test:includes:1.0"; + private static final String EXCLUDES_COMP_ID = "org.jfrog.test:excludes:1.0"; + private static final String COMMONS_TEXT_COMP_ID = "org.apache.commons:commons-text:1.9"; + private static final String COMMONS_LANG3_COMP_ID = "org.apache.commons:commons-lang3:3.11"; + private static final String UNREACHABLE_COMP_ID = "com.example:unreachable:1.0"; + + @Test + public void testExcludedTransitiveIsNotAttributedToTheModuleThatExcludesIt() { + DependencyNode commonsLang3 = vulnerableDependency(COMMONS_LANG3_COMP_ID); + ImpactTreeBuilder.populateImpactTrees(Map.of(COMMONS_LANG3_COMP_ID, commonsLang3), multiModuleDepTree()); + + Assert.assertEquals( + List.of(PROJECT_ROOT_ID + " -> " + INCLUDES_COMP_ID + " -> " + COMMONS_TEXT_COMP_ID + " -> " + COMMONS_LANG3_COMP_ID), + impactPaths(commonsLang3)); + } + + @Test + public void testEveryModuleThatResolvesTheDependencyContributesAPath() { + DependencyNode commonsText = vulnerableDependency(COMMONS_TEXT_COMP_ID); + ImpactTreeBuilder.populateImpactTrees(Map.of(COMMONS_TEXT_COMP_ID, commonsText), multiModuleDepTree()); + + List paths = impactPaths(commonsText); + Assert.assertEquals("commons-text is resolved by both modules: " + paths, 2, paths.size()); + Assert.assertTrue(paths.contains(PROJECT_ROOT_ID + " -> " + INCLUDES_COMP_ID + " -> " + COMMONS_TEXT_COMP_ID)); + Assert.assertTrue(paths.contains(PROJECT_ROOT_ID + " -> " + EXCLUDES_COMP_ID + " -> " + COMMONS_TEXT_COMP_ID)); + } + + @Test + public void testProjectRootIsNotPrependedWhenTheModuleIsTheProjectRoot() { + DepTree singleModule = new DepTree(INCLUDES_COMP_ID, includesModuleNodes(), + List.of(new DepTreeModule(INCLUDES_COMP_ID, includesModuleNodes()))); + DependencyNode commonsLang3 = vulnerableDependency(COMMONS_LANG3_COMP_ID); + ImpactTreeBuilder.populateImpactTrees(Map.of(COMMONS_LANG3_COMP_ID, commonsLang3), singleModule); + + Assert.assertEquals(List.of(INCLUDES_COMP_ID + " -> " + COMMONS_TEXT_COMP_ID + " -> " + COMMONS_LANG3_COMP_ID), + impactPaths(commonsLang3)); + } + + @Test + public void testDependencyNoModuleResolvesStillGetsAnImpactTree() { + DependencyNode unreachable = vulnerableDependency(UNREACHABLE_COMP_ID); + ImpactTreeBuilder.populateImpactTrees(Map.of(UNREACHABLE_COMP_ID, unreachable), multiModuleDepTree()); + + Assert.assertNotNull("a dependency no module resolves must not be left without an impact tree", unreachable.getImpactTree()); + Assert.assertEquals(List.of(PROJECT_ROOT_ID + " -> " + UNREACHABLE_COMP_ID), impactPaths(unreachable)); + } + + /** + * Builds a tree whose merged map unions the children of a component across modules. + */ + private DepTree multiModuleDepTree() { + Map merged = new HashMap<>(); + mergeInto(merged, includesModuleNodes()); + mergeInto(merged, excludesModuleNodes()); + merged.put(PROJECT_ROOT_ID, new DepTreeNode().children(new HashSet<>(Set.of(INCLUDES_COMP_ID, EXCLUDES_COMP_ID)))); + return new DepTree(PROJECT_ROOT_ID, merged, + List.of(new DepTreeModule(INCLUDES_COMP_ID, includesModuleNodes()), + new DepTreeModule(EXCLUDES_COMP_ID, excludesModuleNodes()))); + } + + private void mergeInto(Map merged, Map moduleNodes) { + moduleNodes.forEach((compId, node) -> + merged.computeIfAbsent(compId, id -> new DepTreeNode()).getChildren().addAll(node.getChildren())); + } + + private Map includesModuleNodes() { + Map nodes = new HashMap<>(); + nodes.put(INCLUDES_COMP_ID, new DepTreeNode().children(new HashSet<>(Set.of(COMMONS_TEXT_COMP_ID)))); + nodes.put(COMMONS_TEXT_COMP_ID, new DepTreeNode().children(new HashSet<>(Set.of(COMMONS_LANG3_COMP_ID)))); + nodes.put(COMMONS_LANG3_COMP_ID, new DepTreeNode()); + return nodes; + } + + private Map excludesModuleNodes() { + Map nodes = new HashMap<>(); + nodes.put(EXCLUDES_COMP_ID, new DepTreeNode().children(new HashSet<>(Set.of(COMMONS_TEXT_COMP_ID)))); + nodes.put(COMMONS_TEXT_COMP_ID, new DepTreeNode()); + return nodes; + } + + private DependencyNode vulnerableDependency(String componentId) { + return new DependencyNode().componentId("gav://" + componentId); + } + + private List impactPaths(DependencyNode dependency) { + List paths = new ArrayList<>(); + collectPaths(dependency.getImpactTree().getRoot(), "", paths); + return paths; + } + + private void collectPaths(ImpactTreeNode node, String prefix, List paths) { + String path = prefix.isEmpty() ? node.getName() : prefix + " -> " + node.getName(); + if (node.getChildren().isEmpty()) { + paths.add(path); + return; + } + node.getChildren().forEach(child -> collectPaths(child, path, paths)); + } +} diff --git a/src/test/resources/gradle/moduleImpactPaths/build.gradle b/src/test/resources/gradle/moduleImpactPaths/build.gradle new file mode 100644 index 00000000..b975f03c --- /dev/null +++ b/src/test/resources/gradle/moduleImpactPaths/build.gradle @@ -0,0 +1,21 @@ +allprojects { + group = 'com.example' + version = '1.0' + repositories { mavenCentral() } +} + +subprojects { apply plugin: 'java' } + +project(':moda') { + dependencies { + implementation 'org.apache.commons:commons-text:1.9' + } +} + +project(':modb') { + dependencies { + implementation('org.apache.commons:commons-text:1.9') { + exclude group: 'org.apache.commons', module: 'commons-lang3' + } + } +} diff --git a/src/test/resources/gradle/moduleImpactPaths/settings.gradle b/src/test/resources/gradle/moduleImpactPaths/settings.gradle new file mode 100644 index 00000000..581940d2 --- /dev/null +++ b/src/test/resources/gradle/moduleImpactPaths/settings.gradle @@ -0,0 +1,3 @@ +rootProject.name = 'moduleImpactPaths' +include 'moda' +include 'modb'