diff --git a/core/src/main/java/org/owasp/dependencycheck/data/cache/DataCache.java b/core/src/main/java/org/owasp/dependencycheck/data/cache/DataCache.java index 0e9eba538b4..a60a9d1be3b 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/cache/DataCache.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/cache/DataCache.java @@ -17,7 +17,10 @@ */ package org.owasp.dependencycheck.data.cache; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.jcs3.access.CacheAccess; +import org.apache.commons.jcs3.engine.behavior.ICacheElement; +import org.apache.commons.jcs3.engine.behavior.IElementAttributes; /** * A generic wrapper for the Java Caching System (JCS). @@ -32,6 +35,10 @@ public class DataCache { * A reference to the JCS cache. */ private final CacheAccess cache; + /** + * The intended policy configured for the cache region; for reconciling to existing elements with legacy configuration + */ + private final IElementAttributes regionAttributes; /** * Creates a new generic JCS wrapper. @@ -40,6 +47,7 @@ public class DataCache { */ DataCache(CacheAccess cache) { this.cache = cache; + this.regionAttributes = cache.getDefaultElementAttributes(); } /** @@ -49,7 +57,38 @@ public class DataCache { * @return the cached object */ public T get(String key) { - return cache.get(key); + final ICacheElement element = cache.getCacheElement(key); + if (element == null) { + return null; + } + if (!regionAttributes.getIsEternal() && element.getElementAttributes().getIsEternal()) { + return repairToNonEternal(element); + } + return element.getVal(); + } + + /** + * Releases configured before 13.0.1 accidentally configured the cache regions such that elements were persisted + * to disk as eternal, so JCS would never expire them according to intended policy. + * This resets the element's attributes to the region's current expiry policy while retaining its + * original creation time: an element that is still within its lifetime is kept (and re-persisted with corrected + * attributes so that JCS manages its expiry from here on) rather than being discarded and re-fetched, while an + * element that has outlived the policy is removed. + * + * @param element the eternal element retrieved from the cache + * @return the cached object if it is still within its lifetime; otherwise {@code null} + */ + private T repairToNonEternal(ICacheElement element) { + final IElementAttributes attributes = element.getElementAttributes(); + attributes.setIsEternal(regionAttributes.getIsEternal()); + attributes.setMaxLife(regionAttributes.getMaxLife()); + attributes.setIdleTime(regionAttributes.getIdleTime()); + if (cache.getCacheControl().isExpired(element)) { + cache.remove(element.getKey()); + return null; + } + cache.put(element.getKey(), element.getVal(), attributes); + return element.getVal(); } /** @@ -61,4 +100,9 @@ public T get(String key) { public void put(String key, T content) { cache.put(key, content); } + + @VisibleForTesting + void put(String key, T content, IElementAttributes attributes) { + cache.put(key, content, attributes); + } } diff --git a/core/src/main/java/org/owasp/dependencycheck/data/cache/DataCacheFactory.java b/core/src/main/java/org/owasp/dependencycheck/data/cache/DataCacheFactory.java index 56f8abef3a7..1a41b6c07d6 100644 --- a/core/src/main/java/org/owasp/dependencycheck/data/cache/DataCacheFactory.java +++ b/core/src/main/java/org/owasp/dependencycheck/data/cache/DataCacheFactory.java @@ -18,11 +18,9 @@ package org.owasp.dependencycheck.data.cache; import io.github.jeremylong.jcs3.slf4j.Slf4jAdapter; +import io.github.jeremylong.jcs3.slf4j.Slf4jLogFactory; import org.apache.commons.jcs3.JCS; -import org.apache.commons.jcs3.access.CacheAccess; import org.apache.commons.jcs3.access.exception.CacheException; -import org.apache.commons.jcs3.engine.CompositeCacheAttributes; -import org.apache.commons.jcs3.engine.behavior.ICompositeCacheAttributes; import org.owasp.dependencycheck.data.nexus.MavenArtifact; import org.owasp.dependencycheck.data.nodeaudit.Advisory; import org.owasp.dependencycheck.utils.FileUtils; @@ -79,10 +77,36 @@ private enum CacheType { POM } - static { - System.setProperty("jcs.logSystem", "slf4j"); - if (!LOGGER.isTraceEnabled()) { - Slf4jAdapter.muteLogging(true); + private static synchronized void ensureInitialized(Settings settings) { + if (!initialized) { + JCS.setLogSystem(new Slf4jLogFactory().getName()); + if (!LOGGER.isTraceEnabled()) { + Slf4jAdapter.muteLogging(true); + } + + final File cacheDirectory; + try { + cacheDirectory = new File(settings.getDataDirectory(), CACHE_DIRECTORY); + } catch (IOException ex) { + throw new CacheException("Unable to obtain disk cache directory path", ex); + } + if (!cacheDirectory.isDirectory() && !cacheDirectory.mkdirs()) { + throw new CacheException("Unable to create disk cache: " + cacheDirectory); + } + try (InputStream in = FileUtils.getResourceAsStream(CACHE_PROPERTIES)) { + final Properties properties = new Properties(); + properties.load(in); + properties.put("jcs.auxiliary.ODC.attributes.DiskPath", cacheDirectory.getCanonicalPath()); + for (CacheType t : CacheType.values()) { + final File fp = new File(cacheDirectory, t.toString()); + properties.put("jcs.auxiliary." + t + ".attributes.DiskPath", fp.getCanonicalPath()); + } + + JCS.setConfigProperties(properties); + initialized = true; + } catch (IOException ex) { + throw new CacheException("Error creating disk cache", ex); + } } } @@ -92,33 +116,7 @@ private enum CacheType { * @param settings the configuration settings */ public DataCacheFactory(Settings settings) { - synchronized (DataCacheFactory.class) { - if (!initialized) { - final File cacheDirectory; - try { - cacheDirectory = new File(settings.getDataDirectory(), CACHE_DIRECTORY); - } catch (IOException ex) { - throw new CacheException("Unable to obtain disk cache directory path", ex); - } - if (!cacheDirectory.isDirectory() && !cacheDirectory.mkdirs()) { - throw new CacheException("Unable to create disk cache: " + cacheDirectory); - } - try (InputStream in = FileUtils.getResourceAsStream(CACHE_PROPERTIES)) { - final Properties properties = new Properties(); - properties.load(in); - properties.put("jcs.auxiliary.ODC.attributes.DiskPath", cacheDirectory.getCanonicalPath()); - for (CacheType t : CacheType.values()) { - final File fp = new File(cacheDirectory, t.toString()); - properties.put("jcs.auxiliary." + t + ".attributes.DiskPath", fp.getCanonicalPath()); - } - - JCS.setConfigProperties(properties); - initialized = true; - } catch (IOException ex) { - throw new CacheException("Error creating disk cache", ex); - } - } - } + ensureInitialized(settings); } /** @@ -128,13 +126,7 @@ public DataCacheFactory(Settings settings) { */ public DataCache> getNodeAuditCache() { try { - final ICompositeCacheAttributes attr = new CompositeCacheAttributes(); - attr.setUseDisk(true); - attr.setUseLateral(false); - attr.setUseRemote(false); - final CacheAccess> ca = JCS.getInstance("NODEAUDIT", attr); - final DataCache> dc = new DataCache<>(ca); - return dc; + return new DataCache<>(JCS.getInstance("NODEAUDIT")); } catch (Throwable ex) { //some reports of class not found exception, log and disable the cache. if (ex instanceof CacheException) { @@ -152,13 +144,7 @@ public DataCache> getNodeAuditCache() { */ public DataCache getPomCache() { try { - final ICompositeCacheAttributes attr = new CompositeCacheAttributes(); - attr.setUseDisk(true); - attr.setUseLateral(false); - attr.setUseRemote(false); - final CacheAccess ca = JCS.getInstance("POM", attr); - final DataCache dc = new DataCache<>(ca); - return dc; + return new DataCache<>(JCS.getInstance("POM")); } catch (Throwable ex) { //some reports of class not found exception, log and disable the cache. if (ex instanceof CacheException) { @@ -176,13 +162,7 @@ public DataCache getPomCache() { */ public DataCache> getCentralCache() { try { - final ICompositeCacheAttributes attr = new CompositeCacheAttributes(); - attr.setUseDisk(true); - attr.setUseLateral(false); - attr.setUseRemote(false); - final CacheAccess> ca = JCS.getInstance("CENTRAL", attr); - final DataCache> dc = new DataCache<>(ca); - return dc; + return new DataCache<>(JCS.getInstance("CENTRAL")); } catch (Throwable ex) { //some reports of class not found exception, log and disable the cache. if (ex instanceof CacheException) { diff --git a/core/src/main/resources/dependencycheck-cache.properties b/core/src/main/resources/dependencycheck-cache.properties index 30ae65bd314..42ad2e59c1c 100644 --- a/core/src/main/resources/dependencycheck-cache.properties +++ b/core/src/main/resources/dependencycheck-cache.properties @@ -1,57 +1,31 @@ -# DEFAULT CACHE REGION +# DEFAULT CACHE REGION (all other caches inherit these values) jcs.default=ODC -jcs.default.cacheattributes=org.apache.commons.jcs3.engine.CompositeCacheAttributes -jcs.default.cacheattributes.MemoryCacheName=org.apache.commons.jcs3.engine.memory.lru.LRUMemoryCache -jcs.default.cacheattributes.UseMemoryShrinker=false -jcs.default.cacheattributes.MaxMemoryIdleTimeSeconds=3600 -jcs.default.cacheattributes.ShrinkerIntervalSeconds=60 -jcs.default.elementattributes=org.apache.commons.jcs3.engine.ElementAttributes -jcs.default.elementattributes.IsEternal=false -# use zero max objects with an update pattern to force disk caching +# use zero max objects with an update pattern to force disk-only caching since creation (no memory cache) jcs.default.cacheattributes.MaxObjects=0 jcs.default.cacheattributes.DiskUsagePattern=UPDATE -#24 hour default cache life +jcs.default.cacheattributes.UseDisk=true +jcs.default.cacheattributes.UseRemote=false +jcs.default.cacheattributes.UseLateral=false + +#24 hour default cache life, no idle expiry since access time isn't updated using disk-only caches jcs.default.elementattributes.MaxLife=86400 -jcs.default.elementattributes.IdleTime=1800 +jcs.default.elementattributes.IdleTime=-1 +jcs.default.elementattributes.IsEternal=false jcs.default.elementattributes.IsSpool=true -jcs.default.elementattributes.IsRemote=false jcs.default.elementattributes.IsLateral=false +jcs.default.elementattributes.IsRemote=false -#note - some region attributes are defined at load tiem in the DataCacheFactory. +#30 day cache life for main Central SHA1 search results jcs.region.CENTRAL=ODC -jcs.region.CENTRAL.cacheattributes=org.apache.commons.jcs3.engine.CompositeCacheAttributes -jcs.region.CENTRAL.elementattributes=org.apache.commons.jcs3.engine.ElementAttributes -jcs.region.CENTRAL.cacheattributes.MaxObjects=0 -jcs.region.CENTRAL.cacheattributes.DiskUsagePattern=UPDATE -#30 day cache life for Central jcs.region.CENTRAL.elementattributes.MaxLife=2592000 -jcs.region.CENTRAL.elementattributes.IsSpool=true -jcs.region.CENTRAL.elementattributes.IsRemote=false -jcs.region.CENTRAL.elementattributes.IsLateral=false -#note - some region attributes are defined at load tiem in the DataCacheFactory. +#90 day cache life for POM files from Central jcs.region.POM=ODC -jcs.region.POM.cacheattributes=org.apache.commons.jcs3.engine.CompositeCacheAttributes -jcs.region.POM.elementattributes=org.apache.commons.jcs3.engine.ElementAttributes -jcs.region.POM.cacheattributes.MaxObjects=0 -jcs.region.POM.cacheattributes.DiskUsagePattern=UPDATE -#90 day cache life for POM files from Central - this should likely be higher... jcs.region.POM.elementattributes.MaxLife=7776000 -jcs.region.POM.elementattributes.IsSpool=true -jcs.region.POM.elementattributes.IsRemote=false -jcs.region.POM.elementattributes.IsLateral=false - +#24 hour default cache life for Node Audit jcs.region.NODEAUDIT=ODC -jcs.region.NODEAUDIT.cacheattributes=org.apache.commons.jcs3.engine.CompositeCacheAttributes -jcs.region.NODEAUDIT.elementattributes=org.apache.commons.jcs3.engine.ElementAttributes -jcs.region.NODEAUDIT.cacheattributes.MaxObjects=0 -jcs.region.NODEAUDIT.cacheattributes.DiskUsagePattern=UPDATE -#24 hour default cache life jcs.region.NODEAUDIT.elementattributes.MaxLife=86400 -jcs.region.NODEAUDIT.elementattributes.IsSpool=true -jcs.region.NODEAUDIT.elementattributes.IsRemote=false -jcs.region.NODEAUDIT.elementattributes.IsLateral=false # AVAILABLE AUXILIARY CACHES jcs.auxiliary.ODC=org.apache.commons.jcs3.auxiliary.disk.indexed.IndexedDiskCacheFactory diff --git a/core/src/test/java/org/owasp/dependencycheck/data/cache/DataCacheFactoryTest.java b/core/src/test/java/org/owasp/dependencycheck/data/cache/DataCacheFactoryTest.java index 636125d9569..beefd88ec60 100644 --- a/core/src/test/java/org/owasp/dependencycheck/data/cache/DataCacheFactoryTest.java +++ b/core/src/test/java/org/owasp/dependencycheck/data/cache/DataCacheFactoryTest.java @@ -17,38 +17,242 @@ */ package org.owasp.dependencycheck.data.cache; +import org.apache.commons.jcs3.JCS; +import org.apache.commons.jcs3.access.CacheAccess; +import org.apache.commons.jcs3.auxiliary.AuxiliaryCache; +import org.apache.commons.jcs3.engine.ElementAttributes; +import org.apache.commons.jcs3.engine.behavior.ICacheElement; +import org.apache.commons.jcs3.engine.behavior.ICompositeCacheAttributes; +import org.apache.commons.jcs3.engine.behavior.IElementAttributes; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.owasp.dependencycheck.BaseTest; import org.owasp.dependencycheck.data.nexus.MavenArtifact; import java.io.File; -import java.io.FilenameFilter; import java.io.IOException; +import java.lang.reflect.Field; +import java.time.Duration; import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.LockSupport; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; /** + * Exercises the caches produced by {@link DataCacheFactory} against a real, disk-backed JCS instance configured from + * the default {@code dependencycheck-cache.properties}, including the handling of caches written by earlier releases + * whose elements were persisted as eternal. * * @author Jeremy Long */ class DataCacheFactoryTest extends BaseTest { + private static final String TEST_REGION = "CENTRAL"; + private static final long CENTRAL_MAX_LIFE_SECONDS = Duration.ofDays(30).getSeconds(); + public static final List TEST_VALUE = List.of(new MavenArtifact("org.example", "example", "1.0.0")); + + private DataCacheFactory factory; + + /** + * Shuts JCS down, flushing all pending writes to disk, and re-initialises it from the default properties so that + * subsequent reads are served from the on-disk cache rather than from in-memory purgatory. + */ + @BeforeEach + void initialize() { + JCS.shutdown(); + factory = new DataCacheFactory(getSettings()); + } + + DataCacheFactory restartFactory() { + awaitDiskWritesComplete(); + initialize(); + return factory; + } + /** - * Test of getCache method, of class DataCacheFactory. + * JCS disk writes are asynchronous, and its shutdown does not wait for a write that the event queue's thread pool + * has already picked up, so a put immediately followed by a shutdown can be lost. Pending writes sit in the disk + * cache's purgatory until written, so wait for that to drain before shutting down. */ + private static void awaitDiskWritesComplete() { + final AuxiliaryCache disk = JCS.getInstance(TEST_REGION).getCacheControl().getAuxCacheList().get(0); + final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (purgatorySize(disk) > 0) { + assertTrue(System.nanoTime() < deadline, () -> "Timed out waiting for disk cache writes: " + disk.getStats()); + LockSupport.parkNanos(TimeUnit.MILLISECONDS.toNanos(5)); + } + } + + private static int purgatorySize(AuxiliaryCache disk) { + return disk.getStatistics().getStatElements().stream() + .filter(element -> "Purgatory Size".equals(element.getName())) + .mapToInt(element -> (Integer) element.getData()) + .findFirst() + .orElseThrow(); + } + + @AfterEach + void removeCachedEntries() { + JCS.>getInstance(TEST_REGION).clear(); + } + + @Test + void cachesAreCreatedOnDiskInTheDataDirectory() throws IOException { + assertNotNull(factory.getCentralCache()); + assertNotNull(factory.getPomCache()); + assertNotNull(factory.getNodeAuditCache()); + + final File cacheDirectory = new File(getSettings().getDataDirectory(), "cache"); + assertTrue(cacheDirectory.isDirectory()); + for (String region : List.of("CENTRAL", "POM", "NODEAUDIT")) { + final File[] files = cacheDirectory.listFiles((dir, name) -> name.startsWith(region)); + assertNotNull(files); + assertTrue(files.length > 0, "expected disk cache files for " + region); + } + } + + @Test + void regionsInheritTheDefaultExpiryPolicy() { + assertRegionPolicy("CENTRAL", Duration.ofDays(30)); + assertRegionPolicy("POM", Duration.ofDays(90)); + assertRegionPolicy("NODEAUDIT", Duration.ofDays(1)); + } + + @Test + void entriesSurviveRestartWithNonEternalAttributes() { + final DataCache> cache = factory.getCentralCache(); + final String key = newKey(); + final long before = System.currentTimeMillis(); + cache.put(key, TEST_VALUE); + + final DataCache> reopened = restartFactory().getCentralCache(); + + final List result = reopened.get(key); + assertNotNull(result); + assertEquals(1, result.size()); + assertEquals("org.example", result.get(0).getGroupId()); + assertEquals("example", result.get(0).getArtifactId()); + assertEquals("1.0.0", result.get(0).getVersion()); + + final IElementAttributes attributes = elementAttributes(key); + assertFalse(attributes.getIsEternal()); + assertEquals(CENTRAL_MAX_LIFE_SECONDS, attributes.getMaxLife()); + assertEquals(-1, attributes.getIdleTime()); + assertTrue(attributes.getCreateTime() >= before); + } + @Test - void testGetCache() throws IOException { - DataCacheFactory instance = new DataCacheFactory(getSettings()); - DataCache> result = instance.getCentralCache(); + void entriesOlderThanMaxLifeAreExpiredOnAccess() { + final String key = newKey(); + final IElementAttributes attributes = JCS.getInstance(TEST_REGION).getDefaultElementAttributes(); + setCreateTime(attributes, daysAgo(31)); + factory.getCentralCache().put(key, TEST_VALUE, attributes); + + assertNull(restartFactory().getCentralCache().get(key)); + assertNull(JCS.getInstance(TEST_REGION).getCacheElement(key)); + } + + @Test + void legacyEternalEntryWithinMaxLifeIsReturnedAndRepaired() { + final String key = newKey(); + final long createTime = daysAgo(10); + factory.getCentralCache().put(key, TEST_VALUE, legacyEternalAttributes(createTime)); + + final DataCache> cache = restartFactory().getCentralCache(); + final List result = cache.get(key); assertNotNull(result); + assertEquals("org.example", result.get(0).getGroupId()); + + // immediately after the repair the entry is served from the disk cache's purgatory (pending write queue) + final IElementAttributes pending = elementAttributes(key); + assertFalse(pending.getIsEternal()); + assertEquals(createTime, pending.getCreateTime()); + assertNotNull(cache.get(key)); - File f = new File(getSettings().getDataDirectory(), "cache"); - assertTrue(f.isDirectory()); + // the repaired attributes must have been persisted to disk, retaining the original creation time + final DataCache> reopened = restartFactory().getCentralCache(); + final IElementAttributes repaired = elementAttributes(key); + assertFalse(repaired.getIsEternal()); + assertEquals(CENTRAL_MAX_LIFE_SECONDS, repaired.getMaxLife()); + assertEquals(-1, repaired.getIdleTime()); + assertEquals(createTime, repaired.getCreateTime()); + assertNotNull(reopened.get(key)); + } + + @Test + void legacyEternalEntryBeyondMaxLifeIsRemovedOnAccess() { + final String key = newKey(); + factory.getCentralCache().put(key, TEST_VALUE, legacyEternalAttributes(daysAgo(31))); + + assertNull(restartFactory().getCentralCache().get(key)); + assertNull(JCS.getInstance(TEST_REGION).getCacheElement(key)); - FilenameFilter filter = (File f1, String name) -> name.startsWith("CENTRAL"); - assertTrue(f.listFiles(filter).length > 0); + assertNull(restartFactory().getCentralCache().get(key)); } + private String newKey() { + return "DataCacheFactoryTest-" + UUID.randomUUID(); + } + + private IElementAttributes elementAttributes(String key) { + final CacheAccess> region = JCS.getInstance(TEST_REGION); + final ICacheElement> element = region.getCacheElement(key); + assertNotNull(element, "expected element for " + key); + return element.getElementAttributes(); + } + + /** + * Element attributes as persisted by earlier releases, whose region configuration replaced the default element + * attributes (eternal by default in JCS) and then only set MaxLife and the spool flags. + */ + private static IElementAttributes legacyEternalAttributes(long createTime) { + final ElementAttributes attributes = new ElementAttributes(); + attributes.setMaxLife(CENTRAL_MAX_LIFE_SECONDS); + attributes.setIsSpool(true); + attributes.setIsRemote(false); + attributes.setIsLateral(false); + assertTrue(attributes.getIsEternal(), "JCS default element attributes are expected to be eternal"); + setCreateTime(attributes, createTime); + return attributes; + } + + private static long daysAgo(int days) { + return System.currentTimeMillis() - Duration.ofDays(days).toMillis(); + } + + /** + * JCS only exposes a no-argument {@code setCreateTime()} that uses the current time. + */ + private static void setCreateTime(IElementAttributes attributes, long createTime) { + try { + final Field field = ElementAttributes.class.getDeclaredField("createTime"); + field.setAccessible(true); + field.setLong(attributes, createTime); + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("Unable to set createTime on " + attributes, e); + } + } + + private static void assertRegionPolicy(String region, Duration maxLife) { + final CacheAccess access = JCS.getInstance(region); + final IElementAttributes attributes = access.getDefaultElementAttributes(); + assertFalse(attributes.getIsEternal(), region + " elements must not be eternal"); + assertEquals(maxLife.getSeconds(), attributes.getMaxLife(), region + " MaxLife"); + assertEquals(-1, attributes.getIdleTime(), region + " IdleTime"); + assertTrue(attributes.getIsSpool(), region + " elements must spool to disk"); + assertFalse(attributes.getIsLateral(), region + " elements must not be lateral"); + assertFalse(attributes.getIsRemote(), region + " elements must not be remote"); + + final ICompositeCacheAttributes cacheAttributes = access.getCacheAttributes(); + assertEquals(0, cacheAttributes.getMaxObjects(), region + " MaxObjects"); + assertEquals(ICompositeCacheAttributes.DiskUsagePattern.UPDATE, cacheAttributes.getDiskUsagePattern(), region + " DiskUsagePattern"); + assertTrue(cacheAttributes.isUseDisk(), region + " must use disk"); + } }