SED-4816 Introduce AttachmentStorage

step-controller/step-controller-base-plugins/src/main/java/step/plugins/streaming/StepStreamingResourceManager.java

@@ -1,26 +1,40 @@
 package step.plugins.streaming;
 
+import org.bson.types.ObjectId;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import step.attachments.AttachmentMeta;
+import step.attachments.SkippedAttachmentMeta;
+import step.attachments.StreamingAttachmentMeta;
 import step.constants.LiveReportingConstants;
 import step.core.GlobalContext;
 import step.core.access.User;
 import step.core.deployment.AuthorizationException;
+import step.core.execution.ExecutionContext;
+import step.core.objectenricher.ObjectEnricher;
 import step.core.objectenricher.ObjectHookRegistry;
 import step.framework.server.Session;
 import step.framework.server.access.AuthorizationManager;
-import step.resources.StreamingResourceContentProvider;
-import step.streaming.common.*;
+import step.resources.AttachmentStorage;
+import step.streaming.common.QuotaExceededException;
+import step.streaming.common.StreamingResourceMetadata;
+import step.streaming.common.StreamingResourceReference;
+import step.streaming.common.StreamingResourceUploadContext;
+import step.streaming.common.StreamingResourceUploadContexts;
 import step.streaming.server.DefaultStreamingResourceManager;
 import step.streaming.server.StreamingResourcesStorageBackend;
 
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Objects;
 import java.util.function.Function;
+import java.util.regex.Pattern;
 
-public class StepStreamingResourceManager extends DefaultStreamingResourceManager implements StreamingResourceContentProvider {
+public class StepStreamingResourceManager extends DefaultStreamingResourceManager implements AttachmentStorage {
     private static final Logger logger = LoggerFactory.getLogger(StepStreamingResourceManager.class);
     static final String ATTRIBUTE_STEP_SESSION = "stepSession";
+    private static final Pattern MIME_TYPE_PATTERN = Pattern.compile("^[a-zA-Z0-9!#$&^_.+-]+/[a-zA-Z0-9!#$&^_.+-]+$");
 
 
     private final AuthorizationManager<User, Session<User>> authorizationManager;
@@ -139,8 +153,77 @@ public void checkDownloadPermission(StreamingResource resource, Session<User> st
         }
     }
 
+    /* The following methods implement the AttachmentStorage interface */
+
     @Override
-    public InputStream getResourceContentStream(String resourceId) throws Exception {
+    public InputStream getAttachmentStream(String resourceId) throws Exception {
         return openStream(resourceId, 0, getStatus(resourceId).getCurrentSize());
     }
+
+    @Override
+    public AttachmentMeta saveAttachment(Object executionContext, byte[] content, String filename, String mimeType) {
+        if (!(executionContext instanceof ExecutionContext)) {
+            String className = executionContext == null ? "null" : executionContext.getClass().getName();
+            return new SkippedAttachmentMeta(filename, mimeType, "UNEXPECTED: Invalid execution context type of class " + className);
+        }
+        ExecutionContext context = (ExecutionContext) Objects.requireNonNull(executionContext);
+        StreamingResourceUploadContexts uploadContexts = context.get(StreamingResourceUploadContexts.class);
+
+        if (uploadContexts == null) {
+            return new SkippedAttachmentMeta(filename, mimeType, "UNEXPECTED: no StreamingResourceUploadContexts found in execution context");
+        }
+
+        // We don't need to react to any events, but we need an UploadContext to convey relevant contextual information about the attachment.
+        StreamingResourceUploadContext uploadContext = new StreamingResourceUploadContext();
+        ObjectEnricher enricher = context.getObjectEnricher();
+        if (enricher != null) {
+            uploadContext.getAttributes().put(LiveReportingConstants.ACCESSCONTROL_ENRICHER, enricher);
+        }
+        uploadContext.getAttributes().put(LiveReportingConstants.CONTEXT_EXECUTION_ID, context.getExecutionId());
+        uploadContext.getAttributes().put(LiveReportingConstants.CONTEXT_VARIABLES_MANAGER, context.getVariablesManager());
+        uploadContext.getAttributes().put(LiveReportingConstants.CONTEXT_REPORT_NODE, context.getCurrentReportNode());
+        uploadContexts.registerContext(uploadContext);
+
+        try {
+            return createAttachmentFromContent(content, filename, mimeType, uploadContext.contextId);
+        } catch (Exception e) {
+            return new SkippedAttachmentMeta(filename, mimeType, e.getMessage());
+        } finally {
+            uploadContexts.unregisterContext(uploadContext);
+        }
+    }
+
+    public AttachmentMeta createAttachmentFromContent(byte[] content, String filename, String mimeType, String uploadContextId) throws QuotaExceededException, IOException {
+        mimeType = sanitizeMimeType(mimeType);
+        boolean supportsLineAccess = isLineAccessSupported(mimeType);
+        StreamingResourceMetadata metadata = new StreamingResourceMetadata(filename, mimeType, supportsLineAccess);
+        String resourceId = registerNewResource(metadata, uploadContextId);
+        StreamingAttachmentMeta meta = new StreamingAttachmentMeta(new ObjectId(resourceId), filename, mimeType);
+        // Saving will be completely finished, in a single call, because all data is already present from the beginning.
+        writeChunk(resourceId, new ByteArrayInputStream(content), true);
+        var status = markCompleted(resourceId);
+        meta.setCurrentNumberOfLines(status.getNumberOfLines());
+        meta.setCurrentSize(status.getCurrentSize());
+        meta.setStatus(StreamingAttachmentMeta.Status.COMPLETED);
+        return meta;
+    }
+
+    private static String sanitizeMimeType(String mimeType) {
+        if (mimeType == null) {
+            // fallback if no mimeType was given
+            return "application/octet-stream";
+        }
+        mimeType = mimeType.trim(); // just in case
+        if (!MIME_TYPE_PATTERN.matcher(mimeType).matches()) {
+            logger.warn("Invalid mime type \"{}\", replacing with generic application/octet-stream", mimeType);
+            return "application/octet-stream";
+        }
+        return mimeType;
+    }
+
+    private static boolean isLineAccessSupported(String mimeType) {
+        // Very simple heuristic for now, should catch > 90% of the useful cases
+        // TODO: we can add a few other well-known textual formats if needed
+        return mimeType.startsWith("text/");
+    }
 }

step-controller/step-controller-base-plugins/src/main/java/step/plugins/streaming/StreamingResourcesControllerPlugin.java

@@ -1,6 +1,5 @@
 package step.plugins.streaming;
 
-import ch.exense.commons.app.Configuration;
 import jakarta.servlet.http.HttpSession;
 import jakarta.websocket.Endpoint;
 import jakarta.websocket.HandshakeResponse;
@@ -10,17 +9,14 @@
 import org.slf4j.LoggerFactory;
 import step.constants.LiveReportingConstants;
 import step.core.GlobalContext;
-import step.core.controller.StepControllerPlugin;
 import step.core.deployment.ObjectHookControllerPlugin;
-import step.core.execution.AbstractExecutionEngineContext;
 import step.core.execution.ExecutionContext;
 import step.core.execution.ExecutionEngineContext;
 import step.core.plugins.AbstractControllerPlugin;
 import step.core.plugins.Plugin;
 import step.engine.plugins.AbstractExecutionEnginePlugin;
 import step.engine.plugins.ExecutionEnginePlugin;
 import step.resources.ResourceManagerControllerPlugin;
-import step.resources.StreamingResourceContentProvider;
 import step.streaming.common.StreamingResourceUploadContexts;
 import step.streaming.server.FilesystemStreamingResourcesStorageBackend;
 import step.streaming.server.StreamingResourceManager;
@@ -66,6 +62,7 @@ public void serverStart(GlobalContext context) throws Exception {
         manager = new StepStreamingResourceManager(context, catalog, storage, referenceProducer, uploadContexts);
 
         context.put(StepStreamingResourceManager.class, manager);
+        context.setAttachmentStorage(manager);
         context.getServiceRegistrationCallback().registerService(StreamingResourceServices.class);
 
         context.getServiceRegistrationCallback().registerWebsocketEndpoint(makeUploadConfig(manager));
@@ -113,12 +110,6 @@ private ServerEndpointConfig makeDownloadConfig(StepStreamingResourceManager man
     @Override
     public ExecutionEnginePlugin getExecutionEnginePlugin() {
         return new AbstractExecutionEnginePlugin() {
-            @Override
-            public void initializeExecutionEngineContext(AbstractExecutionEngineContext parentContext, ExecutionEngineContext executionEngineContext) {
-                // required by AP reporting for fetching attachments
-                executionEngineContext.put(StreamingResourceContentProvider.class, manager);
-            }
-
             @Override
             public void initializeExecutionContext(ExecutionEngineContext executionEngineContext, ExecutionContext executionContext) {
                 // Makes streaming available to the execution

step-controller/step-controller-server/src/test/java/step/core/GlobalContextBuilder.java

@@ -60,7 +60,14 @@
 import step.functions.accessor.FunctionAccessor;
 import step.functions.accessor.FunctionEntity;
 import step.functions.accessor.InMemoryFunctionAccessorImpl;
-import step.resources.*;
+import step.resources.InMemoryResourceAccessor;
+import step.resources.InMemoryResourceRevisionAccessor;
+import step.resources.LocalResourceManagerImpl;
+import step.resources.ResourceAccessor;
+import step.resources.ResourceEntity;
+import step.resources.ResourceManager;
+import step.resources.ResourceRevision;
+import step.resources.ResourceRevisionAccessor;
 
 import java.io.File;
 import java.io.IOException;
@@ -102,8 +109,10 @@ public static GlobalContext createGlobalContext() throws CircularDependencyExcep
         ResourceAccessor resourceAccessor = new InMemoryResourceAccessor();
         InMemoryResourceRevisionAccessor resourceRevisionAccessor = new InMemoryResourceRevisionAccessor();
         try {
-            File rootFolder = FileHelper.createTempFolder();
-            ResourceManager resourceManager = new ResourceManagerImpl(rootFolder, resourceAccessor, resourceRevisionAccessor);
+            // WARNING: The following folder is leaked unless it's cleaned up
+            // in the referencing unit tests (e.g. using ResourceManager#cleanup) - tracked in SED-4849
+            File resourceRootFolder = FileHelper.createTempFolder("step-resourceManager-");
+            ResourceManager resourceManager = new LocalResourceManagerImpl(resourceRootFolder, resourceAccessor, resourceRevisionAccessor);
             context.setResourceManager(resourceManager);
         } catch (IOException e) {
             logger.error("Unable to create temp folder for the resource manager", e);

step-core/src/main/java/step/attachments/FileResolver.java

@@ -18,18 +18,18 @@
  ******************************************************************************/
 package step.attachments;
 
-import java.io.Closeable;
-import java.io.File;
-import java.io.IOException;
-
 import org.bson.types.ObjectId;
 import step.resources.Resource;
 import step.resources.ResourceManager;
 import step.resources.ResourceRevisionFileHandle;
 
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+
 public class FileResolver {
 
-    public static final String ATTACHMENT_PREFIX = "attachment:";
+    public static final String ATTACHMENT_PREFIX_OBSOLETE = "attachment:";
     public static final String RESOURCE_PREFIX = "resource:";
     public static final String RESOURCE_PATH_SEPARATOR = ":";
 
@@ -46,9 +46,9 @@ public ResourceManager getResourceManager() {
 
     public File resolve(String path) {
         File file;
-        if (path.startsWith(ATTACHMENT_PREFIX)) {
-            throw new RuntimeException("Attachments have been migrated to the ResourceManager. The reference " + path +
-                " isn't valid anymore. Your attachment should be migrated to the ResourceManager.");
+        if (path.startsWith(ATTACHMENT_PREFIX_OBSOLETE)) {
+            throw new RuntimeException("Attachments have been migrated to use the AttachmentStorage. The reference " + path +
+                " isn't valid anymore. Please update your code to use the AttachmentStorage instead.");
         } else if (path.startsWith(RESOURCE_PREFIX)) {
             file = getResourceRevisionFileHandleForPath(path).getResourceFile();
         } else {
@@ -114,7 +114,7 @@ protected static String extractResourceSubPath(String path) {
     public FileHandle resolveFileHandle(String path) {
         File file;
         ResourceRevisionFileHandle resourceRevisionFileHandle;
-        if (path.startsWith(ATTACHMENT_PREFIX)) {
+        if (path.startsWith(ATTACHMENT_PREFIX_OBSOLETE)) {
             throw new RuntimeException("Attachments have been migrated to the ResourceManager. The reference " + path +
                 " isn't valid anymore. Your attachment should be migrated to the ResourceManager.");
         } else if (path.startsWith(RESOURCE_PREFIX)) {

step-core/src/main/java/step/core/AbstractStepContext.java

@@ -25,7 +25,13 @@
 import step.core.dynamicbeans.DynamicBeanResolver;
 import step.core.dynamicbeans.DynamicValueResolver;
 import step.expressions.ExpressionHandler;
-import step.resources.*;
+import step.resources.AttachmentStorage;
+import step.resources.InMemoryResourceAccessor;
+import step.resources.InMemoryResourceRevisionAccessor;
+import step.resources.LayeredResourceManager;
+import step.resources.LocalAttachmentStorage;
+import step.resources.LocalResourceManagerImpl;
+import step.resources.ResourceManager;
 
 import java.io.File;
 import java.io.IOException;
@@ -42,25 +48,31 @@ public abstract class AbstractStepContext extends AbstractContext {
     private LoadingCache<String, File> fileResolverCache;
     // Keep track of the default resource manager created at initialization of the context
     private LocalResourceManagerImpl localResourceManager;
+    // Similar to the resource manager, attachmentStorage can be shared by multiple context, but only one of them owns it.
+    private AttachmentStorage attachmentStorage;
+    private AttachmentStorage ownedAttachmentStorage;
 
     protected void setDefaultAttributes() {
         expressionHandler = new ExpressionHandler();
         dynamicBeanResolver = new DynamicBeanResolver(new DynamicValueResolver(expressionHandler));
         // Create a local resource manager in a dedicated folder per default
-        localResourceManager = new LocalResourceManagerImpl(getContextFolderAsFile(), new InMemoryResourceAccessor(), new InMemoryResourceRevisionAccessor());
+        localResourceManager = new LocalResourceManagerImpl(getContextFolderAsFile("resources"), new InMemoryResourceAccessor(), new InMemoryResourceRevisionAccessor());
         setResourceManager(localResourceManager);
+        ownedAttachmentStorage = new LocalAttachmentStorage(getContextFolderAsFile("attachments"));
+        setAttachmentStorage(ownedAttachmentStorage);
     }
 
-    private File getContextFolderAsFile() {
+    private File getContextFolderAsFile(String suffix) {
         String tmpPath = System.getProperty("java.io.tmpdir");
         tmpPath = (tmpPath.endsWith(File.separator)) ? tmpPath : tmpPath + File.separator;
-        String dirName = tmpPath + "stepContext_" + getClass().getSimpleName() + "_" + contextId;
+        String dirName = tmpPath + "stepContext_" + getClass().getSimpleName() + "_" + contextId + "_" + suffix;
         return new File(dirName);
     }
 
     protected void useAllAttributesFromParentContext(AbstractStepContext parentContext) {
         ResourceManager resourceManager = new LayeredResourceManager(parentContext.getResourceManager(), true);
         setResourceManager(resourceManager);
+        setAttachmentStorage(parentContext.getAttachmentStorage());
         expressionHandler = parentContext.getExpressionHandler();
         dynamicBeanResolver = parentContext.getDynamicBeanResolver();
     }
@@ -90,6 +102,14 @@ public void setResourceManager(ResourceManager resourceManager) {
         updateFileResolver();
     }
 
+    public AttachmentStorage getAttachmentStorage() {
+        return attachmentStorage;
+    }
+
+    public void setAttachmentStorage(AttachmentStorage attachmentStorage) {
+        this.attachmentStorage = attachmentStorage;
+    }
+
     public FileResolver getFileResolver() {
         return fileResolver;
     }
@@ -115,10 +135,13 @@ public void close() throws IOException {
         try {
             super.close();
         } finally {
-            // Cleanup the default resource manager
+            // Cleanup the default resource manager, and attachment storage
             if (localResourceManager != null) {
                 localResourceManager.cleanup();
             }
+            if (ownedAttachmentStorage != null) {
+                ownedAttachmentStorage.cleanupIfNeeded();
+            }
         }
     }
 }

step-core/src/main/java/step/resources/AttachmentStorage.java

@@ -0,0 +1,25 @@
+package step.resources;
+
+import step.attachments.AttachmentMeta;
+
+import java.io.InputStream;
+
+public interface AttachmentStorage {
+    InputStream getAttachmentStream(String attachmentId) throws Exception;
+
+    /**
+     * Saves an attachment in the given executionContent, with the given metadata.
+     * Implementation note: the executionContext will be a step.core.execution.ExecutionContext, but that class
+     * is out of scope here, hence the use of a generic Object.
+     *
+     * @param executionContext execution context
+     * @param content          content of the attachment, as a byte array
+     * @param filename         file name of the attachment
+     * @param mimeType         MIME type, may be null
+     * @return AttachmentMeta object corresponding to the attachment
+     */
+    AttachmentMeta saveAttachment(Object executionContext, byte[] content, String filename, String mimeType);
+
+    default void cleanupIfNeeded() {
+    }
+}