feat: extract TextContent from text, JSON, and XML files (expanded)

README.md

@@ -34,6 +34,10 @@ Files, folders, and links are organized into logical groups called **Document Bo
         - Extracting and processing attachments
         - Text and HTML extraction
         - Full text searchable email contents
+    - Text, JSON, and XML file processing
+        - Any `text/*` type (`.txt`, `.md`, `.csv`, …)
+        - JSON (`application/json`, `+json`) and XML (`application/xml`, `+xml`)
+        - Extracted text content for full-text search
 - 🌐 Web-scraping
     - Built in support for scraping OGP metadata, favicons, and other details from links stored within
 - 🏢 Multi-Tenancy

packages/docbox-core/tests/test_file_update.rs

@@ -139,7 +139,7 @@ async fn test_update_file_name_success() {
         );
         assert_eq!(first.document_box, document_box.scope);
         assert!(first.page_matches.is_empty());
-        assert_eq!(first.total_hits, 1);
+        assert_eq!(first.total_hits, 2);
         assert!(first.name_match);
         assert!(!first.content_match);
     }
@@ -284,7 +284,7 @@ async fn test_update_file_folder_success() {
         );
         assert_eq!(first.document_box, document_box.scope);
         assert!(first.page_matches.is_empty());
-        assert_eq!(first.total_hits, 1);
+        assert_eq!(first.total_hits, 2);
         assert!(!first.name_match);
         assert!(!first.content_match);
     }

packages/docbox-processing/src/lib.rs

@@ -7,6 +7,7 @@ use crate::{
     image::process_image_async,
     office::{PdfConvertError, process_office},
     pdf::{GeneratePdfImagesError, process_pdf},
+    text::{is_application_file, is_text_file, process_text},
 };
 use ::image::{ImageError, ImageFormat};
 use bytes::Bytes;
@@ -26,6 +27,7 @@ pub mod html_to_text;
 pub mod image;
 pub mod office;
 pub mod pdf;
+pub mod text;
 
 #[derive(Debug, Error)]
 pub enum ProcessingError {
@@ -161,6 +163,11 @@ pub struct ProcessingLayerConfig {
     ///
     /// Default: 300s
     pub process_timeout: Option<Duration>,
+
+    /// Whether to process .json and .xml files, if you don't need
+    /// these to be searchable its recommended to keep this disabled
+    /// as they can fill your search results with unwanted data if added
+    pub process_application_files: Option<bool>,
 }
 
 pub const DEFAULT_PROCESS_TIMEOUT: Duration = Duration::from_secs(300);
@@ -173,6 +180,9 @@ pub enum ProcessingLayerConfigError {
     /// Invalid process timeout seconds
     #[error("DOCBOX_FILE_PROCESSING_TIMEOUT must be a number in seconds")]
     InvalidProcessTimeout(<u64 as FromStr>::Err),
+    /// Invalid process timeout seconds
+    #[error("DOCBOX_PROCESS_APPLICATION_FILES must be a number in seconds")]
+    InvalidProcessApplicationFiles(<bool as FromStr>::Err),
 }
 
 impl ProcessingLayerConfig {
@@ -196,9 +206,19 @@ impl ProcessingLayerConfig {
             })
             .transpose()?;
 
+        let process_application_files = std::env::var("DOCBOX_PROCESS_APPLICATION_FILES")
+            .ok()
+            .map(|value| {
+                value
+                    .parse::<bool>()
+                    .map_err(ProcessingLayerConfigError::InvalidProcessApplicationFiles)
+            })
+            .transpose()?;
+
         Ok(ProcessingLayerConfig {
             max_unpack_iterations,
             process_timeout,
+            process_application_files,
         })
     }
 }
@@ -244,6 +264,22 @@ pub async fn process_file(
         let output = process_image_async(bytes, image_format).await?;
         Ok(Some(output))
     }
+    // File is plain text
+    else if is_text_file(mime) {
+        tracing::debug!("processing text file");
+
+        let output = process_text(&bytes);
+        Ok(Some(output))
+    }
+    // File is JSON or XML
+    else if is_application_file(mime)
+        && layer.config.process_application_files.unwrap_or_default()
+    {
+        tracing::debug!("processing application file");
+
+        let output = process_text(&bytes);
+        Ok(Some(output))
+    }
     // No processing for this file type
     else {
         Ok(None)

packages/docbox-processing/src/office/mod.rs

@@ -51,6 +51,9 @@ pub enum PdfConvertError {
 
     #[error("office document is password protected")]
     EncryptedDocument,
+
+    #[error("no convert server specified")]
+    NotAvailable,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -88,6 +91,7 @@ impl OfficeConverterConfig {
 pub enum OfficeConverter {
     ConverterServer(OfficeConverterServer),
     ConverterLambda(OfficeConverterLambda),
+    Noop,
 }
 
 #[derive(Debug, Error)]
@@ -131,13 +135,15 @@ impl OfficeConverter {
         match self {
             OfficeConverter::ConverterServer(inner) => inner.convert_to_pdf(bytes).await,
             OfficeConverter::ConverterLambda(inner) => inner.convert_to_pdf(bytes).await,
+            OfficeConverter::Noop => Err(PdfConvertError::NotAvailable),
         }
     }
 
     pub fn is_convertable(&self, mime: &Mime) -> bool {
         match self {
             OfficeConverter::ConverterServer(inner) => inner.is_convertable(mime),
             OfficeConverter::ConverterLambda(inner) => inner.is_convertable(mime),
+            OfficeConverter::Noop => false,
         }
     }
 }

packages/docbox-processing/src/text.rs

@@ -0,0 +1,61 @@
+use bytes::Bytes;
+use docbox_database::models::generated_file::GeneratedFileType;
+use docbox_search::models::DocumentPage;
+use mime::Mime;
+
+use super::{ProcessingIndexMetadata, ProcessingOutput, QueuedUpload};
+
+const UTF8_BOM: &[u8] = &[0xEF, 0xBB, 0xBF];
+
+/// Checks if the provided mime should be processed as text
+///
+/// Matches any `text/*` type
+pub fn is_text_file(mime: &Mime) -> bool {
+    if mime.type_() == mime::TEXT {
+        return true;
+    }
+
+    false
+}
+
+/// Checks if the provided mime should be processed as text
+/// but from a application file (json/xml)
+///
+/// Matches any JSON (`application/json` and `+json` subtypes),
+/// and XML (`application/xml` and `+xml` subtypes).
+pub fn is_application_file(mime: &Mime) -> bool {
+    if mime.type_() != mime::APPLICATION {
+        return false;
+    }
+
+    let subtype = mime.subtype().as_str();
+    if subtype == "json" || subtype == "xml" {
+        return true;
+    }
+
+    let suffix = mime.suffix();
+    suffix.is_some_and(|suffix| matches!(suffix.as_str(), "json" | "xml"))
+}
+
+/// Processes a text, JSON, or XML file by decoding the source bytes
+/// and emitting searchable [GeneratedFileType::TextContent]
+pub fn process_text(file_bytes: &[u8]) -> ProcessingOutput {
+    let raw = file_bytes.strip_prefix(UTF8_BOM).unwrap_or(file_bytes);
+    let content = String::from_utf8_lossy(raw).into_owned();
+
+    let pages = vec![DocumentPage {
+        content: content.clone(),
+        page: 0,
+    }];
+
+    ProcessingOutput {
+        encrypted: false,
+        additional_files: Vec::new(),
+        index_metadata: Some(ProcessingIndexMetadata { pages: Some(pages) }),
+        upload_queue: vec![QueuedUpload::new(
+            mime::TEXT_PLAIN,
+            GeneratedFileType::TextContent,
+            Bytes::from(content),
+        )],
+    }
+}

packages/docbox-processing/tests/common/processing.rs

@@ -39,3 +39,12 @@ pub async fn test_processing_layer(
         config,
     }
 }
+
+pub fn noop_processing_layer(config: ProcessingLayerConfig) -> ProcessingLayer {
+    ProcessingLayer {
+        office: OfficeProcessingLayer {
+            converter: OfficeConverter::Noop,
+        },
+        config,
+    }
+}