From 5784af0d66eadbf4492e0f38bb44fbdbd2f0fbcf Mon Sep 17 00:00:00 2001 From: saille Date: Thu, 10 Sep 2026 17:04:07 +1200 Subject: [PATCH 1/3] feat: extract TextContent from text, JSON, and XML files Decode text/*, application/json, and application/xml as source text so they are searchable without a LibreOffice conversion. --- README.md | 4 + packages/docbox-processing/src/lib.rs | 9 + packages/docbox-processing/src/text.rs | 56 +++++ .../tests/test_process_text.rs | 211 ++++++++++++++++++ 4 files changed, 280 insertions(+) create mode 100644 packages/docbox-processing/src/text.rs create mode 100644 packages/docbox-processing/tests/test_process_text.rs diff --git a/README.md b/README.md index f1648d5..d1657b6 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,10 @@ Files, folders, and links are organized into logical groups called **Document Bo - Extracting and processing attachments - Text and HTML extraction - Full text searchable email contents + - Text, JSON, and XML file processing + - Any `text/*` type (`.txt`, `.md`, `.csv`, …) + - JSON (`application/json`, `+json`) and XML (`application/xml`, `+xml`) + - Extracted text content for full-text search - 🌐 Web-scraping - Built in support for scraping OGP metadata, favicons, and other details from links stored within - 🏢 Multi-Tenancy diff --git a/packages/docbox-processing/src/lib.rs b/packages/docbox-processing/src/lib.rs index a785716..7c168b6 100644 --- a/packages/docbox-processing/src/lib.rs +++ b/packages/docbox-processing/src/lib.rs @@ -7,6 +7,7 @@ use crate::{ image::process_image_async, office::{PdfConvertError, process_office}, pdf::{GeneratePdfImagesError, process_pdf}, + text::{is_text_mime, process_text}, }; use ::image::{ImageError, ImageFormat}; use bytes::Bytes; @@ -26,6 +27,7 @@ pub mod html_to_text; pub mod image; pub mod office; pub mod pdf; +pub mod text; #[derive(Debug, Error)] pub enum ProcessingError { @@ -244,6 +246,13 @@ pub async fn process_file( let output = process_image_async(bytes, image_format).await?; Ok(Some(output)) } + // File is plain text, JSON, or XML + else if is_text_mime(mime) { + tracing::debug!("processing text file"); + + let output = process_text(&bytes); + Ok(Some(output)) + } // No processing for this file type else { Ok(None) diff --git a/packages/docbox-processing/src/text.rs b/packages/docbox-processing/src/text.rs new file mode 100644 index 0000000..010a909 --- /dev/null +++ b/packages/docbox-processing/src/text.rs @@ -0,0 +1,56 @@ +use bytes::Bytes; +use docbox_database::models::generated_file::GeneratedFileType; +use docbox_search::models::DocumentPage; +use mime::Mime; + +use super::{ProcessingIndexMetadata, ProcessingOutput, QueuedUpload}; + +const UTF8_BOM: &[u8] = &[0xEF, 0xBB, 0xBF]; + +/// Checks if the provided mime should be processed as source text +/// +/// Matches any `text/*` type, JSON (`application/json` and `+json` subtypes), +/// and XML (`application/xml` and `+xml` subtypes). +pub fn is_text_mime(mime: &Mime) -> bool { + if mime.type_() == mime::TEXT { + return true; + } + + if mime.type_() == mime::APPLICATION { + let subtype = mime.subtype().as_str(); + return subtype == "json" + || subtype == "xml" + || subtype.ends_with("+json") + || subtype.ends_with("+xml"); + } + + false +} + +/// Processes a text, JSON, or XML file by decoding the source bytes +/// and emitting searchable [GeneratedFileType::TextContent] +pub fn process_text(file_bytes: &[u8]) -> ProcessingOutput { + let raw = if file_bytes.starts_with(UTF8_BOM) { + &file_bytes[UTF8_BOM.len()..] + } else { + file_bytes + }; + + let content = String::from_utf8_lossy(raw).into_owned(); + + let pages = vec![DocumentPage { + content: content.clone(), + page: 0, + }]; + + ProcessingOutput { + encrypted: false, + additional_files: Vec::new(), + index_metadata: Some(ProcessingIndexMetadata { pages: Some(pages) }), + upload_queue: vec![QueuedUpload::new( + mime::TEXT_PLAIN, + GeneratedFileType::TextContent, + Bytes::from(content.into_bytes()), + )], + } +} diff --git a/packages/docbox-processing/tests/test_process_text.rs b/packages/docbox-processing/tests/test_process_text.rs new file mode 100644 index 0000000..5b35a8b --- /dev/null +++ b/packages/docbox-processing/tests/test_process_text.rs @@ -0,0 +1,211 @@ +use std::str::FromStr; + +use bytes::Bytes; +use docbox_database::models::generated_file::GeneratedFileType; +use docbox_processing::{ + ProcessingLayer, ProcessingLayerConfig, + office::{OfficeConverter, OfficeProcessingLayer, convert_server::OfficeConverterServer}, + process_file, + text::{is_text_mime, process_text}, +}; +use mime::Mime; + +/// Processing layer that never talks to a convert server. +/// Text processing does not use office conversion. +fn dummy_processing_layer() -> ProcessingLayer { + let converter_server = + OfficeConverterServer::from_addresses(["http://127.0.0.1:1"], false).unwrap(); + + ProcessingLayer { + office: OfficeProcessingLayer { + converter: OfficeConverter::ConverterServer(converter_server), + }, + config: ProcessingLayerConfig::default(), + } +} + +fn mime(value: &str) -> Mime { + Mime::from_str(value).unwrap() +} + +async fn process_with_mime( + contents: &[u8], + mime_type: &str, +) -> Option { + process_file( + &None, + &dummy_processing_layer(), + Bytes::copy_from_slice(contents), + &mime(mime_type), + ) + .await + .unwrap() +} + +fn assert_text_content(output: &docbox_processing::ProcessingOutput, expected: &str) { + assert!(!output.encrypted, "text files are not encrypted"); + assert!( + output.additional_files.is_empty(), + "text files should not produce additional files" + ); + + assert_eq!( + output.upload_queue.len(), + 1, + "text file should produce 1 text content file" + ); + + let first = output.upload_queue.first().unwrap(); + assert_eq!(first.mime, mime::TEXT_PLAIN); + assert!(matches!(first.ty, GeneratedFileType::TextContent)); + assert_eq!(String::from_utf8_lossy(first.bytes.as_ref()), expected); + + let pages = output + .index_metadata + .as_ref() + .expect("text file should produce index metadata") + .pages + .as_ref() + .expect("text file should produce pages"); + + assert_eq!(pages.len(), 1); + assert_eq!(pages[0].page, 0); + assert_eq!(pages[0].content, expected); +} + +#[test] +fn test_is_text_mime() { + assert!(is_text_mime(&mime("text/plain"))); + assert!(is_text_mime(&mime("text/markdown"))); + assert!(is_text_mime(&mime("text/x-markdown"))); + assert!(is_text_mime(&mime("text/csv"))); + assert!(is_text_mime(&mime("text/xml"))); + assert!(is_text_mime(&mime("application/json"))); + assert!(is_text_mime(&mime("application/ld+json"))); + assert!(is_text_mime(&mime("application/xml"))); + assert!(is_text_mime(&mime("application/atom+xml"))); + + assert!(!is_text_mime(&mime("application/pdf"))); + assert!(!is_text_mime(&mime("application/octet-stream"))); + assert!(!is_text_mime(&mime("image/png"))); +} + +#[test] +fn test_process_text_plain_sample() { + let bytes = include_bytes!("samples/documents/sample.txt"); + let output = process_text(bytes); + let expected = String::from_utf8_lossy(bytes); + assert_text_content(&output, expected.as_ref()); +} + +#[test] +fn test_process_text_markdown() { + let source = "# Heading\n\nSome **markdown** content.\n"; + let output = process_text(source.as_bytes()); + assert_text_content(&output, source); +} + +#[test] +fn test_process_text_csv() { + let source = "name,role\nAda,Engineer\n"; + let output = process_text(source.as_bytes()); + assert_text_content(&output, source); +} + +#[test] +fn test_process_text_json() { + let source = r#"{"name":"Ada","role":"Engineer"}"#; + let output = process_text(source.as_bytes()); + assert_text_content(&output, source); +} + +#[test] +fn test_process_text_xml() { + let source = "Ada\n"; + let output = process_text(source.as_bytes()); + assert_text_content(&output, source); +} + +#[test] +fn test_process_text_strips_utf8_bom() { + let mut bytes = vec![0xEF, 0xBB, 0xBF]; + bytes.extend_from_slice(b"hello"); + + let output = process_text(&bytes); + assert_text_content(&output, "hello"); +} + +#[test] +fn test_process_text_empty() { + let output = process_text(b""); + assert_text_content(&output, ""); +} + +#[tokio::test] +async fn test_process_file_text_plain() { + let bytes = include_bytes!("samples/documents/sample.txt"); + let output = process_with_mime(bytes, "text/plain") + .await + .expect("text/plain should produce processing output"); + assert_text_content(&output, String::from_utf8_lossy(bytes).as_ref()); +} + +#[tokio::test] +async fn test_process_file_markdown() { + let source = "# Title\n"; + let output = process_with_mime(source.as_bytes(), "text/markdown") + .await + .expect("text/markdown should produce processing output"); + assert_text_content(&output, source); + + let output = process_with_mime(source.as_bytes(), "text/x-markdown") + .await + .expect("text/x-markdown should produce processing output"); + assert_text_content(&output, source); +} + +#[tokio::test] +async fn test_process_file_csv() { + let source = "a,b\n1,2\n"; + let output = process_with_mime(source.as_bytes(), "text/csv") + .await + .expect("text/csv should produce processing output"); + assert_text_content(&output, source); +} + +#[tokio::test] +async fn test_process_file_json() { + let source = r#"{"ok":true}"#; + let output = process_with_mime(source.as_bytes(), "application/json") + .await + .expect("application/json should produce processing output"); + assert_text_content(&output, source); + + let output = process_with_mime(source.as_bytes(), "application/ld+json") + .await + .expect("application/ld+json should produce processing output"); + assert_text_content(&output, source); +} + +#[tokio::test] +async fn test_process_file_xml() { + let source = ""; + let output = process_with_mime(source.as_bytes(), "application/xml") + .await + .expect("application/xml should produce processing output"); + assert_text_content(&output, source); + + let output = process_with_mime(source.as_bytes(), "text/xml") + .await + .expect("text/xml should produce processing output"); + assert_text_content(&output, source); +} + +#[tokio::test] +async fn test_process_file_unrelated_mime() { + let output = process_with_mime(b"not-a-document", "application/octet-stream").await; + assert!( + output.is_none(), + "unrelated mime types should not be processed" + ); +} From 5bd49355cc6cda314610e03bf77a66c8ec329886 Mon Sep 17 00:00:00 2001 From: jacobtread Date: Mon, 21 Sep 2026 08:24:42 +1200 Subject: [PATCH 2/3] fix: existing tests, code cleanup, separate application file processing and add option to enable application file processing --- .../docbox-core/tests/test_file_update.rs | 4 +- packages/docbox-processing/src/lib.rs | 33 ++++++++++- packages/docbox-processing/src/office/mod.rs | 6 ++ packages/docbox-processing/src/text.rs | 36 ++++++------ .../tests/common/processing.rs | 9 +++ .../tests/test_process_text.rs | 58 +++++++++---------- 6 files changed, 93 insertions(+), 53 deletions(-) diff --git a/packages/docbox-core/tests/test_file_update.rs b/packages/docbox-core/tests/test_file_update.rs index 3e942ed..42e1fe4 100644 --- a/packages/docbox-core/tests/test_file_update.rs +++ b/packages/docbox-core/tests/test_file_update.rs @@ -139,7 +139,7 @@ async fn test_update_file_name_success() { ); assert_eq!(first.document_box, document_box.scope); assert!(first.page_matches.is_empty()); - assert_eq!(first.total_hits, 1); + assert_eq!(first.total_hits, 2); assert!(first.name_match); assert!(!first.content_match); } @@ -284,7 +284,7 @@ async fn test_update_file_folder_success() { ); assert_eq!(first.document_box, document_box.scope); assert!(first.page_matches.is_empty()); - assert_eq!(first.total_hits, 1); + assert_eq!(first.total_hits, 2); assert!(!first.name_match); assert!(!first.content_match); } diff --git a/packages/docbox-processing/src/lib.rs b/packages/docbox-processing/src/lib.rs index 7c168b6..079374b 100644 --- a/packages/docbox-processing/src/lib.rs +++ b/packages/docbox-processing/src/lib.rs @@ -7,7 +7,7 @@ use crate::{ image::process_image_async, office::{PdfConvertError, process_office}, pdf::{GeneratePdfImagesError, process_pdf}, - text::{is_text_mime, process_text}, + text::{is_application_file, is_text_file, process_text}, }; use ::image::{ImageError, ImageFormat}; use bytes::Bytes; @@ -163,6 +163,11 @@ pub struct ProcessingLayerConfig { /// /// Default: 300s pub process_timeout: Option, + + /// Whether to process .json and .xml files, if you don't need + /// these to be searchable its recommended to keep this disabled + /// as they can fill your search results with unwanted data if added + pub process_application_files: Option, } pub const DEFAULT_PROCESS_TIMEOUT: Duration = Duration::from_secs(300); @@ -175,6 +180,9 @@ pub enum ProcessingLayerConfigError { /// Invalid process timeout seconds #[error("DOCBOX_FILE_PROCESSING_TIMEOUT must be a number in seconds")] InvalidProcessTimeout(::Err), + /// Invalid process timeout seconds + #[error("DOCBOX_PROCESS_APPLICATION_FILES must be a number in seconds")] + InvalidProcessApplicationFiles(::Err), } impl ProcessingLayerConfig { @@ -198,9 +206,19 @@ impl ProcessingLayerConfig { }) .transpose()?; + let process_application_files = std::env::var("DOCBOX_PROCESS_APPLICATION_FILES") + .ok() + .map(|value| { + value + .parse::() + .map_err(ProcessingLayerConfigError::InvalidProcessApplicationFiles) + }) + .transpose()?; + Ok(ProcessingLayerConfig { max_unpack_iterations, process_timeout, + process_application_files, }) } } @@ -246,13 +264,22 @@ pub async fn process_file( let output = process_image_async(bytes, image_format).await?; Ok(Some(output)) } - // File is plain text, JSON, or XML - else if is_text_mime(mime) { + // File is plain text + else if is_text_file(mime) { tracing::debug!("processing text file"); let output = process_text(&bytes); Ok(Some(output)) } + // File is JSON or XML + else if is_application_file(mime) + && layer.config.process_application_files.unwrap_or_default() + { + tracing::debug!("processing application file"); + + let output = process_text(&bytes); + Ok(Some(output)) + } // No processing for this file type else { Ok(None) diff --git a/packages/docbox-processing/src/office/mod.rs b/packages/docbox-processing/src/office/mod.rs index 88e753f..30c6e8d 100644 --- a/packages/docbox-processing/src/office/mod.rs +++ b/packages/docbox-processing/src/office/mod.rs @@ -51,6 +51,9 @@ pub enum PdfConvertError { #[error("office document is password protected")] EncryptedDocument, + + #[error("no convert server specified")] + NotAvailable, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -88,6 +91,7 @@ impl OfficeConverterConfig { pub enum OfficeConverter { ConverterServer(OfficeConverterServer), ConverterLambda(OfficeConverterLambda), + Noop, } #[derive(Debug, Error)] @@ -131,6 +135,7 @@ impl OfficeConverter { match self { OfficeConverter::ConverterServer(inner) => inner.convert_to_pdf(bytes).await, OfficeConverter::ConverterLambda(inner) => inner.convert_to_pdf(bytes).await, + OfficeConverter::Noop => Err(PdfConvertError::NotAvailable), } } @@ -138,6 +143,7 @@ impl OfficeConverter { match self { OfficeConverter::ConverterServer(inner) => inner.is_convertable(mime), OfficeConverter::ConverterLambda(inner) => inner.is_convertable(mime), + OfficeConverter::Noop => false, } } } diff --git a/packages/docbox-processing/src/text.rs b/packages/docbox-processing/src/text.rs index 010a909..6e184d1 100644 --- a/packages/docbox-processing/src/text.rs +++ b/packages/docbox-processing/src/text.rs @@ -7,35 +7,35 @@ use super::{ProcessingIndexMetadata, ProcessingOutput, QueuedUpload}; const UTF8_BOM: &[u8] = &[0xEF, 0xBB, 0xBF]; -/// Checks if the provided mime should be processed as source text +/// Checks if the provided mime should be processed as text /// -/// Matches any `text/*` type, JSON (`application/json` and `+json` subtypes), -/// and XML (`application/xml` and `+xml` subtypes). -pub fn is_text_mime(mime: &Mime) -> bool { +/// Matches any `text/*` type +pub fn is_text_file(mime: &Mime) -> bool { if mime.type_() == mime::TEXT { return true; } - if mime.type_() == mime::APPLICATION { - let subtype = mime.subtype().as_str(); - return subtype == "json" - || subtype == "xml" - || subtype.ends_with("+json") - || subtype.ends_with("+xml"); + false +} + +/// Checks if the provided mime should be processed as text +/// but from a application file (json/xml) +/// +/// Matches any JSON (`application/json` and `+json` subtypes), +/// and XML (`application/xml` and `+xml` subtypes). +pub fn is_application_file(mime: &Mime) -> bool { + if mime.type_() != mime::APPLICATION { + return false; } - false + let subtype = mime.subtype().as_str(); + subtype == "json" || subtype == "xml" || subtype.ends_with("+json") || subtype.ends_with("+xml") } /// Processes a text, JSON, or XML file by decoding the source bytes /// and emitting searchable [GeneratedFileType::TextContent] pub fn process_text(file_bytes: &[u8]) -> ProcessingOutput { - let raw = if file_bytes.starts_with(UTF8_BOM) { - &file_bytes[UTF8_BOM.len()..] - } else { - file_bytes - }; - + let raw = file_bytes.strip_prefix(UTF8_BOM).unwrap_or(file_bytes); let content = String::from_utf8_lossy(raw).into_owned(); let pages = vec![DocumentPage { @@ -50,7 +50,7 @@ pub fn process_text(file_bytes: &[u8]) -> ProcessingOutput { upload_queue: vec![QueuedUpload::new( mime::TEXT_PLAIN, GeneratedFileType::TextContent, - Bytes::from(content.into_bytes()), + Bytes::from(content), )], } } diff --git a/packages/docbox-processing/tests/common/processing.rs b/packages/docbox-processing/tests/common/processing.rs index 63666aa..306ab81 100644 --- a/packages/docbox-processing/tests/common/processing.rs +++ b/packages/docbox-processing/tests/common/processing.rs @@ -39,3 +39,12 @@ pub async fn test_processing_layer( config, } } + +pub fn noop_processing_layer(config: ProcessingLayerConfig) -> ProcessingLayer { + ProcessingLayer { + office: OfficeProcessingLayer { + converter: OfficeConverter::Noop, + }, + config, + } +} diff --git a/packages/docbox-processing/tests/test_process_text.rs b/packages/docbox-processing/tests/test_process_text.rs index 5b35a8b..3506955 100644 --- a/packages/docbox-processing/tests/test_process_text.rs +++ b/packages/docbox-processing/tests/test_process_text.rs @@ -3,26 +3,14 @@ use std::str::FromStr; use bytes::Bytes; use docbox_database::models::generated_file::GeneratedFileType; use docbox_processing::{ - ProcessingLayer, ProcessingLayerConfig, - office::{OfficeConverter, OfficeProcessingLayer, convert_server::OfficeConverterServer}, - process_file, - text::{is_text_mime, process_text}, + ProcessingLayerConfig, process_file, + text::{is_application_file, is_text_file, process_text}, }; use mime::Mime; -/// Processing layer that never talks to a convert server. -/// Text processing does not use office conversion. -fn dummy_processing_layer() -> ProcessingLayer { - let converter_server = - OfficeConverterServer::from_addresses(["http://127.0.0.1:1"], false).unwrap(); +use crate::common::processing::noop_processing_layer; - ProcessingLayer { - office: OfficeProcessingLayer { - converter: OfficeConverter::ConverterServer(converter_server), - }, - config: ProcessingLayerConfig::default(), - } -} +mod common; fn mime(value: &str) -> Mime { Mime::from_str(value).unwrap() @@ -34,7 +22,10 @@ async fn process_with_mime( ) -> Option { process_file( &None, - &dummy_processing_layer(), + &noop_processing_layer(ProcessingLayerConfig { + process_application_files: Some(true), + ..Default::default() + }), Bytes::copy_from_slice(contents), &mime(mime_type), ) @@ -75,19 +66,26 @@ fn assert_text_content(output: &docbox_processing::ProcessingOutput, expected: & #[test] fn test_is_text_mime() { - assert!(is_text_mime(&mime("text/plain"))); - assert!(is_text_mime(&mime("text/markdown"))); - assert!(is_text_mime(&mime("text/x-markdown"))); - assert!(is_text_mime(&mime("text/csv"))); - assert!(is_text_mime(&mime("text/xml"))); - assert!(is_text_mime(&mime("application/json"))); - assert!(is_text_mime(&mime("application/ld+json"))); - assert!(is_text_mime(&mime("application/xml"))); - assert!(is_text_mime(&mime("application/atom+xml"))); - - assert!(!is_text_mime(&mime("application/pdf"))); - assert!(!is_text_mime(&mime("application/octet-stream"))); - assert!(!is_text_mime(&mime("image/png"))); + assert!(is_text_file(&mime("text/plain"))); + assert!(is_text_file(&mime("text/markdown"))); + assert!(is_text_file(&mime("text/x-markdown"))); + assert!(is_text_file(&mime("text/csv"))); + assert!(is_text_file(&mime("text/xml"))); + + assert!(!is_text_file(&mime("application/pdf"))); + assert!(!is_text_file(&mime("application/octet-stream"))); + assert!(!is_text_file(&mime("image/png"))); +} + +#[test] +fn test_is_application_mime() { + assert!(is_application_file(&mime("application/json"),)); + assert!(is_application_file(&mime("application/ld+json"))); + assert!(is_application_file(&mime("application/xml"))); + assert!(is_application_file(&mime("application/atom+xml"))); + assert!(!is_application_file(&mime("application/pdf"))); + assert!(!is_application_file(&mime("application/octet-stream"))); + assert!(!is_application_file(&mime("image/png"))); } #[test] From bc520faf7eeb82200114100bdbed3269e0bd4204 Mon Sep 17 00:00:00 2001 From: jacobtread Date: Sat, 26 Sep 2026 11:34:54 +1200 Subject: [PATCH 3/3] fix: subtype suffix handled incorrectly --- packages/docbox-processing/src/text.rs | 7 ++++++- packages/docbox-processing/tests/test_process_text.rs | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/docbox-processing/src/text.rs b/packages/docbox-processing/src/text.rs index 6e184d1..3dcd31c 100644 --- a/packages/docbox-processing/src/text.rs +++ b/packages/docbox-processing/src/text.rs @@ -29,7 +29,12 @@ pub fn is_application_file(mime: &Mime) -> bool { } let subtype = mime.subtype().as_str(); - subtype == "json" || subtype == "xml" || subtype.ends_with("+json") || subtype.ends_with("+xml") + if subtype == "json" || subtype == "xml" { + return true; + } + + let suffix = mime.suffix(); + suffix.is_some_and(|suffix| matches!(suffix.as_str(), "json" | "xml")) } /// Processes a text, JSON, or XML file by decoding the source bytes diff --git a/packages/docbox-processing/tests/test_process_text.rs b/packages/docbox-processing/tests/test_process_text.rs index 3506955..3a9d455 100644 --- a/packages/docbox-processing/tests/test_process_text.rs +++ b/packages/docbox-processing/tests/test_process_text.rs @@ -79,7 +79,7 @@ fn test_is_text_mime() { #[test] fn test_is_application_mime() { - assert!(is_application_file(&mime("application/json"),)); + assert!(is_application_file(&mime("application/json"))); assert!(is_application_file(&mime("application/ld+json"))); assert!(is_application_file(&mime("application/xml"))); assert!(is_application_file(&mime("application/atom+xml")));