11//! Document path recognition and optional provider-neutral Markdown conversion.
22
3+ mod selection;
4+ pub use selection:: DocumentPageSelection ;
5+ #[ cfg( feature = "document-read" ) ]
6+ mod office;
7+ #[ cfg( feature = "document-read" ) ]
8+ mod text_pages;
9+ #[ cfg( feature = "document-read" ) ]
10+ pub use text_pages:: TEXT_CHUNK_CHARS ;
11+
312#[ cfg( feature = "document-read" ) ]
413use std:: collections:: VecDeque ;
514#[ cfg( feature = "document-read" ) ]
@@ -39,6 +48,47 @@ pub struct ConvertedDocument {
3948 pub markdown : Arc < str > ,
4049 pub source_format : & ' static str ,
4150 pub pdf_coverage : Option < PdfTextCoverage > ,
51+ pub pagination : DocumentPagination ,
52+ }
53+
54+ /// Selectable source units (or explicitly synthetic text chunks), not the returned line window.
55+ #[ derive( Debug , Clone , PartialEq , Eq ) ]
56+ pub struct DocumentPagination {
57+ pub page_kind : & ' static str ,
58+ pub page_count : usize ,
59+ pub selected_pages : String ,
60+ pub selected_page_count : usize ,
61+ pub next_page : Option < u32 > ,
62+ }
63+
64+ #[ cfg( feature = "document-read" ) ]
65+ impl DocumentPagination {
66+ fn new (
67+ kind : & ' static str ,
68+ count : usize ,
69+ selection : Option < & DocumentPageSelection > ,
70+ ) -> Result < Self , DocumentConversionError > {
71+ let pages = selection
72+ . map ( |selection| selection. resolve ( count) )
73+ . transpose ( )
74+ . map_err ( |error| DocumentConversionError :: new ( "invalidPages" , error) ) ?;
75+ let last = pages. as_ref ( ) . and_then ( |pages| pages. last ( ) ) . copied ( ) ;
76+ Ok ( Self {
77+ page_kind : kind,
78+ page_count : count,
79+ selected_pages : selection
80+ . map ( ToString :: to_string)
81+ . unwrap_or_else ( || match count {
82+ 0 => String :: new ( ) ,
83+ 1 => "1" . to_string ( ) ,
84+ _ => format ! ( "1-{count}" ) ,
85+ } ) ,
86+ selected_page_count : pages. as_ref ( ) . map_or ( count, Vec :: len) ,
87+ next_page : last
88+ . filter ( |last| ( * last as usize ) < count)
89+ . and_then ( |last| last. checked_add ( 1 ) ) ,
90+ } )
91+ }
4292}
4393
4494/// Source-page coverage, independent of the line window returned from the extracted Markdown.
@@ -88,10 +138,11 @@ impl PdfTextCoverage {
88138}
89139
90140#[ cfg( feature = "document-read" ) ]
91- #[ derive( Clone , Copy , Debug , PartialEq , Eq ) ]
141+ #[ derive( Clone , Debug , PartialEq , Eq ) ]
92142struct DocumentCacheKey {
93143 source_sha256 : [ u8 ; 32 ] ,
94144 format : Format ,
145+ selection : Option < DocumentPageSelection > ,
95146}
96147
97148#[ cfg( feature = "document-read" ) ]
@@ -109,8 +160,8 @@ struct DocumentCache {
109160
110161#[ cfg( feature = "document-read" ) ]
111162impl DocumentCache {
112- fn get ( & mut self , key : DocumentCacheKey ) -> Option < ConvertedDocument > {
113- let index = self . entries . iter ( ) . position ( |entry| entry. key == key) ?;
163+ fn get ( & mut self , key : & DocumentCacheKey ) -> Option < ConvertedDocument > {
164+ let index = self . entries . iter ( ) . position ( |entry| & entry. key == key) ?;
114165 let entry = self . entries . remove ( index) ?;
115166 let document = entry. document . clone ( ) ;
116167 self . entries . push_back ( entry) ;
@@ -190,6 +241,16 @@ pub fn is_supported_document_path(path: &str) -> bool {
190241pub async fn convert_document_to_markdown (
191242 bytes : Vec < u8 > ,
192243 path_hint : String ,
244+ ) -> Result < ConvertedDocument , DocumentConversionError > {
245+ convert_document_pages_to_markdown ( bytes, path_hint, None ) . await
246+ }
247+
248+ /// Select document units before applying the caller's Markdown line window.
249+ #[ cfg( feature = "document-read" ) ]
250+ pub async fn convert_document_pages_to_markdown (
251+ bytes : Vec < u8 > ,
252+ path_hint : String ,
253+ selection : Option < DocumentPageSelection > ,
193254) -> Result < ConvertedDocument , DocumentConversionError > {
194255 if bytes. len ( ) > MAX_DOCUMENT_INPUT_BYTES {
195256 return Err ( DocumentConversionError :: new (
@@ -216,7 +277,7 @@ pub async fn convert_document_to_markdown(
216277 // Keep the permit inside the blocking task. If the async caller is cancelled, the parser
217278 // still occupies its bounded slot until the synchronous conversion actually exits.
218279 let _permit = permit;
219- convert_document_to_markdown_sync ( & bytes, & path_hint)
280+ convert_document_pages_to_markdown_sync ( & bytes, & path_hint, selection . as_ref ( ) )
220281 } )
221282 . await
222283 . map_err ( |error| {
@@ -234,9 +295,10 @@ fn document_conversion_semaphore() -> &'static Arc<Semaphore> {
234295}
235296
236297#[ cfg( feature = "document-read" ) ]
237- fn convert_document_to_markdown_sync (
298+ fn convert_document_pages_to_markdown_sync (
238299 bytes : & [ u8 ] ,
239300 path_hint : & str ,
301+ selection : Option < & DocumentPageSelection > ,
240302) -> Result < ConvertedDocument , DocumentConversionError > {
241303 let format = Format :: from_bytes ( bytes)
242304 . or_else ( || Format :: from_path ( Path :: new ( path_hint) ) )
@@ -249,23 +311,35 @@ fn convert_document_to_markdown_sync(
249311 let cache_key = DocumentCacheKey {
250312 source_sha256 : Sha256 :: digest ( bytes) . into ( ) ,
251313 format,
314+ selection : selection. cloned ( ) ,
252315 } ;
253316 if let Some ( document) = document_cache ( )
254317 . lock ( )
255318 . unwrap_or_else ( |poisoned| poisoned. into_inner ( ) )
256- . get ( cache_key)
319+ . get ( & cache_key)
257320 {
258321 return Ok ( document) ;
259322 }
260323
261324 let source_format = format_name ( format) ;
262- let ( markdown, pdf_coverage) = if format == Format :: Pdf {
263- let ( markdown, coverage) = extract_pdf_text ( bytes) ?;
264- ( markdown, Some ( coverage) )
325+ let ( markdown, pdf_coverage, pagination) = if format == Format :: Pdf {
326+ let ( markdown, coverage) = extract_pdf_text ( bytes, selection) ?;
327+ let pagination = DocumentPagination :: new ( "pdf_page" , coverage. page_count , selection) ?;
328+ ( markdown, Some ( coverage) , pagination)
329+ } else if let Some ( index) = office:: OfficePageIndex :: read ( bytes, format) ? {
330+ let pagination = DocumentPagination :: new ( index. page_kind , index. page_count ( ) , selection) ?;
331+ let selected_bytes = selection
332+ . map ( |selection| index. select ( bytes, selection) )
333+ . transpose ( ) ?;
334+ let markdown =
335+ anydoc:: to_markdown_bytes ( selected_bytes. as_deref ( ) . unwrap_or ( bytes) , format)
336+ . map_err ( |error| DocumentConversionError :: new ( error. code ( ) , error. to_string ( ) ) ) ?;
337+ ( markdown, None , pagination)
265338 } else {
266339 let markdown = anydoc:: to_markdown_bytes ( bytes, format)
267340 . map_err ( |error| DocumentConversionError :: new ( error. code ( ) , error. to_string ( ) ) ) ?;
268- ( markdown, None )
341+ let ( markdown, pagination) = text_pages:: select_text ( markdown, selection) ?;
342+ ( markdown, None , pagination)
269343 } ;
270344 if markdown. len ( ) > MAX_DOCUMENT_MARKDOWN_BYTES {
271345 return Err ( DocumentConversionError :: new (
@@ -281,6 +355,7 @@ fn convert_document_to_markdown_sync(
281355 markdown : Arc :: from ( markdown) ,
282356 source_format,
283357 pdf_coverage,
358+ pagination,
284359 } ;
285360 document_cache ( )
286361 . lock ( )
@@ -292,23 +367,42 @@ fn convert_document_to_markdown_sync(
292367/// anydoc's PDF convenience API rejects the whole document when any page needs OCR.
293368/// Use the same parser's page API to retain reliable text and explicitly mark gaps.
294369#[ cfg( feature = "document-read" ) ]
295- fn extract_pdf_text ( bytes : & [ u8 ] ) -> Result < ( String , PdfTextCoverage ) , DocumentConversionError > {
296- let extraction = pdf_inspector:: extract_pages_markdown_mem ( bytes, None ) . map_err ( |error| {
370+ fn extract_pdf_text (
371+ bytes : & [ u8 ] ,
372+ selection : Option < & DocumentPageSelection > ,
373+ ) -> Result < ( String , PdfTextCoverage ) , DocumentConversionError > {
374+ let map_error = |error : pdf_inspector:: PdfError | {
297375 let code = match & error {
298376 pdf_inspector:: PdfError :: Encrypted => "encrypted" ,
299377 pdf_inspector:: PdfError :: Io ( _) => "io" ,
300378 _ => "malformed" ,
301379 } ;
302380 DocumentConversionError :: new ( code, error. to_string ( ) )
303- } ) ?;
381+ } ;
382+ // Validate against the parsed source page tree before passing zero-based indices to the
383+ // backend, which otherwise returns an OCR placeholder for a nonexistent page.
384+ let source_page_count = selection
385+ . map ( |_| {
386+ pdf_inspector:: classify_pdf_mem ( bytes)
387+ . map ( |classification| classification. page_count as usize )
388+ . map_err ( map_error)
389+ } )
390+ . transpose ( ) ?;
391+ let selected_pages = selection
392+ . map ( |selection| selection. resolve ( source_page_count. unwrap_or ( 0 ) ) )
393+ . transpose ( )
394+ . map_err ( |error| DocumentConversionError :: new ( "invalidPages" , error) ) ?
395+ . map ( |pages| pages. into_iter ( ) . map ( |page| page - 1 ) . collect :: < Vec < _ > > ( ) ) ;
396+ let extraction = pdf_inspector:: extract_pages_markdown_mem ( bytes, selected_pages. as_deref ( ) )
397+ . map_err ( map_error) ?;
304398 if extraction. pages . is_empty ( ) {
305399 return Err ( DocumentConversionError :: new (
306400 "malformed" ,
307401 "PDF contains no pages" ,
308402 ) ) ;
309403 }
310404 let mut coverage = PdfTextCoverage {
311- page_count : extraction. pages . len ( ) ,
405+ page_count : source_page_count . unwrap_or ( extraction. pages . len ( ) ) ,
312406 extracted_pages : Vec :: new ( ) ,
313407 pages_needing_ocr : Vec :: new ( ) ,
314408 } ;
@@ -364,6 +458,14 @@ fn format_name(format: Format) -> &'static str {
364458mod tests {
365459 use super :: * ;
366460
461+ #[ cfg( feature = "document-read" ) ]
462+ fn convert_document_to_markdown_sync (
463+ bytes : & [ u8 ] ,
464+ path : & str ,
465+ ) -> Result < ConvertedDocument , DocumentConversionError > {
466+ convert_document_pages_to_markdown_sync ( bytes, path, None )
467+ }
468+
367469 #[ test]
368470 fn pdf_gap_summary_is_bounded_without_losing_page_metadata ( ) {
369471 let coverage = PdfTextCoverage {
0 commit comments