diff --git a/README.md b/README.md index e374927..06b01cc 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,10 @@
+Similarity is not relevance. ReasonDB replaces broken RAG & vector search. + +
+ ReasonDB Client Demo
@@ -152,6 +156,7 @@ flowchart TD A["Documents / URLs"] -->|Extractor Plugin| B["Markdown"] B -->|Post-Processor Plugin| C["Cleaned Markdown"] C -->|Chunker| D["Semantic Chunks"] + P["Pre-chunked JSON"] -->|"bypasses extract + chunk"| D D --> E["Build Hierarchical Tree"] E -->|Bottom-up| F["LLM Summarizes Each Node"] end @@ -173,9 +178,9 @@ flowchart TD ``` 1. **Extract** - Extractor plugins convert documents and URLs to Markdown (built-in: [MarkItDown](https://github.com/microsoft/markitdown)) -2. **Chunk** - Content is split into semantic chunks with heading detection -3. **Build Tree** - Chunks are organized into a hierarchical tree structure -4. **Summarize** - LLM generates summaries for each node (bottom-up) +2. **Chunk** - Content is split into semantic chunks with heading detection — or bypass entirely with pre-chunked JSON via `/ingest/chunks` +3. **Build Tree** - Chunks are organized into a hierarchical tree structure, preserving per-chunk metadata (page numbers, line ranges, custom attributes) +4. **Summarize** - LLM generates summaries for each node (bottom-up); pre-supplied summaries are used as-is 5. **Search** - 4-phase pipeline: BM25 candidate selection → recursive tree-grep filtering → LLM summary ranking → parallel beam-search traversal 6. **Return** - Relevant content with extracted answers, confidence scores, and the full reasoning path diff --git a/apps/reasondb-client/src-tauri/icons/128x128.png b/apps/reasondb-client/src-tauri/icons/128x128.png index 0714c59..bb58bf7 100644 Binary files a/apps/reasondb-client/src-tauri/icons/128x128.png and b/apps/reasondb-client/src-tauri/icons/128x128.png differ diff --git a/apps/reasondb-client/src-tauri/icons/128x128@2x.png b/apps/reasondb-client/src-tauri/icons/128x128@2x.png index be301eb..13f68e2 100644 Binary files a/apps/reasondb-client/src-tauri/icons/128x128@2x.png and b/apps/reasondb-client/src-tauri/icons/128x128@2x.png differ diff --git a/apps/reasondb-client/src-tauri/icons/32x32.png b/apps/reasondb-client/src-tauri/icons/32x32.png index 4aa5e1d..307112b 100644 Binary files a/apps/reasondb-client/src-tauri/icons/32x32.png and b/apps/reasondb-client/src-tauri/icons/32x32.png differ diff --git a/apps/reasondb-client/src-tauri/icons/64x64.png b/apps/reasondb-client/src-tauri/icons/64x64.png index 5bf33e1..0df0df9 100644 Binary files a/apps/reasondb-client/src-tauri/icons/64x64.png and b/apps/reasondb-client/src-tauri/icons/64x64.png differ diff --git a/apps/reasondb-client/src-tauri/icons/Square107x107Logo.png b/apps/reasondb-client/src-tauri/icons/Square107x107Logo.png index 6463b05..c114a53 100644 Binary files a/apps/reasondb-client/src-tauri/icons/Square107x107Logo.png and b/apps/reasondb-client/src-tauri/icons/Square107x107Logo.png differ diff --git a/apps/reasondb-client/src-tauri/icons/Square142x142Logo.png b/apps/reasondb-client/src-tauri/icons/Square142x142Logo.png index 0892fd5..f458cc2 100644 Binary files a/apps/reasondb-client/src-tauri/icons/Square142x142Logo.png and b/apps/reasondb-client/src-tauri/icons/Square142x142Logo.png differ diff --git a/apps/reasondb-client/src-tauri/icons/Square150x150Logo.png b/apps/reasondb-client/src-tauri/icons/Square150x150Logo.png index 17e9453..078fc7a 100644 Binary files a/apps/reasondb-client/src-tauri/icons/Square150x150Logo.png and b/apps/reasondb-client/src-tauri/icons/Square150x150Logo.png differ diff --git a/apps/reasondb-client/src-tauri/icons/Square284x284Logo.png b/apps/reasondb-client/src-tauri/icons/Square284x284Logo.png index 031bb32..d34fd3b 100644 Binary files a/apps/reasondb-client/src-tauri/icons/Square284x284Logo.png and b/apps/reasondb-client/src-tauri/icons/Square284x284Logo.png differ diff --git a/apps/reasondb-client/src-tauri/icons/Square30x30Logo.png b/apps/reasondb-client/src-tauri/icons/Square30x30Logo.png index 5b86ecc..ee4bf7e 100644 Binary files a/apps/reasondb-client/src-tauri/icons/Square30x30Logo.png and b/apps/reasondb-client/src-tauri/icons/Square30x30Logo.png differ diff --git a/apps/reasondb-client/src-tauri/icons/Square310x310Logo.png b/apps/reasondb-client/src-tauri/icons/Square310x310Logo.png index 8b0ddfc..15f935f 100644 Binary files a/apps/reasondb-client/src-tauri/icons/Square310x310Logo.png and b/apps/reasondb-client/src-tauri/icons/Square310x310Logo.png differ diff --git a/apps/reasondb-client/src-tauri/icons/Square44x44Logo.png b/apps/reasondb-client/src-tauri/icons/Square44x44Logo.png index 8d360ba..113fe6e 100644 Binary files a/apps/reasondb-client/src-tauri/icons/Square44x44Logo.png and b/apps/reasondb-client/src-tauri/icons/Square44x44Logo.png differ diff --git a/apps/reasondb-client/src-tauri/icons/Square71x71Logo.png b/apps/reasondb-client/src-tauri/icons/Square71x71Logo.png index 55fe56a..3c69bde 100644 Binary files a/apps/reasondb-client/src-tauri/icons/Square71x71Logo.png and b/apps/reasondb-client/src-tauri/icons/Square71x71Logo.png differ diff --git a/apps/reasondb-client/src-tauri/icons/Square89x89Logo.png b/apps/reasondb-client/src-tauri/icons/Square89x89Logo.png index 255b4d6..f998ccb 100644 Binary files a/apps/reasondb-client/src-tauri/icons/Square89x89Logo.png and b/apps/reasondb-client/src-tauri/icons/Square89x89Logo.png differ diff --git a/apps/reasondb-client/src-tauri/icons/StoreLogo.png b/apps/reasondb-client/src-tauri/icons/StoreLogo.png index a234c99..97d3c6f 100644 Binary files a/apps/reasondb-client/src-tauri/icons/StoreLogo.png and b/apps/reasondb-client/src-tauri/icons/StoreLogo.png differ diff --git a/apps/reasondb-client/src-tauri/icons/android/mipmap-hdpi/ic_launcher.png b/apps/reasondb-client/src-tauri/icons/android/mipmap-hdpi/ic_launcher.png index 35feb41..cb41a7b 100644 Binary files a/apps/reasondb-client/src-tauri/icons/android/mipmap-hdpi/ic_launcher.png and b/apps/reasondb-client/src-tauri/icons/android/mipmap-hdpi/ic_launcher.png differ diff --git a/apps/reasondb-client/src-tauri/icons/android/mipmap-hdpi/ic_launcher_foreground.png b/apps/reasondb-client/src-tauri/icons/android/mipmap-hdpi/ic_launcher_foreground.png index 61e89ff..2e66439 100644 Binary files a/apps/reasondb-client/src-tauri/icons/android/mipmap-hdpi/ic_launcher_foreground.png and b/apps/reasondb-client/src-tauri/icons/android/mipmap-hdpi/ic_launcher_foreground.png differ diff --git a/apps/reasondb-client/src-tauri/icons/android/mipmap-hdpi/ic_launcher_round.png b/apps/reasondb-client/src-tauri/icons/android/mipmap-hdpi/ic_launcher_round.png index a7728da..a9088ae 100644 Binary files a/apps/reasondb-client/src-tauri/icons/android/mipmap-hdpi/ic_launcher_round.png and b/apps/reasondb-client/src-tauri/icons/android/mipmap-hdpi/ic_launcher_round.png differ diff --git a/apps/reasondb-client/src-tauri/icons/android/mipmap-mdpi/ic_launcher.png b/apps/reasondb-client/src-tauri/icons/android/mipmap-mdpi/ic_launcher.png index 2308e29..071a2f9 100644 Binary files a/apps/reasondb-client/src-tauri/icons/android/mipmap-mdpi/ic_launcher.png and b/apps/reasondb-client/src-tauri/icons/android/mipmap-mdpi/ic_launcher.png differ diff --git a/apps/reasondb-client/src-tauri/icons/android/mipmap-mdpi/ic_launcher_foreground.png b/apps/reasondb-client/src-tauri/icons/android/mipmap-mdpi/ic_launcher_foreground.png index a87a4b3..85ebe6e 100644 Binary files a/apps/reasondb-client/src-tauri/icons/android/mipmap-mdpi/ic_launcher_foreground.png and b/apps/reasondb-client/src-tauri/icons/android/mipmap-mdpi/ic_launcher_foreground.png differ diff --git a/apps/reasondb-client/src-tauri/icons/android/mipmap-mdpi/ic_launcher_round.png b/apps/reasondb-client/src-tauri/icons/android/mipmap-mdpi/ic_launcher_round.png index 4f37c80..7094d5a 100644 Binary files a/apps/reasondb-client/src-tauri/icons/android/mipmap-mdpi/ic_launcher_round.png and b/apps/reasondb-client/src-tauri/icons/android/mipmap-mdpi/ic_launcher_round.png differ diff --git a/apps/reasondb-client/src-tauri/icons/android/mipmap-xhdpi/ic_launcher.png b/apps/reasondb-client/src-tauri/icons/android/mipmap-xhdpi/ic_launcher.png index a167b8d..43e2bcd 100644 Binary files a/apps/reasondb-client/src-tauri/icons/android/mipmap-xhdpi/ic_launcher.png and b/apps/reasondb-client/src-tauri/icons/android/mipmap-xhdpi/ic_launcher.png differ diff --git a/apps/reasondb-client/src-tauri/icons/android/mipmap-xhdpi/ic_launcher_foreground.png b/apps/reasondb-client/src-tauri/icons/android/mipmap-xhdpi/ic_launcher_foreground.png index 51b0dc2..21ff459 100644 Binary files a/apps/reasondb-client/src-tauri/icons/android/mipmap-xhdpi/ic_launcher_foreground.png and b/apps/reasondb-client/src-tauri/icons/android/mipmap-xhdpi/ic_launcher_foreground.png differ diff --git a/apps/reasondb-client/src-tauri/icons/android/mipmap-xhdpi/ic_launcher_round.png b/apps/reasondb-client/src-tauri/icons/android/mipmap-xhdpi/ic_launcher_round.png index 5cb7b41..0dfa97e 100644 Binary files a/apps/reasondb-client/src-tauri/icons/android/mipmap-xhdpi/ic_launcher_round.png and b/apps/reasondb-client/src-tauri/icons/android/mipmap-xhdpi/ic_launcher_round.png differ diff --git a/apps/reasondb-client/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher.png b/apps/reasondb-client/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher.png index 6dc11f7..1da5029 100644 Binary files a/apps/reasondb-client/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher.png and b/apps/reasondb-client/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher.png differ diff --git a/apps/reasondb-client/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher_foreground.png b/apps/reasondb-client/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher_foreground.png index 7b1fe84..ae70e40 100644 Binary files a/apps/reasondb-client/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher_foreground.png and b/apps/reasondb-client/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher_foreground.png differ diff --git a/apps/reasondb-client/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher_round.png b/apps/reasondb-client/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher_round.png index 1f35468..966df05 100644 Binary files a/apps/reasondb-client/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher_round.png and b/apps/reasondb-client/src-tauri/icons/android/mipmap-xxhdpi/ic_launcher_round.png differ diff --git a/apps/reasondb-client/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher.png b/apps/reasondb-client/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher.png index 4e5e18c..d6bde20 100644 Binary files a/apps/reasondb-client/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher.png and b/apps/reasondb-client/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher.png differ diff --git a/apps/reasondb-client/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher_foreground.png b/apps/reasondb-client/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher_foreground.png index e3df2e0..d957a41 100644 Binary files a/apps/reasondb-client/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher_foreground.png and b/apps/reasondb-client/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher_foreground.png differ diff --git a/apps/reasondb-client/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher_round.png b/apps/reasondb-client/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher_round.png index 542c979..14d6c13 100644 Binary files a/apps/reasondb-client/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher_round.png and b/apps/reasondb-client/src-tauri/icons/android/mipmap-xxxhdpi/ic_launcher_round.png differ diff --git a/apps/reasondb-client/src-tauri/icons/icon.icns b/apps/reasondb-client/src-tauri/icons/icon.icns index c906234..cbcb1d3 100644 Binary files a/apps/reasondb-client/src-tauri/icons/icon.icns and b/apps/reasondb-client/src-tauri/icons/icon.icns differ diff --git a/apps/reasondb-client/src-tauri/icons/icon.ico b/apps/reasondb-client/src-tauri/icons/icon.ico index 6955fab..3b1d8bb 100644 Binary files a/apps/reasondb-client/src-tauri/icons/icon.ico and b/apps/reasondb-client/src-tauri/icons/icon.ico differ diff --git a/apps/reasondb-client/src-tauri/icons/icon.png b/apps/reasondb-client/src-tauri/icons/icon.png index 8e461b7..5008dc1 100644 Binary files a/apps/reasondb-client/src-tauri/icons/icon.png and b/apps/reasondb-client/src-tauri/icons/icon.png differ diff --git a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-20x20@1x.png b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-20x20@1x.png index e2d2527..660c0bb 100644 Binary files a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-20x20@1x.png and b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-20x20@1x.png differ diff --git a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-20x20@2x-1.png b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-20x20@2x-1.png index 3c8ec66..6ac70ec 100644 Binary files a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-20x20@2x-1.png and b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-20x20@2x-1.png differ diff --git a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-20x20@2x.png b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-20x20@2x.png index 3c8ec66..6ac70ec 100644 Binary files a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-20x20@2x.png and b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-20x20@2x.png differ diff --git a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-20x20@3x.png b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-20x20@3x.png index b8701e2..8f35869 100644 Binary files a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-20x20@3x.png and b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-20x20@3x.png differ diff --git a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-29x29@1x.png b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-29x29@1x.png index ed1e424..d61921d 100644 Binary files a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-29x29@1x.png and b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-29x29@1x.png differ diff --git a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-29x29@2x-1.png b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-29x29@2x-1.png index 1cfac5d..518df15 100644 Binary files a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-29x29@2x-1.png and b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-29x29@2x-1.png differ diff --git a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-29x29@2x.png b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-29x29@2x.png index 1cfac5d..518df15 100644 Binary files a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-29x29@2x.png and b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-29x29@2x.png differ diff --git a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-29x29@3x.png b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-29x29@3x.png index 72a0aa6..56533a7 100644 Binary files a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-29x29@3x.png and b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-29x29@3x.png differ diff --git a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-40x40@1x.png b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-40x40@1x.png index 3c8ec66..6ac70ec 100644 Binary files a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-40x40@1x.png and b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-40x40@1x.png differ diff --git a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-40x40@2x-1.png b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-40x40@2x-1.png index 6bafffb..6a2229a 100644 Binary files a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-40x40@2x-1.png and b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-40x40@2x-1.png differ diff --git a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-40x40@2x.png b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-40x40@2x.png index 6bafffb..6a2229a 100644 Binary files a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-40x40@2x.png and b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-40x40@2x.png differ diff --git a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-40x40@3x.png b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-40x40@3x.png index 28c5947..150ee0f 100644 Binary files a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-40x40@3x.png and b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-40x40@3x.png differ diff --git a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-512@2x.png b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-512@2x.png index e11729f..35db209 100644 Binary files a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-512@2x.png and b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-512@2x.png differ diff --git a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-60x60@2x.png b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-60x60@2x.png index 28c5947..150ee0f 100644 Binary files a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-60x60@2x.png and b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-60x60@2x.png differ diff --git a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-60x60@3x.png b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-60x60@3x.png index 520f8ba..1195bd7 100644 Binary files a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-60x60@3x.png and b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-60x60@3x.png differ diff --git a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-76x76@1x.png b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-76x76@1x.png index bcb840d..25e3cb4 100644 Binary files a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-76x76@1x.png and b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-76x76@1x.png differ diff --git a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-76x76@2x.png b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-76x76@2x.png index 720fdf5..4fc6578 100644 Binary files a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-76x76@2x.png and b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-76x76@2x.png differ diff --git a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-83.5x83.5@2x.png b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-83.5x83.5@2x.png index 8c8cacc..04892bb 100644 Binary files a/apps/reasondb-client/src-tauri/icons/ios/AppIcon-83.5x83.5@2x.png and b/apps/reasondb-client/src-tauri/icons/ios/AppIcon-83.5x83.5@2x.png differ diff --git a/apps/reasondb-client/src/assets/logo.png b/apps/reasondb-client/src/assets/logo.png index 99e8c84..769075b 100644 Binary files a/apps/reasondb-client/src/assets/logo.png and b/apps/reasondb-client/src/assets/logo.png differ diff --git a/apps/reasondb-client/src/assets/logo.svg b/apps/reasondb-client/src/assets/logo.svg index b6f837d..3c3a0c4 100644 --- a/apps/reasondb-client/src/assets/logo.svg +++ b/apps/reasondb-client/src/assets/logo.svg @@ -1,7 +1,4 @@ - - - - - - + + + diff --git a/assets/banner.png b/assets/banner.png new file mode 100644 index 0000000..2496855 Binary files /dev/null and b/assets/banner.png differ diff --git a/assets/logo-black.png b/assets/logo-black.png index 52b3c2f..ea8cd7d 100644 Binary files a/assets/logo-black.png and b/assets/logo-black.png differ diff --git a/assets/logo.svg b/assets/logo.svg index 0c1c687..3c3a0c4 100644 --- a/assets/logo.svg +++ b/assets/logo.svg @@ -1,7 +1,4 @@ - - - - - - + + + diff --git a/crates/reasondb-core/src/model/node.rs b/crates/reasondb-core/src/model/node.rs index 49e012f..69aea4f 100644 --- a/crates/reasondb-core/src/model/node.rs +++ b/crates/reasondb-core/src/model/node.rs @@ -191,6 +191,12 @@ pub struct NodeMetadata { /// Page number in the source document (if applicable) pub page_number: Option, + /// Line number where this node starts in the source file + pub start_line: Option, + + /// Line number where this node ends in the source file + pub end_line: Option, + /// Type of section (e.g., "chapter", "section", "paragraph") pub section_type: Option, @@ -224,6 +230,13 @@ impl NodeMetadata { self } + /// Set the source line range. + pub fn with_lines(mut self, start: u32, end: u32) -> Self { + self.start_line = Some(start); + self.end_line = Some(end); + self + } + /// Add a custom attribute. pub fn with_attribute(mut self, key: &str, value: &str) -> Self { self.attributes.insert(key.to_string(), value.to_string()); diff --git a/crates/reasondb-core/src/store/migration.rs b/crates/reasondb-core/src/store/migration.rs index 69917eb..57266f1 100644 --- a/crates/reasondb-core/src/store/migration.rs +++ b/crates/reasondb-core/src/store/migration.rs @@ -150,6 +150,8 @@ impl From for PageNode { image_path: v1.image_path, metadata: NodeMetadata { page_number: v1.metadata.page_number, + start_line: None, + end_line: None, section_type: v1.metadata.section_type, confidence_score: v1.metadata.confidence_score, token_count: v1.metadata.token_count, diff --git a/crates/reasondb-ingest/src/chunker.rs b/crates/reasondb-ingest/src/chunker.rs index c42ddf8..3f14cd3 100644 --- a/crates/reasondb-ingest/src/chunker.rs +++ b/crates/reasondb-ingest/src/chunker.rs @@ -39,6 +39,15 @@ pub struct TextChunk { pub start_page: Option, /// End page number pub end_page: Option, + /// Start line number in the source file + pub start_line: Option, + /// End line number in the source file + pub end_line: Option, + /// Extra caller-supplied attributes passed through to NodeMetadata.attributes + pub attributes: std::collections::HashMap, + /// Optional pre-computed summary. When set, the BatchSummarizer will skip + /// this node and use this value directly as the node summary. + pub summary: Option, } /// Chunking strategy to use during document ingestion @@ -174,6 +183,10 @@ impl SemanticChunker { word_count, start_page: None, end_page: None, + start_line: None, + end_line: None, + attributes: Default::default(), + summary: None, }); } diff --git a/crates/reasondb-ingest/src/lib.rs b/crates/reasondb-ingest/src/lib.rs index e2b75d2..ce2ae2e 100644 --- a/crates/reasondb-ingest/src/lib.rs +++ b/crates/reasondb-ingest/src/lib.rs @@ -52,7 +52,8 @@ pub use chunker::{ pub use error::{IngestError, Result}; pub use extractor::{DocumentType, ExtractionResult, SmartExtractor}; pub use pipeline::{ - IngestPipeline, IngestResult, IngestStats, NoOpReasoner, PipelineBuilder, PipelineConfig, + ChunkInput, IngestPipeline, IngestResult, IngestStats, NoOpReasoner, PipelineBuilder, + PipelineConfig, }; pub use summarizer::{BatchSummarizer, MockSummarizer, NodeSummarizer, SummarizerConfig}; pub use tree_builder::TreeBuilder; diff --git a/crates/reasondb-ingest/src/pipeline.rs b/crates/reasondb-ingest/src/pipeline.rs index 8d36573..c57ae77 100644 --- a/crates/reasondb-ingest/src/pipeline.rs +++ b/crates/reasondb-ingest/src/pipeline.rs @@ -7,6 +7,7 @@ //! 4. Generate summaries //! 5. Store in database +use std::collections::HashMap; use std::path::Path; use std::sync::Arc; use tracing::{debug, info, warn}; @@ -86,6 +87,34 @@ pub struct IngestStats { pub total_time_ms: u64, } +/// A pre-split chunk of text with caller-supplied metadata. +/// +/// Used with [`IngestPipeline::ingest_chunks`] to bypass the extraction and +/// chunking stages and feed content directly into the tree builder. +#[derive(Debug, Clone)] +pub struct ChunkInput { + /// The text content of this chunk + pub text: String, + /// Optional heading that names this chunk (becomes the node title) + pub heading: Option, + /// Optional pre-computed summary for this chunk. + /// + /// When provided, this summary is applied directly to the node and the LLM + /// summarization step is skipped for that node. If absent, a summary is + /// generated automatically by the `BatchSummarizer`. + pub summary: Option, + /// Free-form metadata — any key-value pairs from the caller. + /// + /// Well-known keys extracted to typed `NodeMetadata` fields: + /// - `"page_number"` → `NodeMetadata.page_number` + /// - `"start_line"` → `NodeMetadata.start_line` + /// - `"end_line"` → `NodeMetadata.end_line` + /// - `"section_type"` → `NodeMetadata.section_type` + /// + /// All other keys are stored in `NodeMetadata.attributes`. + pub metadata: HashMap, +} + /// The main ingestion pipeline pub struct IngestPipeline { config: PipelineConfig, @@ -318,6 +347,10 @@ impl IngestPipeline { word_count, start_page: None, end_page: None, + start_line: None, + end_line: None, + attributes: Default::default(), + summary: None, } }) .collect(), @@ -550,6 +583,10 @@ impl IngestPipeline { word_count, start_page: None, end_page: None, + start_line: None, + end_line: None, + attributes: Default::default(), + summary: None, }); } @@ -685,6 +722,172 @@ impl IngestPipeline { Ok(IngestResult { stats, ..result }) } + /// Ingest pre-chunked content without extraction or chunking. + /// + /// Each [`ChunkInput`] is converted directly to a tree node. The + /// extraction and chunking stages are skipped entirely. Summarization + /// still runs if `config.generate_summaries` is enabled. + pub async fn ingest_chunks( + &self, + title: &str, + table_id: &str, + chunks: Vec, + ) -> Result { + let start = std::time::Instant::now(); + let mut stats = IngestStats::default(); + + info!( + "Starting pre-chunked ingestion: {} ({} chunks)", + title, + chunks.len() + ); + + let text_chunks = self.convert_chunk_inputs(chunks); + stats.chunks_created = text_chunks.len(); + + let (document, mut nodes) = self.tree_builder.build(title, table_id, text_chunks)?; + stats.nodes_created = nodes.len(); + + if self.config.generate_summaries { + if let Some(ref reasoner) = self.reasoner { + let summarizer = + BatchSummarizer::new(reasoner, 10, self.config.summarizer.max_concurrent); + summarizer.summarize_batch(&mut nodes).await?; + stats.summaries_generated = nodes.len(); + } + } + + stats.total_time_ms = start.elapsed().as_millis() as u64; + Ok(IngestResult { + document, + nodes, + stats, + }) + } + + /// Ingest pre-chunked content and store in database. + /// + /// The `table_id` must reference an existing table in the database. + pub async fn ingest_chunks_and_store( + &self, + title: &str, + table_id: &str, + chunks: Vec, + store: Arc, + ) -> Result + where + R: Clone + Send + Sync + 'static, + { + if !self.config.store_in_db { + return self.ingest_chunks(title, table_id, chunks).await; + } + + let start = std::time::Instant::now(); + let mut stats = IngestStats::default(); + + info!( + "Starting pre-chunked ingestion with store: {} ({} chunks)", + title, + chunks.len() + ); + + let text_chunks = self.convert_chunk_inputs(chunks); + stats.chunks_created = text_chunks.len(); + + let (document, mut nodes) = self.tree_builder.build(title, table_id, text_chunks)?; + stats.nodes_created = nodes.len(); + + // Early flush: store document and nodes before summarization + store.insert_document(&document)?; + store.insert_nodes(&nodes)?; + + if let Some(ref cb) = self.checkpoint_callback { + cb(document.id.clone()); + } + + if self.config.generate_summaries { + if let Some(ref reasoner) = self.reasoner { + let summarizer = + BatchSummarizer::new(reasoner, 10, self.config.summarizer.max_concurrent) + .with_store(store.clone()); + summarizer.summarize_batch(&mut nodes).await?; + stats.summaries_generated = nodes.len(); + } + } + + stats.total_time_ms = start.elapsed().as_millis() as u64; + Ok(IngestResult { + document, + nodes, + stats, + }) + } + + /// Convert [`ChunkInput`]s into [`TextChunk`]s, extracting well-known + /// metadata fields and storing the rest in `NodeMetadata.attributes` + /// (threaded through via `TextChunk.start_line` / `end_line` / etc.). + fn convert_chunk_inputs(&self, chunks: Vec) -> Vec { + use crate::chunker::DetectedHeading; + + chunks + .into_iter() + .enumerate() + .map(|(i, c)| { + let page_number = c + .metadata + .get("page_number") + .and_then(|v| v.as_u64()) + .map(|n| n as usize); + + let start_line = c + .metadata + .get("start_line") + .and_then(|v| v.as_u64()) + .map(|n| n as u32); + + let end_line = c + .metadata + .get("end_line") + .and_then(|v| v.as_u64()) + .map(|n| n as u32); + + let heading = c.heading.map(|text| DetectedHeading { + text, + level: 1, + offset: 0, + page_number, + }); + + // All keys other than the well-known ones are stored in + // NodeMetadata.attributes so callers can pass arbitrary fields. + let known_keys = ["page_number", "start_line", "end_line", "section_type"]; + let attributes: HashMap = c + .metadata + .iter() + .filter(|(k, _)| !known_keys.contains(&k.as_str())) + .map(|(k, v)| (k.clone(), v.to_string())) + .collect(); + + let char_count = c.text.chars().count(); + let word_count = c.text.split_whitespace().count(); + + TextChunk { + id: format!("chunk_{}", i), + content: c.text, + heading, + char_count, + word_count, + start_page: page_number, + end_page: None, + start_line, + end_line, + attributes, + summary: c.summary, + } + }) + .collect() + } + /// Ingest URL and store in database /// /// The `table_id` must reference an existing table in the database. @@ -911,6 +1114,10 @@ impl IngestPipeline { word_count, start_page: None, end_page: None, + start_line: None, + end_line: None, + attributes: Default::default(), + summary: None, } }) .collect(), diff --git a/crates/reasondb-ingest/src/summarizer.rs b/crates/reasondb-ingest/src/summarizer.rs index b449223..a1682b5 100644 --- a/crates/reasondb-ingest/src/summarizer.rs +++ b/crates/reasondb-ingest/src/summarizer.rs @@ -88,9 +88,11 @@ impl<'a, R: ReasoningEngine> NodeSummarizer<'a, R> { depth ); - // Collect all inputs for this depth level (immutable reads) + // Collect all inputs for this depth level (immutable reads). + // Skip nodes that already carry a pre-supplied summary. let work_items: Vec<(usize, String, SummarizationContext)> = indices_at_depth .iter() + .filter(|&&idx| nodes[idx].summary.is_empty()) .map(|&idx| { let node = &nodes[idx]; let content = self.get_summarization_content(node, nodes, &node_map); @@ -304,7 +306,7 @@ impl<'a, R: ReasoningEngine> BatchSummarizer<'a, R> { let indices: Vec = nodes .iter() .enumerate() - .filter(|(_, n)| n.depth == depth) + .filter(|(_, n)| n.depth == depth && n.summary.is_empty()) .map(|(i, _)| i) .collect(); diff --git a/crates/reasondb-ingest/src/tree_builder.rs b/crates/reasondb-ingest/src/tree_builder.rs index 3986ea2..2246165 100644 --- a/crates/reasondb-ingest/src/tree_builder.rs +++ b/crates/reasondb-ingest/src/tree_builder.rs @@ -20,7 +20,12 @@ struct BuildNode { children: Vec, parent_id: Option, page_number: Option, + start_line: Option, + end_line: Option, + attributes: std::collections::HashMap, is_leaf: bool, + /// Pre-supplied summary from the caller; skips LLM generation when set. + summary: Option, } /// Builder for constructing hierarchical document trees @@ -109,7 +114,11 @@ impl TreeBuilder { children: Vec::new(), parent_id: None, page_number: None, + start_line: None, + end_line: None, + attributes: std::collections::HashMap::new(), is_leaf: false, + summary: None, }); // Process each chunk @@ -143,7 +152,11 @@ impl TreeBuilder { children: Vec::new(), parent_id: Some(parent_id.clone()), page_number: chunk.start_page.map(|p| p as u32), + start_line: chunk.start_line, + end_line: chunk.end_line, + attributes: chunk.attributes.clone(), is_leaf: true, // Will update if children are added + summary: chunk.summary.clone(), }; nodes.push(node); @@ -183,12 +196,13 @@ impl TreeBuilder { /// Convert BuildNode to PageNode fn to_page_node(&self, node: BuildNode, document_id: &str) -> PageNode { + let pre_supplied_summary = node.summary.clone(); let mut page_node = if node.is_leaf && !node.content.is_empty() { PageNode::new_leaf( document_id.to_string(), node.title.clone(), node.content.clone(), - String::new(), // Summary will be filled by summarizer + pre_supplied_summary.clone().unwrap_or_default(), node.depth, ) } else { @@ -207,9 +221,17 @@ impl TreeBuilder { page_node.start_index = 0; page_node.end_index = node.content.len(); + let mut metadata = NodeMetadata::default(); if let Some(page_num) = node.page_number { - page_node.metadata = NodeMetadata::default().with_page(page_num); + metadata = metadata.with_page(page_num); } + if let (Some(start), Some(end)) = (node.start_line, node.end_line) { + metadata = metadata.with_lines(start, end); + } + for (k, v) in node.attributes { + metadata = metadata.with_attribute(&k, &v); + } + page_node.metadata = metadata; page_node } @@ -271,6 +293,10 @@ impl TreeBuilder { word_count: char_count / 5, // Rough estimate start_page: heading.page_number, end_page: headings.get(i + 1).and_then(|h| h.page_number), + start_line: None, + end_line: None, + attributes: Default::default(), + summary: None, }); } } @@ -341,6 +367,10 @@ mod tests { word_count: 3, start_page: Some(1), end_page: Some(5), + start_line: None, + end_line: None, + attributes: Default::default(), + summary: None, }, TextChunk { id: "2".to_string(), @@ -355,6 +385,10 @@ mod tests { word_count: 2, start_page: Some(2), end_page: Some(3), + start_line: None, + end_line: None, + attributes: Default::default(), + summary: None, }, TextChunk { id: "3".to_string(), @@ -369,6 +403,10 @@ mod tests { word_count: 3, start_page: Some(10), end_page: Some(20), + start_line: None, + end_line: None, + attributes: Default::default(), + summary: None, }, ]; @@ -411,6 +449,10 @@ mod tests { word_count: 2, start_page: None, end_page: None, + start_line: None, + end_line: None, + attributes: Default::default(), + summary: None, }, TextChunk { id: "2".to_string(), @@ -425,6 +467,10 @@ mod tests { word_count: 2, start_page: None, end_page: None, + start_line: None, + end_line: None, + attributes: Default::default(), + summary: None, }, TextChunk { id: "3".to_string(), @@ -439,6 +485,10 @@ mod tests { word_count: 2, start_page: None, end_page: None, + start_line: None, + end_line: None, + attributes: Default::default(), + summary: None, }, ]; diff --git a/crates/reasondb-server/src/jobs.rs b/crates/reasondb-server/src/jobs.rs index 98b300f..95207d3 100644 --- a/crates/reasondb-server/src/jobs.rs +++ b/crates/reasondb-server/src/jobs.rs @@ -3,7 +3,9 @@ //! Jobs are persisted to redb so they survive server restarts. //! Clients poll `/v1/jobs/:id` for status updates. -use crate::routes::ingest::{IngestResponse, IngestTextRequest, IngestUrlRequest}; +use crate::routes::ingest::{ + IngestChunksRequest, IngestResponse, IngestTextRequest, IngestUrlRequest, +}; use crate::state::AppState; use chrono::{DateTime, Utc}; use reasondb_core::llm::ReasoningEngine; @@ -72,6 +74,7 @@ pub enum JobRequest { Text(IngestTextRequest), Url(IngestUrlRequest), File(IngestFileRequest), + Chunks(IngestChunksRequest), } impl JobRequest { @@ -80,6 +83,7 @@ impl JobRequest { JobRequest::Text(r) => &r.title, JobRequest::Url(r) => &r.url, JobRequest::File(r) => &r.filename, + JobRequest::Chunks(r) => &r.title, } } @@ -88,6 +92,7 @@ impl JobRequest { JobRequest::Text(r) => &r.table_id, JobRequest::Url(r) => &r.table_id, JobRequest::File(r) => &r.table_id, + JobRequest::Chunks(r) => &r.table_id, } } } @@ -454,12 +459,14 @@ async fn process_job( JobRequest::File(r) => r .generate_summaries .unwrap_or(state.config.generate_summaries), + JobRequest::Chunks(_) => true, }; let chunk_strategy_str = match &job.request { JobRequest::Text(r) => r.chunk_strategy.as_deref(), JobRequest::Url(r) => r.chunk_strategy.as_deref(), JobRequest::File(r) => r.chunk_strategy.as_deref(), + JobRequest::Chunks(_) => None, } .or_else(|| Some(state.config.chunk_strategy.as_str())); @@ -590,6 +597,48 @@ async fn process_job( result.document = doc; } + result + } + JobRequest::Chunks(req) => { + let chunk_inputs: Vec = req + .chunks + .iter() + .map(|c| reasondb_ingest::ChunkInput { + text: c.text.clone(), + heading: c.heading.clone(), + summary: c.summary.clone(), + metadata: c.metadata.clone().unwrap_or_default(), + }) + .collect(); + + let mut result = pipeline + .ingest_chunks_and_store( + &req.title, + &req.table_id, + chunk_inputs, + state.store.clone(), + ) + .await + .map_err(|e| e.to_string())?; + + let mut doc = result.document.clone(); + let mut needs_update = false; + if let Some(tags) = &req.tags { + doc.tags = tags.clone(); + needs_update = true; + } + if let Some(metadata) = &req.metadata { + doc.metadata = metadata.clone(); + needs_update = true; + } + if needs_update { + state + .store + .update_document(&doc) + .map_err(|e| e.to_string())?; + result.document = doc; + } + result } }; diff --git a/crates/reasondb-server/src/routes/ingest.rs b/crates/reasondb-server/src/routes/ingest.rs index db4930f..961dfaf 100644 --- a/crates/reasondb-server/src/routes/ingest.rs +++ b/crates/reasondb-server/src/routes/ingest.rs @@ -604,3 +604,127 @@ pub async fn ingest_url_for_table, + /// Optional pre-computed summary. If provided, the LLM summarization step + /// is skipped for this node. If absent, a summary is auto-generated. + #[serde(default)] + #[schema(example = "Introduces the core concepts of the topic.")] + pub summary: Option, + /// Free-form metadata — pass any key-value pairs. + /// Well-known keys: page_number (u32), start_line (u32), end_line (u32), section_type (string). + /// All other keys are stored in NodeMetadata.attributes. + #[serde(default)] + #[schema(example = json!({"page_number": 1, "start_line": 10, "end_line": 25, "source": "pdfplumber"}))] + pub metadata: Option>, +} + +/// Body for pre-chunked ingestion (table name comes from the URL path) +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] +pub struct IngestChunksBody { + /// Document title + #[schema(example = "My Research Paper")] + pub title: String, + /// Optional document description (stored in document metadata) + #[serde(default)] + #[schema(example = "A paper about machine learning techniques")] + pub description: Option, + /// Pre-split text chunks with their metadata + pub chunks: Vec, + /// Document tags for filtering + #[serde(default)] + #[schema(example = json!(["research", "ml"]))] + pub tags: Option>, + /// Document-level metadata (key-value pairs) + #[serde(default)] + #[schema(example = json!({"author": "Alice", "year": 2026}))] + pub metadata: Option>, +} + +/// Internal job-queue payload for pre-chunked ingestion. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IngestChunksRequest { + pub title: String, + pub table_id: String, + pub chunks: Vec, + #[serde(default)] + pub tags: Option>, + #[serde(default)] + pub metadata: Option>, +} + +/// Ingest pre-chunked content into a specific table (async — returns a job ID) +/// +/// Accepts an array of text chunks with optional per-chunk metadata. +/// Bypasses the extraction and chunking stages — content is fed directly +/// into the tree builder and summarizer. Per-chunk metadata (e.g. +/// `start_line`, `end_line`, `page_number`) is preserved on each node. +#[utoipa::path( + post, + path = "/v1/tables/{table_name}/ingest/chunks", + tag = "ingestion", + params(("table_name" = String, Path, description = "Table name or slug")), + request_body = IngestChunksBody, + responses( + (status = 202, description = "Ingestion job queued", body = JobStatusResponse), + (status = 404, description = "Table not found", body = ErrorResponse), + (status = 422, description = "Validation failed", body = ErrorResponse), + ) +)] +pub async fn ingest_chunks_for_table( + State(state): State>>, + Path(table_name): Path, + Json(body): Json, +) -> ApiResult> { + let table_id = state + .store + .resolve_table_id(&table_name) + .map_err(ApiError::from)?; + + state + .store + .get_table(&table_id) + .map_err(ApiError::from)? + .ok_or_else(|| ApiError::NotFound(format!("Table '{}' not found", table_name)))?; + + info!( + "Queuing chunks ingestion: '{}' ({} chunks) into table: {}", + body.title, + body.chunks.len(), + table_id + ); + + let request = IngestChunksRequest { + title: body.title, + table_id, + chunks: body.chunks, + tags: body.tags, + metadata: body.metadata, + }; + + let job_id = state + .job_queue + .enqueue(JobRequest::Chunks(request)) + .map_err(ApiError::Internal)?; + + let status = state + .job_queue + .get_status(&job_id) + .ok_or_else(|| ApiError::Internal("Job not found after enqueue".to_string()))?; + + Ok(Json(status)) +} diff --git a/crates/reasondb-server/src/routes/mod.rs b/crates/reasondb-server/src/routes/mod.rs index 20f2b42..31bc491 100644 --- a/crates/reasondb-server/src/routes/mod.rs +++ b/crates/reasondb-server/src/routes/mod.rs @@ -76,6 +76,10 @@ fn v1_routes( "/tables/:table_name/ingest/url", post(ingest::ingest_url_for_table::), ) + .route( + "/tables/:table_name/ingest/chunks", + post(ingest::ingest_chunks_for_table::), + ) // Jobs .route("/jobs", get(jobs::list_jobs::)) .route("/jobs/:id", get(jobs::get_job::)) diff --git a/docs/guides/ingestion.mdx b/docs/guides/ingestion.mdx index 6377fba..7c69f53 100644 --- a/docs/guides/ingestion.mdx +++ b/docs/guides/ingestion.mdx @@ -15,6 +15,7 @@ All file and URL extraction is handled by the [plugin system](/guides/plugins). Text[Text/Markdown] File[PDF/Word/Excel] URL[Web Pages/YouTube] + Chunks[Pre-chunked JSON] end subgraph Pipeline @@ -33,6 +34,7 @@ All file and URL extraction is handled by the [plugin system](/guides/plugins). Text --> Extract File --> Extract URL --> Extract + Chunks -->|bypasses extract + chunk| Tree Extract --> Post --> Chunk --> Tree --> Summary --> DB Summary --> Index`} @@ -231,6 +233,114 @@ curl -X POST http://localhost:4444/v1/tables/videos/ingest/url \ Poll the returned `job_id` for completion. The video transcript will be extracted and organized into a searchable tree. +## Ingest Pre-chunked Content + +If you have already split your document into chunks — for example using `pdfplumber`, a custom NLP pipeline, or any other tool — you can ingest them directly via the `/ingest/chunks` endpoint. This **bypasses the extraction and chunking stages entirely** and feeds your content straight into the tree builder and LLM summarizer. + +This is useful when: +- You need precise per-chunk metadata (exact page numbers, source line ranges, bounding boxes) +- You are ingesting output from an existing chunking pipeline +- You want full control over how your content is split + +```bash +curl -X POST http://localhost:4444/v1/tables/my_documents/ingest/chunks \ + -H "Content-Type: application/json" \ + -d '{ + "title": "Research Paper", + "description": "Extracted with pdfplumber", + "chunks": [ + { + "text": "The mitochondria is the powerhouse of the cell...", + "heading": "Biology Basics", + "metadata": { + "page_number": 1, + "start_line": 1, + "end_line": 12, + "source": "pdfplumber" + } + }, + { + "text": "Photosynthesis converts sunlight into glucose...", + "heading": "Plant Biology", + "metadata": { + "page_number": 2, + "start_line": 13, + "end_line": 28 + } + } + ], + "tags": ["biology", "research"], + "metadata": {"author": "Alice", "year": 2026} + }' +``` + +Response (HTTP 202) — same job polling pattern as other endpoints: + +```json +{ + "job_id": "job_abc123", + "status": "queued", + "created_at": "2026-01-30T10:00:00Z", + "updated_at": "2026-01-30T10:00:00Z" +} +``` + +LLM summaries and cross-references are always generated automatically for pre-chunked content. + +### Chunk parameters + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `text` | Yes | The text content of this chunk | +| `heading` | No | Heading / title for the chunk — used as the node title in the tree | +| `summary` | No | Pre-computed summary. If provided, the LLM summarization step is skipped for this node | +| `metadata` | No | Free-form JSON object. Well-known keys are promoted to typed fields; all others are stored in `NodeMetadata.attributes` | + +### Well-known metadata keys + +| Key | Type | Description | +|-----|------|-------------| +| `page_number` | integer | Page number in the source document | +| `start_line` | integer | Line number where this chunk starts in the source file | +| `end_line` | integer | Line number where this chunk ends | +| `section_type` | string | Section type (e.g. `"chapter"`, `"paragraph"`, `"table"`) | + +Any other keys you pass in `metadata` are stored as custom attributes on the node and are available for filtering via RQL. + +### Providing pre-computed summaries + +If your pipeline has already generated summaries (e.g. via a separate LLM call), pass them in the `summary` field to skip LLM generation for that chunk: + +```bash +curl -X POST http://localhost:4444/v1/tables/my_documents/ingest/chunks \ + -H "Content-Type: application/json" \ + -d '{ + "title": "My Doc", + "chunks": [ + { + "text": "Full text of the section...", + "heading": "Introduction", + "summary": "This section introduces the core concepts.", + "metadata": {"page_number": 1} + } + ] + }' +``` + + +Mix pre-computed and auto-generated summaries freely. Chunks with a `summary` field use it directly; chunks without one get an LLM-generated summary automatically. + + +### Body parameters + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `title` | Yes | Document title | +| `description` | No | Optional document description | +| `chunks` | Yes | Array of chunk objects (see above) | +| `tags` | No | Array of tags for filtering | +| `metadata` | No | Document-level metadata key-value pairs | + ## Metadata and Tags Add metadata and tags during ingestion for filtering: diff --git a/tutorials/01-rql-basics/public/logo.svg b/tutorials/01-rql-basics/public/logo.svg index 0c1c687..3c3a0c4 100644 --- a/tutorials/01-rql-basics/public/logo.svg +++ b/tutorials/01-rql-basics/public/logo.svg @@ -1,7 +1,4 @@ - - - - - - + + + diff --git a/tutorials/02-legal-search/public/logo.svg b/tutorials/02-legal-search/public/logo.svg index 0c1c687..3c3a0c4 100644 --- a/tutorials/02-legal-search/public/logo.svg +++ b/tutorials/02-legal-search/public/logo.svg @@ -1,7 +1,4 @@ - - - - - - + + + diff --git a/tutorials/03-research-papers/public/logo.svg b/tutorials/03-research-papers/public/logo.svg index 0c1c687..3c3a0c4 100644 --- a/tutorials/03-research-papers/public/logo.svg +++ b/tutorials/03-research-papers/public/logo.svg @@ -1,7 +1,4 @@ - - - - - - + + + diff --git a/tutorials/04-knowledge-base/public/logo.svg b/tutorials/04-knowledge-base/public/logo.svg index 0c1c687..3c3a0c4 100644 --- a/tutorials/04-knowledge-base/public/logo.svg +++ b/tutorials/04-knowledge-base/public/logo.svg @@ -1,7 +1,4 @@ - - - - - - + + + diff --git a/tutorials/05-pdf-financials/public/logo.svg b/tutorials/05-pdf-financials/public/logo.svg index 0c1c687..3c3a0c4 100644 --- a/tutorials/05-pdf-financials/public/logo.svg +++ b/tutorials/05-pdf-financials/public/logo.svg @@ -1,7 +1,4 @@ - - - - - - + + + diff --git a/tutorials/06-insurance/public/logo.svg b/tutorials/06-insurance/public/logo.svg index 0c1c687..3c3a0c4 100644 --- a/tutorials/06-insurance/public/logo.svg +++ b/tutorials/06-insurance/public/logo.svg @@ -1,7 +1,4 @@ - - - - - - + + +