From cdcb216ff4f0084818a2b61d1c075900c16d0152 Mon Sep 17 00:00:00 2001 From: OKAN TIKTAS <346542017+OKAN-SOFTWARE@users.noreply.github.com> Date: Fri, 6 Nov 2026 15:46:46 -0500 Subject: [PATCH] fix: preserve PDF words and generate FAQs from Word section headings --- docs/WEB-DEMO.md | 2 +- src/test/js/browser-documents.test.cjs | 21 +++++++++++++++++++ web-demo/documents.js | 28 +++++++++++++++++++------- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/docs/WEB-DEMO.md b/docs/WEB-DEMO.md index c268b34..57d8522 100644 --- a/docs/WEB-DEMO.md +++ b/docs/WEB-DEMO.md @@ -16,7 +16,7 @@ The public-facing route is `https://okantiktas.com/projects/doc2faq/`. The portf | PDF/DOCX import and export | Yes; browser PDF.js / Word XML extraction and PDF-LIB / docx exports | Yes; PDFBox / Apache POI | | OCR, external AI, accounts/billing | No | Not implemented | -PDF and Word files are processed on the visitor's device. PDF import is limited to 5 MB, 50 pages and 50,000 extracted characters; DOCX import is limited to 5 MB, 50,000 extracted characters, 1,000 archive entries and an 8 MB expanded main XML part. Legacy `.doc`, password-protected PDFs and image-only/scanned PDFs are not supported. Word extraction reads main-document paragraphs and tables, preserving standard Heading 1–6 styles as Markdown headings. Headers/footers, images and complex layouts may not be retained. Browser PDF export uses Helvetica with Latin character coverage; unsupported characters produce an error recommending Word, never silently dropped text. Word exports preserve Unicode. Both exports include every FAQ regardless of filters and optionally include source references. +PDF and Word files are processed on the visitor's device. PDF import is limited to 5 MB, 50 pages and 50,000 extracted characters; DOCX import is limited to 5 MB, 50,000 extracted characters, 1,000 archive entries and an 8 MB expanded main XML part. Legacy `.doc`, password-protected PDFs and image-only/scanned PDFs are not supported. Word extraction reads main-document paragraphs and tables, mapping standard Heading 1–6 styles to Markdown section headings (levels 2–6) so the generator treats Heading 1 as a section rather than a document title. Headers/footers, images and complex layouts may not be retained. Browser PDF export uses Helvetica with Latin character coverage; unsupported characters produce an error recommending Word, never silently dropped text. Word exports preserve Unicode. Both exports include every FAQ regardless of filters and optionally include source references. The demo demonstrates the review workflow; it is not a hosted Java backend and does not claim identical generation results. Its development features may be newer than the latest published JAR. No pasted text is sent to an FAQ server. Optional autosave and downloaded project files contain source text; clear saved data on shared devices. Normal website hosting still serves the page and assets. The document adapter lazily loads version-pinned PDF.js, PDF-LIB, JSZip and docx code from jsDelivr. An internet connection is needed on first use; load failures leave the workspace available for retry. See [third-party libraries](../web-demo/THIRD-PARTY.md). diff --git a/src/test/js/browser-documents.test.cjs b/src/test/js/browser-documents.test.cjs index a0eb6b9..ef071fe 100644 --- a/src/test/js/browser-documents.test.cjs +++ b/src/test/js/browser-documents.test.cjs @@ -54,3 +54,24 @@ test('PDF line wrapping preserves blank lines and splits long tokens within the assert.ok(lines.every(line=>line.length<=20)); assert.ok(lines.includes('')); assert.equal(lines.slice(0,3).join(''),'a'.repeat(51)); }); + +test('Word Heading 1 becomes a section the browser generator can draft from', () => { + const {draft} = require('../../../web-demo/engine.js'); + const text = tools.headingPrefix('Heading1') + 'Services\nOnboarding guides.\n' + tools.headingPrefix('Heading2') + 'Support\nAsk the team.'; + const result = draft(text, 'generate'); + assert.equal(result.count,2); + assert.equal(result.items[0].answer,'Onboarding guides.'); + assert.equal(tools.headingPrefix('Title'),''); + assert.equal(tools.headingPrefix('Heading6'),'###### '); +}); + +test('contiguous PDF fragments keep words intact while positioned word gaps get spaces', async () => { + const fragments = [ + {str:'Doc',width:18,transform:[12,0,0,12,50,100]}, + {str:'2FAQ',width:28,transform:[12,0,0,12,68,100]}, + {str:'works',width:30,transform:[12,0,0,12,102,100]}, + {str:' well',width:25,transform:[12,0,0,12,132,100],hasEOL:true} + ]; + const pdfjs={getDocument(){return {promise:Promise.resolve({numPages:1,getPage:async()=>({getTextContent:async()=>({items:fragments}),cleanup(){}})}),destroy:async()=>{}};}}; + assert.equal(await tools.extractPdf(new TextEncoder().encode('%PDF-1.7'),pdfjs),'Doc2FAQ works well'); +}); diff --git a/web-demo/documents.js b/web-demo/documents.js index f47b95d..16b6b13 100644 --- a/web-demo/documents.js +++ b/web-demo/documents.js @@ -41,14 +41,25 @@ let text = ''; for (let i = 1; i <= doc.numPages; i++) { const page = await doc.getPage(i), content = await page.getTextContent(); - let line = '', pageText = '', lastY = null; + let line = '', pageText = '', lastY = null, previous = null; for (const item of content.items) { if (typeof item.str !== 'string') continue; const y = item.transform?.[5]; - if (line && lastY !== null && Number.isFinite(y) && Math.abs(y - lastY) > 3) { pageText += line.trim() + '\n'; line = ''; } - line += (line && !line.endsWith(' ') ? ' ' : '') + item.str; + if (line && lastY !== null && Number.isFinite(y) && Math.abs(y - lastY) > 3) { pageText += line.trim() + '\n'; line = ''; previous = null; } + let separator = ''; + if (line && !/\s$/.test(line) && !/^\s/.test(item.str) && previous) { + const x = item.transform?.[4], beforeX = previous.transform?.[4]; + if (Number.isFinite(x) && Number.isFinite(beforeX) && Number.isFinite(previous.width) && Number.isFinite(item.width)) { + const gap = item.dir === 'rtl' && previous.dir === 'rtl' + ? beforeX - (x + item.width) : x - (beforeX + previous.width); + const size = Math.hypot(item.transform?.[2] || 0, item.transform?.[3] || 0) || 10; + if (gap > Math.max(0.5, size * 0.12)) separator = ' '; + } + } + line += separator + item.str; + previous = item; lastY = y; - if (item.hasEOL) { pageText += line.trim() + '\n'; line = ''; lastY = null; } + if (item.hasEOL) { pageText += line.trim() + '\n'; line = ''; lastY = null; previous = null; } if (text.length + pageText.length + line.length > MAX_TEXT) throw new Error('Extracted text exceeds 50,000 characters.'); } text += pageText + line.trim() + (i < doc.numPages ? '\n\n' : ''); @@ -95,6 +106,10 @@ stream.resume(); }); } + function headingPrefix(style) { + const level = /^Heading([1-6])$/i.exec(style); + return level ? '#'.repeat(Math.min(Number(level[1]) + 1, 6)) + ' ' : ''; + } function wordXmlText(xml, Parser = root.DOMParser) { if (/ MAX_TEXT + 1) throw new Error('Extracted text exceeds 50,000 characters.'); lines.push(text); @@ -231,7 +245,7 @@ throw new Error('Unsupported document operation.'); } catch (error) { return {ok: false, json: async () => ({message: error.message || 'Could not process this document.'})}; } } - const api = {request, validateInput, checkedText, validateZip, boundedEntry, wordXmlText, extractDocx, extractPdf, validateExport, references, wrap, exportPdf, exportDocx, libraries}; + const api = {request, validateInput, checkedText, validateZip, boundedEntry, headingPrefix, wordXmlText, extractDocx, extractPdf, validateExport, references, wrap, exportPdf, exportDocx, libraries}; if (typeof module !== 'undefined' && module.exports) module.exports = api; else root.Doc2FaqDocuments = api; })(globalThis);