diff --git a/.env.sample b/.env.sample index 035f9b30f..31b50eb1f 100644 --- a/.env.sample +++ b/.env.sample @@ -5,7 +5,7 @@ # Use DEFAULT for the meta portal NUXT_PUBLIC_APP_API_ORG_CODE=CRI -# Custom favicon +# Custom favicon # (can be an absolute path, else will be relative to NUXT_PUBLIC_APP_PUBLIC_BINARIES_PREFIX) NUXT_PUBLIC_APP_FAVICON=favicon.ico # url prefix for a binary file storage (images...) @@ -77,7 +77,7 @@ PLAYWRIGHT_TEST_PASSWORD= ### examples suggestions separated by § character # NUXT_PUBLIC_APP_CHATBOT_EXEMPLES=Tell me about this platform§Find research projects on renewable energy§Find researchers and publication on climate change -# ==== OpenAI PRIVATE settings ==== +# ==== OpenAI PRIVATE settings ==== # OpenAI API Key for the chatbot # optional # NUXT_APP_OPENAI_API_KEY=sk-proj-XXXXXXXXXXXXXXXXXXXXXXXXXXXXX @@ -96,4 +96,12 @@ PLAYWRIGHT_TEST_PASSWORD= # ==== GOTENBERG pdf (optional) Settings ==== # NUXT_APP_GOTENBERG_SERVER_URL=http://127.0.0.1:3333 -# NUXT_PUBLIC_APP_GOTENBERG_ENABLED='true' \ No newline at end of file +# NUXT_PUBLIC_APP_GOTENBERG_ENABLED='true' + +# ==== VECTOR DATABASE (optional) Settings ==== +# NUXT_APP_VECTOR_DB_URL=postgresql://myuser:mypassword@localhost:5432/mydb +# NUXT_APP_VECTOR_EMBEDDING_MODEL=text-embedding-3-small +# NUXT_APP_VECTOR_EMBEDDING_API_KEY=XXXXX +# NUXT_APP_VECTOR_EMBEDDING_DIMENSIONS=1536 +# NUXT_APP_VECTOR_TOOL_PROMPT="use this tool to answer about neuroscience project" +# NUXT_APP_VECTOR_TABLE_NAME=projects-vector diff --git a/.zed/debug.json b/.zed/debug.json new file mode 100644 index 000000000..c2327420e --- /dev/null +++ b/.zed/debug.json @@ -0,0 +1,9 @@ +[ + { + "label": "Attach to Nuxt", + "adapter": "JavaScript", + "request": "attach", + "port": 9229, + "sourceMaps": true + } +] diff --git a/k8s/projects-frontend/values.dev.yaml b/k8s/projects-frontend/values.dev.yaml index 62f6d3376..5d86ab1ff 100644 --- a/k8s/projects-frontend/values.dev.yaml +++ b/k8s/projects-frontend/values.dev.yaml @@ -1,11 +1,17 @@ config: nonSensitive: NUXT_PUBLIC_APP_KEYCLOAK_URL: https://keycloak.k8s.lp-i.dev - NUXT_PUBLIC_APP_VERSION: "{{ .Values.application.revision }}" + NUXT_PUBLIC_APP_VERSION: '{{ .Values.application.revision }}' NUXT_PUBLIC_APP_ENVIRONMENT: dev NUXT_APP_GOTENBERG_SERVER_URL: 'http://projects-gotenberg' NUXT_PUBLIC_APP_GOTENBERG_ENABLED: 'true' NUXT_APP_LANGCHAIN_PROMPT: '' + NUXT_APP_VECTOR_DB_URL: '' + NUXT_APP_VECTOR_EMBEDDING_API_KEY: '' + NUXT_APP_VECTOR_EMBEDDING_MODEL: '' + NUXT_APP_VECTOR_EMBEDDING_DIMENSIONS: '' + NUXT_APP_VECTOR_TOOL_PROMPT: '' + NUXT_APP_VECTOR_TABLE_NAME: '' domain: k8s.lp-i.dev diff --git a/k8s/projects-frontend/values.yaml b/k8s/projects-frontend/values.yaml index bc74cdd53..652b891bd 100644 --- a/k8s/projects-frontend/values.yaml +++ b/k8s/projects-frontend/values.yaml @@ -45,6 +45,12 @@ config: NUXT_APP_GOTENBERG_SERVER_URL: '' NUXT_PUBLIC_APP_GOTENBERG_ENABLED: '' NUXT_APP_LANGCHAIN_PROMPT: '' + NUXT_APP_VECTOR_DB_URL: '' + NUXT_APP_VECTOR_EMBEDDING_API_KEY: '' + NUXT_APP_VECTOR_EMBEDDING_MODEL: '' + NUXT_APP_VECTOR_EMBEDDING_DIMENSIONS: '' + NUXT_APP_VECTOR_TOOL_PROMPT: '' + NUXT_APP_VECTOR_TABLE_NAME: '' e2eEnv: nonSensitive: USER_ADMIN_EMAIL: testautomatatiquedministrateur1@outlook.fr diff --git a/nuxt.config.ts b/nuxt.config.ts index 41b1dfd68..0e1a646a3 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -131,7 +131,7 @@ export default defineNuxtConfig({ appGeocodingApiKey: '', appOpenaiApiKey: '', appOpenaiApiPromptId: '', - appOpenaiApiPromptVersion: '', + appOpeyanaiApiPromptVersion: '', appOpenaiApiVectorStoreId: '', appMcpServerUrl: '', appSorbobotApiToken: '', @@ -139,6 +139,12 @@ export default defineNuxtConfig({ appSorbobotApiTrace: 0, appGotenbergServerUrl: '', appLangchainPrompt: '', + appVectorDbUrl: '', + appVectorEmbeddingApiKey: '', + appVectorTableName: '', + appVectorEmbeddingModel: '', + appVectorEmbeddingDimensions: '', + appVectorToolPrompt: '', public: { appVersion: '', appApiOrgCode: '', diff --git a/package.json b/package.json index 729f6e4dc..d904c37be 100644 --- a/package.json +++ b/package.json @@ -42,9 +42,11 @@ "@ckpack/vue-color": "^1.5.0", "@hocuspocus/provider": "^2.15.0", "@intlify/vue-i18n-loader": "^4.2.0", - "@langchain/core": "^1.1.17", + "@langchain/community": "^1.1.23", + "@langchain/core": "^1.1.32", "@langchain/langgraph": "^1.1.2", "@langchain/openai": "^1.2.3", + "@langchain/textsplitters": "^1.0.1", "@mdi/font": "^6", "@modelcontextprotocol/sdk": "^1.20.2", "@nuxt/test-utils": "^3.19.2", @@ -84,6 +86,7 @@ "deep-chat": "^2.3.0", "es-toolkit": "^1.40.0", "highlight.js": "^11", + "install": "^0.13.0", "langchain": "^1.2.14", "leaflet": "^1", "leaflet.markercluster": "^1.5.3", @@ -92,6 +95,8 @@ "nuxt": "^3.19.3", "nuxt-svgo": "4.2.6", "openai": "^5.23.0", + "pdf-parse": "^1", + "pg": "^8.20.0", "pinia": "^3.0.3", "prosemirror-markdown": "^1.13.1", "prosemirror-model": "^1.24.1", diff --git a/src/app/useAdminPagesRoutes.ts b/src/app/useAdminPagesRoutes.ts index d5a963d19..52a49fd57 100644 --- a/src/app/useAdminPagesRoutes.ts +++ b/src/app/useAdminPagesRoutes.ts @@ -130,6 +130,12 @@ export default function useAdminPagesRoutes() { component: () => import('../pages/AdminPortalPageV2/Tabs/TermsAdminTab.vue'), meta: { checkAccessRequestEnabled: true }, }, + { + path: 'vector-store', + name: 'VectorStoreAdminTab', + component: () => import('../pages/AdminPortalPageV2/Tabs/VectorStoreAdminTab.vue'), + meta: { checkAccessRequestEnabled: true }, + }, ], meta: { resetScroll: true, diff --git a/src/components/VectorStoreDocumentList.vue b/src/components/VectorStoreDocumentList.vue new file mode 100644 index 000000000..565d53079 --- /dev/null +++ b/src/components/VectorStoreDocumentList.vue @@ -0,0 +1,130 @@ + + + diff --git a/src/components/VectorStoreDocumentShow.vue b/src/components/VectorStoreDocumentShow.vue new file mode 100644 index 000000000..db7d3d02c --- /dev/null +++ b/src/components/VectorStoreDocumentShow.vue @@ -0,0 +1,72 @@ + + + diff --git a/src/components/VectorStoreIngestionForm.vue b/src/components/VectorStoreIngestionForm.vue new file mode 100644 index 000000000..f62189a13 --- /dev/null +++ b/src/components/VectorStoreIngestionForm.vue @@ -0,0 +1,130 @@ + + + diff --git a/src/components/people/UserProfile/UserProjectsSearch.vue b/src/components/people/UserProfile/UserProjectsSearch.vue index d892a14c4..aac71bd9d 100644 --- a/src/components/people/UserProfile/UserProjectsSearch.vue +++ b/src/components/people/UserProfile/UserProjectsSearch.vue @@ -119,6 +119,7 @@ export default { }, updateProjectList(response) { + if (!response) return this.updatePagination(response) // Set new projects and end loading const maxResults = response.max_results || this.limit @@ -147,6 +148,7 @@ export default { }, updatePagination(response) { + if (!response) return this.pagination.currentPage = response.current_page const maxResults = response.max_results || 12 this.pagination.total = response.total_page || Math.ceil(response.count / maxResults) diff --git a/src/i18n/locales/ca.json b/src/i18n/locales/ca.json index 518d273e3..1d2e56b6a 100644 --- a/src/i18n/locales/ca.json +++ b/src/i18n/locales/ca.json @@ -220,7 +220,8 @@ "requests": "Peticions", "groups": "Grups", "settings": "Arranjament", - "terms": "ToS" + "terms": "ToS", + "vector-store": "Vector store" }, "page-title": "Administrador", "portal": { @@ -3144,5 +3145,20 @@ "Presentation": "Presentació", "UNKNOWN": "Desconegut" } + }, + "vector-store": { + "confirm-deletion": "Confirmar supressió", + "confirm-deletion-of": "Voleu realment suprimir \"{title}\"?", + "document-deleted": "Document suprimit.", + "add-document": "Afegir un document", + "chunks": "fragments", + "no-document-yet": "Encara no hi ha cap document", + "title-exists": "El títol ja existeix", + "add-document": "Afegir un document", + "edit-document": "Editar un document", + "title-field": "Títol", + "file-field": "Fitxer", + "document-added": "Document afegit.", + "document-updated": "Document actualitzat." } } diff --git a/src/i18n/locales/de.json b/src/i18n/locales/de.json index 38657d45b..56e9b2863 100644 --- a/src/i18n/locales/de.json +++ b/src/i18n/locales/de.json @@ -220,7 +220,8 @@ "requests": "Anträge", "groups": "Gruppen", "settings": "Einstellungen", - "terms": "ToS" + "terms": "ToS", + "vector-store": "Vector store" }, "page-title": "Admin", "portal": { @@ -3144,5 +3145,20 @@ "Presentation": "Präsentation", "UNKNOWN": "Unbekannt" } + }, + "vector-store": { + "confirm-deletion": "Löschung bestätigen", + "confirm-deletion-of": "Möchten Sie \"{title}\" wirklich löschen?", + "document-deleted": "Dokument gelöscht.", + "add-document": "Ein Dokument hinzufügen", + "chunks": "Fragmente", + "no-document-yet": "Noch kein Dokument", + "title-exists": "Titel existiert bereits", + "add-document": "Ein Dokument hinzufügen", + "edit-document": "Ein Dokument bearbeiten", + "title-field": "Titel", + "file-field": "Datei", + "document-added": "Dokument hinzugefügt.", + "document-updated": "Dokument bearbeitet." } } diff --git a/src/i18n/locales/en.json b/src/i18n/locales/en.json index 52f080cd3..b545b68ae 100644 --- a/src/i18n/locales/en.json +++ b/src/i18n/locales/en.json @@ -227,7 +227,8 @@ "requests": "Requests", "groups": "Groups", "settings": "Settings", - "terms": "ToS" + "terms": "ToS", + "vector-store": "Vector store" }, "page-title": "Admin", "portal": { @@ -3144,5 +3145,20 @@ "error-delete": "An error occurred by deleting the image.", "success-create": "Photo created.", "error-create": "An error occurred created image." + }, + "vector-store": { + "confirm-deletion": "Confirm deletion", + "confirm-deletion-of": "Do you really want to delete \"{title}\"?", + "document-deleted": "Document deleted.", + "add-document": "Add a document", + "chunks": "chunks", + "no-document-yet": "No document yet", + "title-exists": "Title already exists", + "add-document": "Add a document", + "edit-document": "Edit a document", + "title-field": "Title", + "file-field": "File", + "document-added": "Document added.", + "document-updated": "Document edited." } } diff --git a/src/i18n/locales/es.json b/src/i18n/locales/es.json index e2f8e5899..47ac5fc06 100644 --- a/src/i18n/locales/es.json +++ b/src/i18n/locales/es.json @@ -220,7 +220,8 @@ "requests": "Solicitudes", "groups": "Grupos", "settings": "Ajustes", - "terms": "ToS" + "terms": "ToS", + "vector-store": "Vector store" }, "page-title": "Admin", "portal": { @@ -3144,5 +3145,21 @@ "Presentation": "Presentación", "UNKNOWN": "Desconocida" } + }, + + "vector-store": { + "confirm-deletion": "Confirmar eliminación", + "confirm-deletion-of": "¿Realmente desea eliminar \"{title}\"?", + "document-deleted": "Documento eliminado.", + "add-document": "Agregar un documento", + "chunks": "fragmentos", + "no-document-yet": "Aún no hay documentos", + "title-exists": "El título ya existe", + "add-document": "Agregar un documento", + "edit-document": "Editar un documento", + "title-field": "Título", + "file-field": "Archivo", + "document-added": "Documento agregado.", + "document-updated": "Documento actualizado." } } diff --git a/src/i18n/locales/et.json b/src/i18n/locales/et.json index 5719e8927..30cd2fc18 100644 --- a/src/i18n/locales/et.json +++ b/src/i18n/locales/et.json @@ -220,7 +220,8 @@ "requests": "Taotlused", "groups": "Grupid", "settings": "Seadistused", - "terms": "Tos" + "terms": "Tos", + "vector-store": "Vector store" }, "page-title": "Admin", "portal": { @@ -3144,5 +3145,20 @@ "Presentation": "Esitlus", "UNKNOWN": "Tundmatu" } + }, + "vector-store": { + "confirm-deletion": "Kinnita kustutamine", + "confirm-deletion-of": "Kas soovite tõesti \"{title}\" kustutada?", + "document-deleted": "Dokument kustutatud.", + "add-document": "Lisa dokument", + "chunks": "fragmendid", + "no-document-yet": "Veel ühtegi dokumenti", + "title-exists": "Pealkiri eksisteerib juba", + "add-document": "Lisa dokument", + "edit-document": "Muuda dokumenti", + "title-field": "Pealkiri", + "file-field": "Fail", + "document-added": "Dokument lisatud.", + "document-updated": "Dokument uuendatud." } } diff --git a/src/i18n/locales/fr.json b/src/i18n/locales/fr.json index df9d04e94..ba59d9257 100644 --- a/src/i18n/locales/fr.json +++ b/src/i18n/locales/fr.json @@ -227,7 +227,8 @@ "requests": "Demandes", "groups": "Groupes", "settings": "Paramètres", - "terms": "CGU" + "terms": "CGU", + "vector-store": "Vector store" }, "page-title": "Admin", "portal": { @@ -3144,5 +3145,20 @@ "Presentation": "Présentation", "UNKNOWN": "Inconnu" } + }, + "vector-store": { + "confirm-deletion": "Confirmer la suppression", + "confirm-deletion-of": "Voulez-vous vraiment supprimer « {title} » ?", + "document-deleted": "Document supprimé.", + "add-document": "Ajouter un document", + "chunks": "segments", + "no-document-yet": "Aucun document pour le moment", + "title-exists": "Le titre existe déjà", + "add-document": "Ajouter un document", + "edit-document": "Modifier un document", + "title-field": "Titre", + "file-field": "Fichier", + "document-added": "Document ajouté.", + "document-updated": "Document modifié." } } diff --git a/src/i18n/locales/nl.json b/src/i18n/locales/nl.json index 6cf608d54..5f36d0b58 100644 --- a/src/i18n/locales/nl.json +++ b/src/i18n/locales/nl.json @@ -220,7 +220,8 @@ "requests": "Verzoeken", "groups": "Groepen", "settings": "Instellingen", - "terms": "TOS" + "terms": "TOS", + "vector-store": "Vector store" }, "page-title": "Beheerder", "portal": { @@ -3144,5 +3145,20 @@ "Presentation": "Presentatie", "UNKNOWN": "Onbekend" } + }, + "vector-store": { + "confirm-deletion": "Verwijdering bevestigen", + "confirm-deletion-of": "Wilt u \"{title}\" echt verwijderen?", + "document-deleted": "Document verwijderd.", + "add-document": "Een document toevoegen", + "chunks": "fragmenten", + "no-document-yet": "Nog geen document", + "title-exists": "Titel bestaat al", + "add-document": "Een document toevoegen", + "edit-document": "Een document bewerken", + "title-field": "Titel", + "file-field": "Bestand", + "document-added": "Document toegevoegd.", + "document-updated": "Document bijgewerkt." } } diff --git a/src/pages/AdminPortalPageV2/AdminPortalPage.vue b/src/pages/AdminPortalPageV2/AdminPortalPage.vue index a64f43916..0e56f2eaa 100644 --- a/src/pages/AdminPortalPageV2/AdminPortalPage.vue +++ b/src/pages/AdminPortalPageV2/AdminPortalPage.vue @@ -1,8 +1,10 @@ + + diff --git a/src/server/routes/api/chat-lg-stream.ts b/src/server/routes/api/chat-lg-stream.ts index cf2f80b5c..b292d5c6b 100644 --- a/src/server/routes/api/chat-lg-stream.ts +++ b/src/server/routes/api/chat-lg-stream.ts @@ -1,19 +1,24 @@ // import OpenAI from 'openai' import { ChatOpenAI } from '@langchain/openai' -import { createAgent } from 'langchain' +import { createAgent, createMiddleware } from 'langchain' import { MemorySaver } from '@langchain/langgraph' -import { SystemMessage, HumanMessage } from '@langchain/core/messages' +import { SystemMessage, HumanMessage, AIMessage, BaseMessageChunk } from '@langchain/core/messages' import { v4 as uuidv4 } from 'uuid' +import getVectorStore from '@/server/utils/vector-db.js' +// import { tool } from '@langchain/core/tools' +import { createRetrieverTool } from '@langchain/classic/tools/retriever' + const runtimeConfig = useRuntimeConfig() const { // appOpenaiApiPromptId, // appOpenaiApiPromptVersion, appOpenaiApiKey, - appOpenaiApiVectorStoreId, + // appOpenaiApiVectorStoreId, appMcpServerUrl, appMcpServerTrace, appSorbobotApiTrace, appLangchainPrompt, + appVectorToolPrompt, } = runtimeConfig const { appChatbotEnabled } = runtimeConfig.public @@ -96,17 +101,94 @@ export default defineLazyEventHandler(() => { }) } - if (appOpenaiApiVectorStoreId) { - tools.push({ - type: 'file_search', - vector_store_ids: [appOpenaiApiVectorStoreId], + // if (appOpenaiApiVectorStoreId) { + // tools.push({ + // // name: 'openai_vectorstore', + // type: 'file_search', + // vector_store_ids: [appOpenaiApiVectorStoreId], + // }) + // } + + const { vectorStore } = await getVectorStore() + if (vectorStore) { + const { appApiOrgCode } = useRuntimeConfig().public + console.log(`Configure vector tool with prompt "${appVectorToolPrompt}"`) + const retriever = vectorStore.asRetriever({ + k: 5, + filter: { + orgCode: appApiOrgCode, + }, + }) // Top 4 similar docs + // Create tool from retriever + // const retrieverTool = tool( + // async (query) => { + // console.log(query) + // const docs = await retriever.invoke(query) + // return docs.map((doc: Document) => doc.pageContent).join('\n\n') + // }, + // { + // name: 'pgvector_search', + // type: 'file_search', + // // description: 'Search for information in the document database. Use this tool when you need to answer questions about the uploaded content.', + // description: appVectorToolPrompt, + // schema: { + // type: 'object', + // properties: { + // query: { + // type: 'string', + // description: 'Search query', + // }, + // }, + // required: ['query'], + // }, + // require_approval: 'never', + // } + // ) + const retrieverTool = createRetrieverTool(retriever, { + name: 'pgvector_tool', + description: appVectorToolPrompt, }) + tools.push(retrieverTool) } + + const toolMonitoringMiddleware = createMiddleware({ + name: 'ToolMonitoringMiddleware', + wrapToolCall: async (request, handler) => { + // console.log(`Executing tool: ${request.toolCall.name}`) + // console.log(`Arguments: ${JSON.stringify(request.toolCall.args)}`) + try { + const result = await handler(request) + const content: string = (result as { content: string }).content + console.log( + `Tool completed successfully: "${content.substring(0, 100)}${content.length > 100 ? ' (...)' : ''}"` + ) + return result + } catch (e) { + console.log(`Tool failed: ${e}`) + throw e + } + }, + }) + + // const loggingMiddleware = createMiddleware({ + // name: 'LoggingMiddleware', + // beforeModel: (state) => { + // console.log(`About to call model with ${state.messages.length} messages`) + // console.log(JSON.stringify(state.messages, null, 2)) + // return + // }, + // afterModel: (state) => { + // const lastMessage = state.messages[state.messages.length - 1] + // console.log(`Model returned: ${lastMessage.content}`) + // return + // }, + // }) + const model = appOpenaiApiKey ? new ChatOpenAI({ apiKey: appOpenaiApiKey, model: 'gpt-4o-mini', - temperature: 0, + temperature: 0.7, }) : null @@ -118,14 +200,15 @@ export default defineLazyEventHandler(() => { content: [ { type: 'text', - content: appLangchainPrompt, + text: appLangchainPrompt, }, ], }), + middleware: [toolMonitoringMiddleware /* , loggingMiddleware*/] as any, }) traceMcp( - `Starting chat stream for conversation ${conversationId} with ${messages.length} messages` + `Starting chat stream for conversation ${conversationId} with ${tools.map((t) => `"${t.name}"`).join(', ')} tools and ${messages.length} messages` ) tokenMap.set(conversationId, { @@ -169,23 +252,42 @@ export default defineLazyEventHandler(() => { */ // TODO: fix typescript mess with agent.stream return type for await (const [token, metadata] of (await agent.stream( - { messages: messages.map((msg) => new HumanMessage(msg.text)) } as any, + // { messages: messages.map((msg) => new HumanMessage(msg.text)) } as any, + { + messages: messages.map((msg) => + msg.role === 'ai' || msg.role === 'assistant' + ? new AIMessage(msg.text) + : new HumanMessage(msg.text) + ), + } as any, { ...config, streamMode: 'messages' } // ,{ options: { stream: true }, previous_response_id: conversationId,} - )) as AsyncIterableIterator< - [ - { - contentBlocks?: Array<{ type: string; index: number; text: string }> - }, - { status: string; langgraph_node?: any }, - ] - >) { + )) as AsyncIterableIterator<[BaseMessageChunk, { status: string; langgraph_node?: any }]>) { // TODO: handle tools and reaoning chunks // console.log('chunk from lg node', metadata.langgraph_node) + // console.log('-----------------------------') + // console.log('--METADATA chunk from lg node\n', JSON.stringify(metadata, null, 2)) + // console.log('-----------------------------') + // console.log('--TOKEN chunk from lg node\n', JSON.stringify(token, null, 2)) + // console.log('-----------------------------') + + // console.log( + // '-----> TOKEN ID\n', + // token.constructor.name, + // (token.lc_id && token.lc_id.join && token.lc_id.join(', ')) || token.lc_id + // ) + // console.log('-----> META NODE STEP\n', metadata.langgraph_node, metadata.langgraph_step) + + // console.log('---->', token.contentBlocks.map((b) => b.type).join(', ')) + + // prevent writing tool message to front + if (token.lc_id[token.lc_id.length - 1] != 'AIMessageChunk') continue + const content = token.contentBlocks || [] // sort in ascending index order and join texts (is it really necessary ?) - const ordered_content = content.sort((a, b) => a.index - b.index) - const text = ordered_content + // const ordered_content = content.sort((a: int, b: int) => a.index - b.index) + // const text = ordered_content + const text = content .filter((part) => part.type == 'text') .map((part) => part.text) .join('') diff --git a/src/server/routes/api/vector-store/delete.delete.ts b/src/server/routes/api/vector-store/delete.delete.ts new file mode 100644 index 000000000..f59cf5a6b --- /dev/null +++ b/src/server/routes/api/vector-store/delete.delete.ts @@ -0,0 +1,26 @@ +import getVectorStore from '@/server/utils/vector-db.js' +import checkVectorDbRights from '@/server/utils/check-vector-db-rights.js' + +export default defineLazyEventHandler(() => { + return defineEventHandler(async (event) => { + await checkVectorDbRights(event) + const title = getQuery(event)?.title as string + + const { appApiOrgCode } = useRuntimeConfig().public + const { vectorStore } = await getVectorStore() + // return 404 if not configured + if (!vectorStore) { + setResponseStatus(event, 404) + return + } + + await vectorStore.delete({ + filter: { + title: title, + orgCode: appApiOrgCode, + }, + }) + + return { status: 'ok' } + }) +}) diff --git a/src/server/routes/api/vector-store/get.get.ts b/src/server/routes/api/vector-store/get.get.ts new file mode 100644 index 000000000..a9413b0af --- /dev/null +++ b/src/server/routes/api/vector-store/get.get.ts @@ -0,0 +1,45 @@ +import getVectorStore from '@/server/utils/vector-db.js' +import checkVectorDbRights from '@/server/utils/check-vector-db-rights.js' + +export default defineLazyEventHandler(() => { + return defineEventHandler(async (event) => { + await checkVectorDbRights(event) + const title = getQuery(event)?.title as string + + const { appApiOrgCode } = useRuntimeConfig().public + const { pool } = await getVectorStore() + const runtimeConfig = useRuntimeConfig() + const vectorTableName = runtimeConfig.appVectorTableName + // return 404 if not configured + if (!pool || !vectorTableName) { + setResponseStatus(event, 404) + return + } + + const client = await pool.connect() + + console.log('title', title) + try { + const result = await client.query( + ` + SELECT v.content as content, v.metadata as metadata + FROM ${vectorTableName} v + WHERE v.metadata->>'orgCode' = $1 + AND v.metadata->>'title' = $2 + ORDER BY v.metadata->'loc'->'pageNumber' ASC, v.metadata->'loc'->'lines'->'from' ASC + `, + [appApiOrgCode, title] + ) + + const docs = result.rows + console.log('docs', docs.length) + // docs.forEach((doc) => { + // console.log(doc.title, doc.chunks) + // }) + + return docs + } finally { + client.release() + } + }) +}) diff --git a/src/server/routes/api/vector-store/ingest.post.ts b/src/server/routes/api/vector-store/ingest.post.ts new file mode 100644 index 000000000..125561092 --- /dev/null +++ b/src/server/routes/api/vector-store/ingest.post.ts @@ -0,0 +1,60 @@ +import getVectorStore from '@/server/utils/vector-db.js' +import { PDFLoader } from '@langchain/community/document_loaders/fs/pdf' +import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters' +import checkVectorDbRights from '@/server/utils/check-vector-db-rights.js' + +export default defineLazyEventHandler(() => { + return defineEventHandler(async (event) => { + await checkVectorDbRights(event) + + const { appApiOrgCode } = useRuntimeConfig().public + const { vectorStore } = await getVectorStore() + // return 404 if not configured + if (!vectorStore) { + setResponseStatus(event, 404) + return + } + + const formData = await readFormData(event) + + const file = formData.get('file') as File + const title = formData.get('title') as string + + if (!file || !title) { + throw createError({ + statusCode: 400, + message: 'Missing required fields: file and title', + }) + } + + // const arrayBuffer = await file.arrayBuffer() + // const buffer = Buffer.from(arrayBuffer) + const loader = new PDFLoader(file) + + const fileDocs = await loader.load() + const extraMetadata = { + title: title, + timestamp: new Date().toISOString(), + orgCode: appApiOrgCode, + } + const fileDocsWithMeta = fileDocs.map((d) => ({ + ...d, + metadata: { ...d.metadata, ...extraMetadata }, + })) + + // chunking express + const splitter = new RecursiveCharacterTextSplitter({ chunkSize: 500, chunkOverlap: 100 }) + const chunks = await splitter.splitDocuments(fileDocsWithMeta) + + await vectorStore.delete({ + filter: { + title: extraMetadata.title, + orgCode: extraMetadata.orgCode, + }, + }) + + await vectorStore.addDocuments(chunks) + + return { status: 'ok', chunks } + }) +}) diff --git a/src/server/routes/api/vector-store/list.get.ts b/src/server/routes/api/vector-store/list.get.ts new file mode 100644 index 000000000..12fe2cdb2 --- /dev/null +++ b/src/server/routes/api/vector-store/list.get.ts @@ -0,0 +1,41 @@ +import getVectorStore from '@/server/utils/vector-db.js' +import checkVectorDbRights from '@/server/utils/check-vector-db-rights.js' + +export default defineLazyEventHandler(() => { + return defineEventHandler(async (event) => { + await checkVectorDbRights(event) + + const { appApiOrgCode } = useRuntimeConfig().public + const { pool } = await getVectorStore() + const runtimeConfig = useRuntimeConfig() + const vectorTableName = runtimeConfig.appVectorTableName + // return 404 if not configured + if (!pool || !vectorTableName) { + setResponseStatus(event, 404) + return + } + + const client = await pool.connect() + + try { + const result = await client.query( + ` + SELECT distinct(v.metadata->>'title') as title, count(*) as chunks + FROM ${vectorTableName} v + WHERE v.metadata->>'orgCode' = $1 + GROUP BY v.metadata->>'title' + `, + [appApiOrgCode] + ) + + const docs = result.rows + docs.forEach((doc) => { + console.log(doc.title, doc.chunks) + }) + + return docs + } finally { + client.release() + } + }) +}) diff --git a/src/server/utils/check-vector-db-rights.js b/src/server/utils/check-vector-db-rights.js new file mode 100644 index 000000000..51d1b15ce --- /dev/null +++ b/src/server/utils/check-vector-db-rights.js @@ -0,0 +1,45 @@ +function atob(b64Encoded) { + return Buffer.from(b64Encoded, 'base64').toString() +} + +export function parseJwt(token) { + let base64Url = token.split('.')[1] + let base64 = base64Url.replace(/-/g, '+').replace(/_/g, '/') + let jsonPayload = decodeURIComponent( + atob(base64) + .split('') + .map(function (c) { + return '%' + ('00' + c.charCodeAt(0).toString(16)).slice(-2) + }) + .join('') + ) + + return JSON.parse(jsonPayload) +} + +export default async function checkVectorDbRights(event) { + const tokenHeader = getRequestHeader(event, 'authorization') || '' + if (!tokenHeader) { + throw createError({ + statusCode: 401, + statusMessage: 'Unauthorized', + message: 'You must authenticate to access this resource.', + }) + } + + const jwt = parseJwt(tokenHeader) + + const kcId = jwt.sub + + const runtimeConfig = useRuntimeConfig() + const baseUrl = runtimeConfig.public.appApiUrl + runtimeConfig.public.appApiDefaultVersion + '/' + const user = await $fetch(`${baseUrl}/user/${kcId}/`, { headers: { Authorization: tokenHeader } }) + + if (!user.is_superuser) { + throw createError({ + statusCode: 403, + statusMessage: 'Forbidden', + message: 'You do not have permission to access this resource.', + }) + } +} diff --git a/src/server/utils/vector-db.js b/src/server/utils/vector-db.js new file mode 100644 index 000000000..46b01899a --- /dev/null +++ b/src/server/utils/vector-db.js @@ -0,0 +1,73 @@ +import { PGVectorStore } from '@langchain/community/vectorstores/pgvector' +import { OpenAIEmbeddings } from '@langchain/openai' // Or any other embedding model +// import { Client } from 'pg' +import { parse } from 'pg-connection-string' +import pg from 'pg' + +export default async () => { + const runtimeConfig = useRuntimeConfig() + // 1. Configure the database connection + const connectionString = runtimeConfig.appVectorDbUrl + const modelName = runtimeConfig.appVectorEmbeddingModel + const vectorDimensions = runtimeConfig.appVectorEmbeddingDimensions + ? parseInt(runtimeConfig.appVectorEmbeddingDimensions) + : null + const apiKey = runtimeConfig.appVectorEmbeddingApiKey + const vectorTableName = runtimeConfig.appVectorTableName + + let vectorStore = null + let pool = null + try { + if (!connectionString || !modelName || !vectorDimensions || !apiKey || !vectorTableName) { + throw new Error('Missing required configuration for vector store.') + } + + // 2. Create a PgPool or Client instance (LangChain accepts both, but Pool is better for production) + // For simplicity in initialization, we'll pass the connection string directly to the store constructor + // which handles the connection internally for setup. + + const embeddings = new OpenAIEmbeddings({ + modelName, + apiKey, + }) + + pool = new pg.Pool(parse(connectionString)) + + const config = { + pool, + dimensions: vectorDimensions, + tableName: vectorTableName, + columns: { + idColumnName: 'id', + vectorColumnName: 'vector', + contentColumnName: 'content', + metadataColumnName: 'metadata', + }, + // supported distance strategies: cosine (default), innerProduct, or euclidean + distanceStrategy: 'cosine', // as DistanceStrategy, + } + + // 3. Initialize the VectorStore - skip if not needed + vectorStore = await PGVectorStore.initialize( + embeddings, + config + // Optional: Specify a custom table name if you don't want the default 'langchain_pg_embedding' + // tableName: "my_custom_table", + // Optional: Use JSONB for metadata (default is usually JSONB in newer versions) + // useJsonb: true, + ) + + console.log('Database connection established and tables initialized.') + + // 4. (Optional) Add some documents to verify it works + // const docs = [ + // new Document({ pageContent: "Hello world", metadata: { source: "test" } }), + // ]; + // await vectorStore.addDocuments(docs); + } catch (error) { + console.error('Error initializing vector store:', error) + // Handle the error as needed, e.g., fallback to an in-memory store or disable vector features + } + + return { pool, vectorStore } +} diff --git a/yarn.lock b/yarn.lock index 177bc56e3..884b11bab 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1281,20 +1281,53 @@ resolved "https://registry.yarnpkg.com/@kwsites/promise-deferred/-/promise-deferred-1.1.1.tgz#8ace5259254426ccef57f3175bc64ed7095ed919" integrity sha512-GaHYm+c0O9MjZRu0ongGBRbinu8gVAMd2UZjji6jVmqKtZluZnptXGWhz1E8j8D2HJ3f/yMxKAUC0b+57wncIw== -"@langchain/core@^1.1.17": - version "1.1.17" - resolved "https://registry.yarnpkg.com/@langchain/core/-/core-1.1.17.tgz#b5a9a9fb51e9f1963581d2f7ce1e22867865d8a6" - integrity sha512-g7/kcKbKEwNZSyyT7aT0utxn7wTOtKErqz0cL6VjrV4v/aOb9g+dKcfj17YkSm42YQmJp/rB2IXGc17vQPEBqA== +"@langchain/classic@1.0.23": + version "1.0.23" + resolved "https://registry.yarnpkg.com/@langchain/classic/-/classic-1.0.23.tgz#9578052a115a94e89341fa3fe80427461f011fcf" + integrity sha512-bqL1hUE6eG9Yvyiocg8jJ8NR7Vm+zVhB1wRr9od+dbyFVC3+utrY7m1zm6PortVx1wcdd9UPRu3elEa1sdQA8A== + dependencies: + "@langchain/openai" "1.2.13" + "@langchain/textsplitters" "1.0.1" + handlebars "^4.7.8" + js-yaml "^4.1.1" + jsonpointer "^5.0.1" + openapi-types "^12.1.3" + uuid "^10.0.0" + yaml "^2.2.1" + zod "^3.25.76 || ^4" + optionalDependencies: + langsmith ">=0.4.0 <1.0.0" + +"@langchain/community@^1.1.23": + version "1.1.23" + resolved "https://registry.yarnpkg.com/@langchain/community/-/community-1.1.23.tgz#5e58dc39c9edd38d8cb462e37fc6f9d8874f7ef3" + integrity sha512-b4Kpau7wd6wYQEz2Sl38wA3COJiEKMAvI5PksaXGWI151vbYMNWSGVh7HPVhyIotd20NEXQAc5KdifTqV48jPQ== + dependencies: + "@langchain/classic" "1.0.23" + "@langchain/openai" "1.2.13" + binary-extensions "^2.2.0" + flat "^5.0.2" + js-yaml "^4.1.1" + langsmith ">=0.4.0 <1.0.0" + math-expression-evaluator "^2.0.0" + uuid "^10.0.0" + zod "^3.25.76 || ^4" + +"@langchain/core@^1.1.32": + version "1.1.32" + resolved "https://registry.yarnpkg.com/@langchain/core/-/core-1.1.32.tgz#a9baf37e5cb40e8b877160d5d03ef3bf6e8e250d" + integrity sha512-ZZNiER5tceFXqZOghfrxNHzM60gcQL5XK/8Ow5+o4OuKHrP1p/RUQBDM9Y1nddi/VmKQj+ncaXXM5KovXTEGGQ== dependencies: "@cfworker/json-schema" "^4.0.2" + "@standard-schema/spec" "^1.1.0" ansi-styles "^5.0.0" camelcase "6" decamelize "1.2.0" js-tiktoken "^1.0.12" - langsmith ">=0.4.0 <1.0.0" + langsmith ">=0.5.0 <1.0.0" mustache "^4.2.0" p-queue "^6.6.2" - uuid "^10.0.0" + uuid "^11.1.0" zod "^3.25.76 || ^4" "@langchain/langgraph-checkpoint@^1.0.0": @@ -1323,6 +1356,15 @@ "@standard-schema/spec" "1.1.0" uuid "^10.0.0" +"@langchain/openai@1.2.13": + version "1.2.13" + resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-1.2.13.tgz#ce3f1dd08b93458d43a414128caeb70be4b2a462" + integrity sha512-zyspKSBnyxICxZyGYMxsSkCUJR09Uj1omMGBrUsvpSu9j+zIbfNg4+NQTIRmnzULoKB+keaioOdLGHxOQru3+w== + dependencies: + js-tiktoken "^1.0.12" + openai "^6.27.0" + zod "^3.25.76 || ^4" + "@langchain/openai@^1.2.3": version "1.2.3" resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-1.2.3.tgz#39f4b843ea72d71bf653aa21896f151311027e27" @@ -1332,6 +1374,13 @@ openai "^6.16.0" zod "^3.25.76 || ^4" +"@langchain/textsplitters@1.0.1", "@langchain/textsplitters@^1.0.1": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@langchain/textsplitters/-/textsplitters-1.0.1.tgz#292f9c93239178c248b3338acf7b68aa47aa9830" + integrity sha512-rheJlB01iVtrOUzttscutRgLybPH9qR79EyzBEbf1u97ljWyuxQfCwIWK+SjoQTM9O8M7GGLLRBSYE26Jmcoww== + dependencies: + js-tiktoken "^1.0.12" + "@lifeomic/attempt@^3.0.2": version "3.1.0" resolved "https://registry.yarnpkg.com/@lifeomic/attempt/-/attempt-3.1.0.tgz#7fc703559177b81a008b9d263e3d9a001d11d08a" @@ -2646,7 +2695,7 @@ resolved "https://registry.yarnpkg.com/@speed-highlight/core/-/core-1.2.7.tgz#eeaa7c1e7198559abbb98e4acbc93d108d35f2d3" integrity sha512-0dxmVj4gxg3Jg879kvFS/msl4s9F3T9UXC1InxgOf7t5NvcPD97u/WTA5vL/IxWHMn7qSxBozqrnnE2wvl1m8g== -"@standard-schema/spec@1.1.0": +"@standard-schema/spec@1.1.0", "@standard-schema/spec@^1.1.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@standard-schema/spec/-/spec-1.1.0.tgz#a79b55dbaf8604812f52d140b2c9ab41bc150bb8" integrity sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w== @@ -4400,7 +4449,7 @@ big.js@^5.2.2: resolved "https://registry.yarnpkg.com/big.js/-/big.js-5.2.2.tgz#65f0af382f578bcdc742bd9c281e9cb2d7768328" integrity sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ== -binary-extensions@^2.0.0: +binary-extensions@^2.0.0, binary-extensions@^2.2.0: version "2.3.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522" integrity sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw== @@ -4662,7 +4711,7 @@ chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.2: ansi-styles "^4.1.0" supports-color "^7.1.0" -chalk@^5.4.1: +chalk@^5.4.1, chalk@^5.6.2: version "5.6.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-5.6.2.tgz#b1238b6e23ea337af71c7f8a295db5af0c158aea" integrity sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA== @@ -6341,6 +6390,11 @@ flat-cache@^6.1.13: flatted "^3.3.3" hookified "^1.12.0" +flat@^5.0.2: + version "5.0.2" + resolved "https://registry.yarnpkg.com/flat/-/flat-5.0.2.tgz#8ca6fe332069ffa9d324c327198c598259ceb241" + integrity sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ== + flatted@^3.2.9: version "3.3.2" resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.3.2.tgz#adba1448a9841bec72b42c532ea23dbbedef1a27" @@ -6719,6 +6773,18 @@ h3@^1.12.0, h3@^1.15.3, h3@^1.15.4: ufo "^1.6.1" uncrypto "^0.1.3" +handlebars@^4.7.8: + version "4.7.8" + resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.7.8.tgz#41c42c18b1be2365439188c77c6afae71c0cd9e9" + integrity sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ== + dependencies: + minimist "^1.2.5" + neo-async "^2.6.2" + source-map "^0.6.1" + wordwrap "^1.0.0" + optionalDependencies: + uglify-js "^3.1.4" + happy-dom@^17.0.0: version "17.6.3" resolved "https://registry.yarnpkg.com/happy-dom/-/happy-dom-17.6.3.tgz#2caa37058e6f9605f1b8b7ae03045253e6089477" @@ -7001,6 +7067,11 @@ ini@^1.3.4, ini@^1.3.5: resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== +install@^0.13.0: + version "0.13.0" + resolved "https://registry.yarnpkg.com/install/-/install-0.13.0.tgz#6af6e9da9dd0987de2ab420f78e60d9c17260776" + integrity sha512-zDml/jzr2PKU9I8J/xyZBQn8rPCAY//UOYNmR01XwNwyfhEWObo2SWfSl1+0tm1u6PhxLwDnfsT/6jB7OUxqFA== + invariant@^2.2.2: version "2.2.4" resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.4.tgz#610f3c92c9359ce1db616e538008d23ff35158e6" @@ -7352,6 +7423,13 @@ js-yaml@^4.1.0: dependencies: argparse "^2.0.1" +js-yaml@^4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-4.1.1.tgz#854c292467705b699476e1a2decc0c8a3458806b" + integrity sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA== + dependencies: + argparse "^2.0.1" + jsesc@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-1.3.0.tgz#46c3fec8c1892b12b0833db9bc7622176dbab34b" @@ -7442,6 +7520,11 @@ jsonparse@^1.2.0: resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" integrity sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg== +jsonpointer@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/jsonpointer/-/jsonpointer-5.0.1.tgz#2110e0af0900fd37467b5907ecd13a7884a1b559" + integrity sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ== + keyv@^4.5.4: version "4.5.4" resolved "https://registry.yarnpkg.com/keyv/-/keyv-4.5.4.tgz#a879a99e29452f942439f2a405e3af8b31d4de93" @@ -7519,6 +7602,18 @@ langchain@^1.2.14: semver "^7.6.3" uuid "^10.0.0" +"langsmith@>=0.5.0 <1.0.0": + version "0.5.9" + resolved "https://registry.yarnpkg.com/langsmith/-/langsmith-0.5.9.tgz#7cfe57302bf008d2d03f6042c0b1505581cb8573" + integrity sha512-mT88HlYTz7IZzVJP/OM7D6PAR3kzcbPGjYDJpx64ua9WiJXFqQx6YKewxIIxkVSZCb9I/9TiYvPYfZ4rawOdEg== + dependencies: + "@types/uuid" "^10.0.0" + chalk "^5.6.2" + console-table-printer "^2.12.1" + p-queue "^6.6.2" + semver "^7.6.3" + uuid "^10.0.0" + launch-editor@^2.11.1: version "2.11.1" resolved "https://registry.yarnpkg.com/launch-editor/-/launch-editor-2.11.1.tgz#61a0b7314a42fd84a6cbb564573d9e9ffcf3d72b" @@ -7885,6 +7980,11 @@ material-colors@^1.2.6: resolved "https://registry.yarnpkg.com/material-colors/-/material-colors-1.2.6.tgz#6d1958871126992ceecc72f4bcc4d8f010865f46" integrity sha512-6qE4B9deFBIa9YSpOc9O0Sgc43zTeVYbgDT5veRKSlB2+ZuHNoVVxA1L/ckMUayV9Ay9y7Z/SZCLcGteW9i7bg== +math-expression-evaluator@^2.0.0: + version "2.0.7" + resolved "https://registry.yarnpkg.com/math-expression-evaluator/-/math-expression-evaluator-2.0.7.tgz#dc99a80ce2bf7f9b7df878126feb5c506c1fdf5f" + integrity sha512-uwliJZ6BPHRq4eiqNWxZBDzKUiS5RIynFFcgchqhBOloVLVBpZpNG8jRYkedLcBvhph8TnRyWEuxPqiQcwIdog== + math-intrinsics@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/math-intrinsics/-/math-intrinsics-1.1.0.tgz#a0dd74be81e2aa5c2f27e65ce283605ee4e2b7f9" @@ -8087,7 +8187,7 @@ minimist-options@4.1.0: is-plain-obj "^1.1.0" kind-of "^6.0.3" -minimist@^1.2.6: +minimist@^1.2.5, minimist@^1.2.6: version "1.2.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== @@ -8209,6 +8309,11 @@ negotiator@^1.0.0: resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-1.0.0.tgz#b6c91bb47172d69f93cfd7c357bbb529019b5f6a" integrity sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg== +neo-async@^2.6.2: + version "2.6.2" + resolved "https://registry.yarnpkg.com/neo-async/-/neo-async-2.6.2.tgz#b4aafb93e3aeb2d8174ca53cf163ab7d7308305f" + integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== + nitropack@^2.12.6: version "2.12.7" resolved "https://registry.yarnpkg.com/nitropack/-/nitropack-2.12.7.tgz#8c6dac6c9896c6f41374f805f077bdf6e100f93f" @@ -8298,6 +8403,11 @@ node-addon-api@^7.0.0: resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-7.1.1.tgz#1aba6693b0f255258a049d621329329322aad558" integrity sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ== +node-ensure@^0.0.0: + version "0.0.0" + resolved "https://registry.yarnpkg.com/node-ensure/-/node-ensure-0.0.0.tgz#ecae764150de99861ec5c810fd5d096b183932a7" + integrity sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw== + node-fetch-native@^1.6.4, node-fetch-native@^1.6.5, node-fetch-native@^1.6.6, node-fetch-native@^1.6.7: version "1.6.7" resolved "https://registry.yarnpkg.com/node-fetch-native/-/node-fetch-native-1.6.7.tgz#9d09ca63066cc48423211ed4caf5d70075d76a71" @@ -8624,6 +8734,16 @@ openai@^6.16.0: resolved "https://registry.yarnpkg.com/openai/-/openai-6.16.0.tgz#53661d9f6307dc7523e89637ff7c6ccd0be16c67" integrity sha512-fZ1uBqjFUjXzbGc35fFtYKEOxd20kd9fDpFeqWtsOZWiubY8CZ1NAlXHW3iathaFvqmNtCWMIsosCuyeI7Joxg== +openai@^6.27.0: + version "6.27.0" + resolved "https://registry.yarnpkg.com/openai/-/openai-6.27.0.tgz#eabaea9ce6904a8b2e6b2ac0a9d69b527d95ecf6" + integrity sha512-osTKySlrdYrLYTt0zjhY8yp0JUBmWDCN+Q+QxsV4xMQnnoVFpylgKGgxwN8sSdTNw0G4y+WUXs4eCMWpyDNWZQ== + +openapi-types@^12.1.3: + version "12.1.3" + resolved "https://registry.yarnpkg.com/openapi-types/-/openapi-types-12.1.3.tgz#471995eb26c4b97b7bd356aacf7b91b73e777dd3" + integrity sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw== + optionator@^0.9.3: version "0.9.4" resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.4.tgz#7ea1c1a5d91d764fb282139c88fe11e182a3a734" @@ -9025,6 +9145,13 @@ pathval@^2.0.0: resolved "https://registry.yarnpkg.com/pathval/-/pathval-2.0.1.tgz#8855c5a2899af072d6ac05d11e46045ad0dc605d" integrity sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ== +pdf-parse@^1: + version "1.1.4" + resolved "https://registry.yarnpkg.com/pdf-parse/-/pdf-parse-1.1.4.tgz#97bca6f46758130dafb1fdd9df905efd07581f4a" + integrity sha512-XRIRcLgk6ZnUbsHsYXExMw+krrPE81hJ6FQPLdBNhhBefqIQKXu/WeTgNBGSwPrfU0v+UCEwn7AoAUOsVKHFvQ== + dependencies: + node-ensure "^0.0.0" + perfect-debounce@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/perfect-debounce/-/perfect-debounce-1.0.0.tgz#9c2e8bc30b169cc984a58b7d5b28049839591d2a" @@ -9035,6 +9162,62 @@ perfect-debounce@^2.0.0: resolved "https://registry.yarnpkg.com/perfect-debounce/-/perfect-debounce-2.0.0.tgz#0ff94f1ecbe0a6bca4b1703a2ed08bbe43739aa7" integrity sha512-fkEH/OBiKrqqI/yIgjR92lMfs2K8105zt/VT6+7eTjNwisrsh47CeIED9z58zI7DfKdH3uHAn25ziRZn3kgAow== +pg-cloudflare@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.3.0.tgz#386035d4bfcf1a7045b026f8b21acf5353f14d65" + integrity sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ== + +pg-connection-string@^2.12.0: + version "2.12.0" + resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.12.0.tgz#4084f917902bb2daae3dc1376fe24ac7b4eaccf2" + integrity sha512-U7qg+bpswf3Cs5xLzRqbXbQl85ng0mfSV/J0nnA31MCLgvEaAo7CIhmeyrmJpOr7o+zm0rXK+hNnT5l9RHkCkQ== + +pg-int8@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" + integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw== + +pg-pool@^3.13.0: + version "3.13.0" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.13.0.tgz#416482e9700e8f80c685a6ae5681697a413c13a3" + integrity sha512-gB+R+Xud1gLFuRD/QgOIgGOBE2KCQPaPwkzBBGC9oG69pHTkhQeIuejVIk3/cnDyX39av2AxomQiyPT13WKHQA== + +pg-protocol@^1.13.0: + version "1.13.0" + resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.13.0.tgz#fdaf6d020bca590d58bb991b4b16fc448efe0511" + integrity sha512-zzdvXfS6v89r6v7OcFCHfHlyG/wvry1ALxZo4LqgUoy7W9xhBDMaqOuMiF3qEV45VqsN6rdlcehHrfDtlCPc8w== + +pg-types@2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3" + integrity sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA== + dependencies: + pg-int8 "1.0.1" + postgres-array "~2.0.0" + postgres-bytea "~1.0.0" + postgres-date "~1.0.4" + postgres-interval "^1.1.0" + +pg@^8.20.0: + version "8.20.0" + resolved "https://registry.yarnpkg.com/pg/-/pg-8.20.0.tgz#1a274de944cb329fd6dd77a6d371a005ba6b136d" + integrity sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA== + dependencies: + pg-connection-string "^2.12.0" + pg-pool "^3.13.0" + pg-protocol "^1.13.0" + pg-types "2.2.0" + pgpass "1.0.5" + optionalDependencies: + pg-cloudflare "^1.3.0" + +pgpass@1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d" + integrity sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug== + dependencies: + split2 "^4.1.0" + picocolors@^1.0.0, picocolors@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b" @@ -9402,6 +9585,28 @@ postcss@^8.4.31, postcss@^8.4.38, postcss@^8.4.43, postcss@^8.5.0, postcss@^8.5. picocolors "^1.1.1" source-map-js "^1.2.1" +postgres-array@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-2.0.0.tgz#48f8fce054fbc69671999329b8834b772652d82e" + integrity sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA== + +postgres-bytea@~1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/postgres-bytea/-/postgres-bytea-1.0.1.tgz#c40b3da0222c500ff1e51c5d7014b60b79697c7a" + integrity sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ== + +postgres-date@~1.0.4: + version "1.0.7" + resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.7.tgz#51bc086006005e5061c591cee727f2531bf641a8" + integrity sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q== + +postgres-interval@^1.1.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.2.0.tgz#b460c82cb1587507788819a06aa0fffdb3544695" + integrity sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ== + dependencies: + xtend "^4.0.0" + prelude-ls@^1.2.1: version "1.2.1" resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" @@ -10422,6 +10627,11 @@ split2@^3.0.0, split2@^3.2.2: dependencies: readable-stream "^3.0.0" +split2@^4.1.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/split2/-/split2-4.2.0.tgz#c9c5920904d148bab0b9f67145f245a86aadbfa4" + integrity sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg== + sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" @@ -11133,6 +11343,11 @@ ufo@1.6.1, ufo@^1.1.2, ufo@^1.5.4, ufo@^1.6.1: resolved "https://registry.yarnpkg.com/ufo/-/ufo-1.6.1.tgz#ac2db1d54614d1b22c1d603e3aef44a85d8f146b" integrity sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA== +uglify-js@^3.1.4: + version "3.19.3" + resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-3.19.3.tgz#82315e9bbc6f2b25888858acd1fff8441035b77f" + integrity sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ== + ultrahtml@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/ultrahtml/-/ultrahtml-1.6.0.tgz#0d1aad7bbfeae512438d30e799c11622127a1ac8" @@ -11367,6 +11582,11 @@ uuid@^10.0.0: resolved "https://registry.yarnpkg.com/uuid/-/uuid-10.0.0.tgz#5a95aa454e6e002725c79055fd42aaba30ca6294" integrity sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ== +uuid@^11.1.0: + version "11.1.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-11.1.0.tgz#9549028be1753bb934fc96e2bca09bb4105ae912" + integrity sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A== + uuid@^13.0.0: version "13.0.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-13.0.0.tgz#263dc341b19b4d755eb8fe36b78d95a6b65707e8" @@ -11803,6 +12023,11 @@ word-wrap@^1.2.5: resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.5.tgz#d2c45c6dd4fbce621a66f136cbe328afd0410b34" integrity sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA== +wordwrap@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" + integrity sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q== + "wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" @@ -11893,6 +12118,11 @@ xml-name-validator@^4.0.0: resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-4.0.0.tgz#79a006e2e63149a8600f15430f0a4725d1524835" integrity sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw== +xtend@^4.0.0: + version "4.0.2" + resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" + integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== + y-prosemirror@^1.2.11: version "1.3.7" resolved "https://registry.yarnpkg.com/y-prosemirror/-/y-prosemirror-1.3.7.tgz#f88e553da4ea33278b114cf0b6a0ea978b154e84" @@ -11959,6 +12189,11 @@ yaml@^2.0.0, yaml@^2.7.0: resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.7.0.tgz#aef9bb617a64c937a9a748803786ad8d3ffe1e98" integrity sha512-+hSoy/QHluxmC9kCIJyL/uyFmLmc+e5CFR5Wa+bpIhIj85LVb9ZH2nVnqrHoSvKogwODv0ClqZkmiSSaIH5LTA== +yaml@^2.2.1: + version "2.8.2" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.8.2.tgz#5694f25eca0ce9c3e7a9d9e00ce0ddabbd9e35c5" + integrity sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A== + yaml@^2.8.0: version "2.8.1" resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.8.1.tgz#1870aa02b631f7e8328b93f8bc574fac5d6c4d79"