Chatbot with langchain vector store

.env.sample

@@ -5,7 +5,7 @@
 
 # Use DEFAULT for the meta portal
 NUXT_PUBLIC_APP_API_ORG_CODE=CRI
-# Custom favicon 
+# Custom favicon
 # (can be an absolute path, else will be relative to NUXT_PUBLIC_APP_PUBLIC_BINARIES_PREFIX)
 NUXT_PUBLIC_APP_FAVICON=favicon.ico
 # url prefix for a binary file storage (images...)
@@ -77,7 +77,7 @@ PLAYWRIGHT_TEST_PASSWORD=
 ### examples suggestions separated by § character
 # NUXT_PUBLIC_APP_CHATBOT_EXEMPLES=Tell me about this platform§Find research projects on renewable energy§Find researchers and publication on climate change
 
-# ==== OpenAI PRIVATE settings ==== 
+# ==== OpenAI PRIVATE settings ====
 # OpenAI API Key for the chatbot
 # optional
 # NUXT_APP_OPENAI_API_KEY=sk-proj-XXXXXXXXXXXXXXXXXXXXXXXXXXXXX
@@ -96,4 +96,12 @@ PLAYWRIGHT_TEST_PASSWORD=
 
 # ==== GOTENBERG pdf (optional) Settings ====
 # NUXT_APP_GOTENBERG_SERVER_URL=http://127.0.0.1:3333
-# NUXT_PUBLIC_APP_GOTENBERG_ENABLED='true'
+# NUXT_PUBLIC_APP_GOTENBERG_ENABLED='true'
+
+# ==== VECTOR DATABASE (optional) Settings ====
+# NUXT_APP_VECTOR_DB_URL=postgresql://myuser:mypassword@localhost:5432/mydb
+# NUXT_APP_VECTOR_EMBEDDING_MODEL=text-embedding-3-small
+# NUXT_APP_VECTOR_EMBEDDING_API_KEY=XXXXX
+# NUXT_APP_VECTOR_EMBEDDING_DIMENSIONS=1536
+# NUXT_APP_VECTOR_TOOL_PROMPT="use this tool to answer about neuroscience project"
+# NUXT_APP_VECTOR_TABLE_NAME=projects-vector

.zed/debug.json

@@ -0,0 +1,9 @@
+[
+  {
+    "label": "Attach to Nuxt",
+    "adapter": "JavaScript",
+    "request": "attach",
+    "port": 9229,
+    "sourceMaps": true
+  }
+]

k8s/projects-frontend/values.dev.yaml

@@ -1,11 +1,17 @@
 config:
   nonSensitive:
     NUXT_PUBLIC_APP_KEYCLOAK_URL: https://keycloak.k8s.lp-i.dev
-    NUXT_PUBLIC_APP_VERSION: "{{ .Values.application.revision }}"
+    NUXT_PUBLIC_APP_VERSION: '{{ .Values.application.revision }}'
     NUXT_PUBLIC_APP_ENVIRONMENT: dev
     NUXT_APP_GOTENBERG_SERVER_URL: 'http://projects-gotenberg'
     NUXT_PUBLIC_APP_GOTENBERG_ENABLED: 'true'
     NUXT_APP_LANGCHAIN_PROMPT: ''
+    NUXT_APP_VECTOR_DB_URL: ''
+    NUXT_APP_VECTOR_EMBEDDING_API_KEY: ''
+    NUXT_APP_VECTOR_EMBEDDING_MODEL: ''
+    NUXT_APP_VECTOR_EMBEDDING_DIMENSIONS: ''
+    NUXT_APP_VECTOR_TOOL_PROMPT: ''
+    NUXT_APP_VECTOR_TABLE_NAME: ''
 
 domain: k8s.lp-i.dev
 

k8s/projects-frontend/values.yaml

@@ -45,6 +45,12 @@ config:
     NUXT_APP_GOTENBERG_SERVER_URL: ''
     NUXT_PUBLIC_APP_GOTENBERG_ENABLED: ''
     NUXT_APP_LANGCHAIN_PROMPT: ''
+    NUXT_APP_VECTOR_DB_URL: ''
+    NUXT_APP_VECTOR_EMBEDDING_API_KEY: ''
+    NUXT_APP_VECTOR_EMBEDDING_MODEL: ''
+    NUXT_APP_VECTOR_EMBEDDING_DIMENSIONS: ''
+    NUXT_APP_VECTOR_TOOL_PROMPT: ''
+    NUXT_APP_VECTOR_TABLE_NAME: ''
 e2eEnv:
   nonSensitive:
     USER_ADMIN_EMAIL: testautomatatiquedministrateur1@outlook.fr

nuxt.config.ts

@@ -131,14 +131,20 @@ export default defineNuxtConfig({
     appGeocodingApiKey: '',
     appOpenaiApiKey: '',
     appOpenaiApiPromptId: '',
-    appOpenaiApiPromptVersion: '',
+    appOpeyanaiApiPromptVersion: '',
     appOpenaiApiVectorStoreId: '',
     appMcpServerUrl: '',
     appSorbobotApiToken: '',
     appMcpServerTrace: 0,
     appSorbobotApiTrace: 0,
     appGotenbergServerUrl: '',
     appLangchainPrompt: '',
+    appVectorDbUrl: '',
+    appVectorEmbeddingApiKey: '',
+    appVectorTableName: '',
+    appVectorEmbeddingModel: '',
+    appVectorEmbeddingDimensions: '',
+    appVectorToolPrompt: '',
     public: {
       appVersion: '',
       appApiOrgCode: '',

package.json

@@ -42,9 +42,11 @@
     "@ckpack/vue-color": "^1.5.0",
     "@hocuspocus/provider": "^2.15.0",
     "@intlify/vue-i18n-loader": "^4.2.0",
-    "@langchain/core": "^1.1.17",
+    "@langchain/community": "^1.1.23",
+    "@langchain/core": "^1.1.32",
     "@langchain/langgraph": "^1.1.2",
     "@langchain/openai": "^1.2.3",
+    "@langchain/textsplitters": "^1.0.1",
     "@mdi/font": "^6",
     "@modelcontextprotocol/sdk": "^1.20.2",
     "@nuxt/test-utils": "^3.19.2",
@@ -84,6 +86,7 @@
     "deep-chat": "^2.3.0",
     "es-toolkit": "^1.40.0",
     "highlight.js": "^11",
+    "install": "^0.13.0",
     "langchain": "^1.2.14",
     "leaflet": "^1",
     "leaflet.markercluster": "^1.5.3",
@@ -92,6 +95,8 @@
     "nuxt": "^3.19.3",
     "nuxt-svgo": "4.2.6",
     "openai": "^5.23.0",
+    "pdf-parse": "^1",
+    "pg": "^8.20.0",
     "pinia": "^3.0.3",
     "prosemirror-markdown": "^1.13.1",
     "prosemirror-model": "^1.24.1",

src/app/useAdminPagesRoutes.ts

@@ -130,6 +130,12 @@ export default function useAdminPagesRoutes() {
           component: () => import('../pages/AdminPortalPageV2/Tabs/TermsAdminTab.vue'),
           meta: { checkAccessRequestEnabled: true },
         },
+        {
+          path: 'vector-store',
+          name: 'VectorStoreAdminTab',
+          component: () => import('../pages/AdminPortalPageV2/Tabs/VectorStoreAdminTab.vue'),
+          meta: { checkAccessRequestEnabled: true },
+        },
       ],
       meta: {
         resetScroll: true,

src/components/VectorStoreDocumentList.vue

@@ -0,0 +1,130 @@
+<script setup>
+import useUsersStore from '@/stores/useUsers'
+
+const emit = defineEmits(['show-document', 'delete-document', 'edit-document'])
+
+const usersStore = useUsersStore()
+
+const isAsyncing = ref(false)
+const documentList = ref([])
+
+const refresh = async () => {
+  isAsyncing.value = true
+  let headers = {}
+  const accessToken = usersStore.accessToken // localStorage?.getItem('ACCESS_TOKEN')
+  if (accessToken) headers = { Authorization: `Bearer ${accessToken}` }
+
+  try {
+    const response = await fetch(`/api/vector-store/list`, {
+      headers,
+    })
+    const data = await response.json()
+    console.log(data)
+    documentList.value = data
+  } catch (e) {
+    console.log(e.toString())
+  } finally {
+    isAsyncing.value = false
+    close()
+  }
+}
+defineExpose({ refresh })
+
+refresh()
+</script>
+<template>
+  <div v-if="isAsyncing" class="loader">
+    <LoaderSimple />
+  </div>
+  <p v-else-if="!documentList.length" class="no-document">
+    {{ $t('vector-store.no-document-yet') }}
+  </p>
+  <ul v-else>
+    <li
+      v-for="document in documentList"
+      :key="document.title + '-' + document.chunks"
+      class="document"
+    >
+      <div class="icon">
+        <IconImage name="Article" />
+      </div>
+      <div class="title">
+        {{ document.title }}
+        <span class="chunk-count">({{ document.chunks }} {{ $t('vector-store.chunks') }})</span>
+      </div>
+      <div class="actions">
+        <ContextActionButton
+          action-icon="Eye"
+          secondary
+          no-border
+          @click.prevent="emit('show-document', document.title)"
+        />
+        <ContextActionButton
+          action-icon="Pen"
+          secondary
+          no-border
+          @click.prevent="emit('edit-document', document.title)"
+        />
+        <ContextActionButton
+          action-icon="TrashCanOutline"
+          secondary
+          no-border
+          @click.prevent="emit('delete-document', document.title)"
+        />
+      </div>
+    </li>
+  </ul>
+</template>
+<style lang="scss" scoped>
+.document {
+  display: flex;
+  gap: 1rem;
+  padding: 0.6rem;
+  border: 1px solid $primary;
+  border-radius: 0.6rem;
+  justify-content: stretch;
+  align-items: center;
+}
+
+.document ~ .document {
+  margin-top: 1rem;
+}
+
+.icon,
+.actions {
+  flex-grow: 0;
+  flex-shrink: 0;
+}
+
+.title {
+  flex-grow: 1;
+  font-size: 1.2rem;
+  color: $primary-dark;
+}
+
+.chunk-count {
+  font-weight: normal;
+  color: $light-gray;
+}
+
+.icon svg {
+  width: 2rem;
+  fill: $light-gray;
+}
+
+.actions {
+  display: flex;
+  gap: 0.6rem;
+}
+
+.loader {
+  display: flex;
+  justify-content: center;
+  padding: 2rem;
+}
+
+.no-document {
+  text-align: center;
+  font-style: italic;
+}
+</style>

src/components/VectorStoreDocumentShow.vue

@@ -0,0 +1,72 @@
+<script setup>
+import useUsersStore from '@/stores/useUsers'
+
+const usersStore = useUsersStore()
+
+const props = defineProps({ documentTitle: { type: String, required: true } })
+const emit = defineEmits(['close'])
+
+const isAsyncing = ref(false)
+const chunkList = ref([])
+
+let headers = {}
+const accessToken = usersStore.accessToken // localStorage?.getItem('ACCESS_TOKEN')
+if (accessToken) headers = { Authorization: `Bearer ${accessToken}` }
+
+const close = () => emit('close')
+
+const query = new URLSearchParams()
+query.set('title', props.documentTitle)
+
+try {
+  const response = await fetch(`/api/vector-store/get?${query.toString()}`, {
+    headers,
+  })
+  const data = await response.json()
+  console.log(data)
+  chunkList.value = data
+} catch (e) {
+  console.log(e.toString())
+} finally {
+  isAsyncing.value = false
+  close()
+}
+</script>
+<template>
+  <ConfirmModal
+    :title="documentTitle"
+    no-second-button
+    cancel-button-label="common.close"
+    @cancel="close"
+    @confirm="close"
+  >
+    <ul>
+      <div v-for="(chunk, i) in chunkList" :key="i" class="chunk">
+        <h4 class="chunk-header">
+          Chunk {{ i + 1 }}/{{ chunkList.length }} - Page {{ chunk.metadata.loc.pageNumber }}, Line
+          {{ chunk.metadata.loc.lines.from }}-{{ chunk.metadata.loc.lines.to }}
+        </h4>
+        <div>
+          {{ chunk.content }}
+        </div>
+      </div>
+    </ul>
+  </ConfirmModal>
+</template>
+<style lang="scss" scoped>
+.chunk {
+  padding-top: 1rem;
+  padding-bottom: 1rem;
+}
+
+.chunk ~ .chunk {
+  border-top: 1px solid $light-gray;
+}
+
+.chunk-header {
+  text-align: center;
+  color: $light-gray;
+  font-style: italic;
+  font-weight: normal;
+}
+</style>