[embeddings-computer]: Create

README.md

@@ -45,6 +45,7 @@ All contributing instructions are in [CONTRIBUTING](CONTRIBUTING.md).
 - [data-wrapper](./services/data-wrapper) [![Docker Pulls](https://img.shields.io/docker/pulls/cnrsinist/ws-data-wrapper.svg)](https://hub.docker.com/r/cnrsinist/ws-data-wrapper/)
 - [diseases-ner](./services/diseases-ner) [![Docker Pulls](https://img.shields.io/docker/pulls/cnrsinist/ws-diseases-ner.svg)](https://hub.docker.com/r/cnrsinist/ws-diseases-ner/)
 - [domains-classifier](./services/domains-classifier) [![Docker Pulls](https://img.shields.io/docker/pulls/cnrsinist/ws-domains-classifier.svg)](https://hub.docker.com/r/cnrsinist/ws-domains-classifier/)
+- [embeddings-computer](./services/embeddings-computer) [![Docker Pulls](https://img.shields.io/docker/pulls/cnrsinist/ws-embeddings-computer.svg)](https://hub.docker.com/r/cnrsinist/ws-embeddings-computer/)
 - [funder-ner](./services/funder-ner) [![Docker Pulls](https://img.shields.io/docker/pulls/cnrsinist/ws-funder-ner.svg)](https://hub.docker.com/r/cnrsinist/ws-funder-ner/)
 - [hal-classifier](./services/hal-classifier) [![Docker Pulls](https://img.shields.io/docker/pulls/cnrsinist/ws-hal-classifier.svg)](https://hub.docker.com/r/cnrsinist/ws-hal-classifier/)
 - [irc3-species](./services/irc3-species) [![Docker Pulls](https://img.shields.io/docker/pulls/cnrsinist/ws-irc3-species.svg)](https://hub.docker.com/r/cnrsinist/ws-irc3-species/)

package.json

@@ -63,6 +63,7 @@
     "services/data-wrapper",
     "services/diseases-ner",
     "services/domains-classifier",
+    "services/embeddings-computer",
     "services/funder-ner",
     "services/hal-classifier",
     "services/irc3-species",

services/embeddings-computer/.dockerignore

@@ -0,0 +1,9 @@
+# Ignore all files by default
+*
+
+# White list only the required files
+!config.json
+!v1
+!swagger.json
+!download-model.sh
+!embed-local.mjs

services/embeddings-computer/Dockerfile

@@ -0,0 +1,20 @@
+# syntax=docker/dockerfile:1.2
+FROM cnrsinist/ezs-python-server:py3.9-no24-1.0.13
+
+WORKDIR /app
+# Install all python dependencies
+# RUN pip install --no-cache-dir \
+#     unidecode==1.3.7
+
+# Install all node dependencies
+RUN npm install --omit=dev \
+    @xenova/transformers@2.17.2 && \
+    npm cache clean --force
+
+WORKDIR /app/public
+# Declare files to copy in .dockerignore
+COPY --chown=daemon:daemon . /app/public/
+COPY --chown=daemon:daemon ./config.json /app/config.json
+
+RUN ./download-model.sh && \
+    chmod +x ./embed-local.mjs

services/embeddings-computer/README.md

@@ -0,0 +1,5 @@
+# ws-embeddings-computer@0.1.0
+
+Vectorisation de texte
+
+Des services de génération de vecteurs numériques représentant du texte (plongements lexicaux, ou embeddings en anglais).

services/embeddings-computer/config.json

@@ -0,0 +1,14 @@
+{
+    "environnement": {
+        "EZS_TITLE": "Vectorisation de texte",
+        "EZS_DESCRIPTION": "Des services de génération de vecteurs numériques représentant du texte (plongements lexicaux, ou embeddings en anglais).",
+        "EZS_METRICS": true,
+        "EZS_CONCURRENCY": 2,
+        "EZS_CONTINUE_DELAY": 60,
+        "EZS_NSHARDS": 32,
+        "EZS_CACHE": true,
+        "EZS_VERBOSE": false,
+        "NODE_OPTIONS": "--max_old_space_size=1024",
+        "NODE_ENV": "production"
+    }
+}

services/embeddings-computer/download-model.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MODEL_ID="Xenova/all-MiniLM-L6-v2"
+REVISION="main"  # ou un hash de commit si tu veux figer la version
+BASE_URL="https://huggingface.co/${MODEL_ID}/resolve/${REVISION}"
+OUT_DIR="${1:-models/${MODEL_ID}}"
+
+echo "Downloading ${MODEL_ID} into ${OUT_DIR}"
+
+mkdir -p "${OUT_DIR}"
+
+# Liste des fichiers nécessaires pour transformers.js
+files=(
+  "config.json"
+  "tokenizer_config.json"
+  "tokenizer.json"
+  "special_tokens_map.json"
+  "vocab.txt"
+  "onnx/model_quantized.onnx"
+)
+
+for file in "${files[@]}"; do
+  url="${BASE_URL}/${file}?download=true"
+  dest="${OUT_DIR}/${file}"
+  dest_dir="$(dirname "${dest}")"
+
+  mkdir -p "${dest_dir}"
+
+  echo "-> ${url}"
+  curl -fL "${url}" -o "${dest}" || {
+    echo "Échec du téléchargement de ${url}" >&2
+    exit 1
+  }
+done
+
+echo "Done."

services/embeddings-computer/embed-local.mjs

@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+
+// embeddings.ts
+// Pour Node
+// @ts-ignore
+globalThis.self = globalThis;
+
+import { env, pipeline } from "@xenova/transformers";
+import * as readline from "node:readline/promises";
+import { stdin as input, stdout as output } from "node:process";
+
+// 1) Forcer les modèles locaux uniquement
+env.allowRemoteModels = false;
+
+// 2) Répertoire racine des modèles pré-téléchargés
+//   => par exemple monté dans le container à /app/public/models/
+env.localModelPath = "models";
+
+// 3) Préchargement (optionnel) au démarrage
+/** @type {Promise<any> | null} */
+let embedderPromise = null;
+
+export function initEmbedder() {
+    if (!embedderPromise) {
+        embedderPromise = pipeline(
+            "feature-extraction",
+            // IMPORTANT : chemin *logique* du modèle à l’intérieur de models/
+            // Si tu as models/Xenova/all-MiniLM-L6-v2, tu mets exactement :
+            "Xenova/all-MiniLM-L6-v2"
+        );
+    }
+    return embedderPromise;
+}
+
+/**
+ * @param texts {string[]}
+ * @return {Promise<number[][]>}
+ */
+export async function embedTexts(texts) {
+    const extractor = await initEmbedder();
+    const output = await extractor(texts, {
+        pooling: "mean",
+        normalize: true,
+    });
+
+    /** @type {Float32Array} */
+    const data = output.data;
+    const [batchSize, dim] = output.dims;
+    /** @type {number[][]} */
+    const vectors = [];
+    for (let i = 0; i < batchSize; i++) {
+        const start = i * dim;
+        const end = start + dim;
+        vectors.push(Array.from(data.subarray(start, end)));
+    }
+    return vectors;
+}
+
+
+///////////////////
+const rl = readline.createInterface({ input, output });
+for await (const line of rl) {
+    const json = JSON.parse(line);
+    const { value = [] } = json;
+    const texts = Array.isArray(value) ? value : [value];
+    const vectors = await embedTexts(texts);
+    json.value = vectors;
+    console.log(JSON.stringify(json));
+}

services/embeddings-computer/examples.http

@@ -0,0 +1,17 @@
+# These examples can be used directly in VSCode, using HTTPYac extension (anweber.vscode-httpyac)
+# They are important, because used to generate the tests.hurl file.
+
+# Décommenter/commenter les lignes voulues pour tester localement
+@host=http://localhost:31976
+# @host=https://embeddings-computer.services.istex.fr
+
+###
+# @name v1embedSmallText
+# Calcule les embeddings de petits textes (sans les découper)
+POST {{host}}/v1/embed-small-text?indent=true HTTP/1.1
+Content-Type: application/json
+
+[
+	{ "value": "This is a test text for the embeddings model. It should be able to handle multiple texts at once." },
+	{ "value": "Et maintenant, je mets une phrase en français." }
+]

services/embeddings-computer/package.json

@@ -0,0 +1,37 @@
+{
+  "private": true,
+  "name": "ws-embeddings-computer",
+  "version": "0.1.0",
+  "description": "Vectorisation de texte",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Inist-CNRS/web-services.git"
+  },
+  "keywords": [
+    "ezmaster"
+  ],
+  "author": "François Parmentier <francois.parmentier@gmail.com>",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/Inist-CNRS/web-services/issues"
+  },
+  "homepage": "https://github.com/Inist-CNRS/web-services/#readme",
+  "scripts": {
+    "version:insert:readme": "sed -i \"s#\\(${npm_package_name}.\\)\\([\\.a-z0-9]\\+\\)#\\1${npm_package_version}#g\" README.md && git add README.md",
+    "version:insert:swagger": "sed -i \"s/\\\"version\\\": \\\"[0-9]\\+.[0-9]\\+.[0-9]\\+\\\"/\\\"version\\\": \\\"${npm_package_version}\\\"/g\" swagger.json && git add swagger.json",
+    "version:insert": "npm run version:insert:readme && npm run version:insert:swagger",
+    "version:commit": "git commit -a -m \"release ${npm_package_name}@${npm_package_version}\"",
+    "version:tag": "git tag \"${npm_package_name}@${npm_package_version}\" -m \"${npm_package_name}@${npm_package_version}\"",
+    "version:push": "git push && git push --tags",
+    "version": "npm run version:insert && npm run version:commit && npm run version:tag",
+    "postversion": "npm run version:push",
+    "build:check": "DOCKER_BUILDKIT=1 docker build --check -t cnrsinist/${npm_package_name}:latest . && docker run --rm -i hadolint/hadolint hadolint - < Dockerfile ",
+    "build:dev": "docker build -t cnrsinist/${npm_package_name}:latest .",
+    "start:dev": "npm run build:dev && docker run --name dev --rm --detach -p 31976:31976 cnrsinist/${npm_package_name}:latest",
+    "stop:dev": "docker stop dev",
+    "build": "docker build -t cnrsinist/${npm_package_name}:${npm_package_version} .",
+    "start": "docker run --rm -p 31976:31976 cnrsinist/${npm_package_name}:${npm_package_version}",
+    "publish": "docker push cnrsinist/${npm_package_name}:${npm_package_version}"
+  },
+  "avoid-testing": false
+}

services/embeddings-computer/swagger.json

@@ -0,0 +1,33 @@
+{
+    "openapi": "3.0.0",
+    "info": {
+        "title": "embeddings-computer - Vectorisation de texte",
+        "description": "Des services de génération de vecteurs numériques représentant du texte (plongements lexicaux, ou embeddings en anglais).",
+        "version": "0.1.0",
+        "termsOfService": "https://services.istex.fr/",
+        "contact": {
+            "name": "Inist-CNRS",
+            "url": "https://www.inist.fr/nous-contacter/"
+        }
+    },
+    "servers": [
+        {
+            "x-comment": "Will be automatically completed by the ezs server."
+        },
+        {
+            "url": "http://vptdmservices.intra.inist.fr:49225/",
+            "description": "Latest version for production",
+            "#DISABLED#x-profil": "Standard"
+        }
+    ],
+    "tags": [
+        {
+            "name": "embeddings-computer",
+            "description": "Vectorisation de texte",
+            "externalDocs": {
+                "description": "Plus de documentation",
+                "url": "https://github.com/inist-cnrs/web-services/tree/main/services/embeddings-computer"
+            }
+        }
+    ]
+}