diff --git a/README.md b/README.md index 5653bc5a2..45e2897ff 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ All contributing instructions are in [CONTRIBUTING](CONTRIBUTING.md). - [data-wrapper](./services/data-wrapper) [![Docker Pulls](https://img.shields.io/docker/pulls/cnrsinist/ws-data-wrapper.svg)](https://hub.docker.com/r/cnrsinist/ws-data-wrapper/) - [diseases-ner](./services/diseases-ner) [![Docker Pulls](https://img.shields.io/docker/pulls/cnrsinist/ws-diseases-ner.svg)](https://hub.docker.com/r/cnrsinist/ws-diseases-ner/) - [domains-classifier](./services/domains-classifier) [![Docker Pulls](https://img.shields.io/docker/pulls/cnrsinist/ws-domains-classifier.svg)](https://hub.docker.com/r/cnrsinist/ws-domains-classifier/) +- [embeddings-computer](./services/embeddings-computer) [![Docker Pulls](https://img.shields.io/docker/pulls/cnrsinist/ws-embeddings-computer.svg)](https://hub.docker.com/r/cnrsinist/ws-embeddings-computer/) - [funder-ner](./services/funder-ner) [![Docker Pulls](https://img.shields.io/docker/pulls/cnrsinist/ws-funder-ner.svg)](https://hub.docker.com/r/cnrsinist/ws-funder-ner/) - [hal-classifier](./services/hal-classifier) [![Docker Pulls](https://img.shields.io/docker/pulls/cnrsinist/ws-hal-classifier.svg)](https://hub.docker.com/r/cnrsinist/ws-hal-classifier/) - [irc3-species](./services/irc3-species) [![Docker Pulls](https://img.shields.io/docker/pulls/cnrsinist/ws-irc3-species.svg)](https://hub.docker.com/r/cnrsinist/ws-irc3-species/) diff --git a/package.json b/package.json index 1ee4b5f16..fd9ec0e55 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "services/data-wrapper", "services/diseases-ner", "services/domains-classifier", + "services/embeddings-computer", "services/funder-ner", "services/hal-classifier", "services/irc3-species", diff --git a/services/embeddings-computer/.dockerignore b/services/embeddings-computer/.dockerignore new file mode 100644 index 000000000..03693b9a9 --- /dev/null +++ b/services/embeddings-computer/.dockerignore @@ -0,0 +1,9 @@ +# Ignore all files by default +* + +# White list only the required files +!config.json +!v1 +!swagger.json +!download-model.sh +!embed-local.mjs diff --git a/services/embeddings-computer/Dockerfile b/services/embeddings-computer/Dockerfile new file mode 100644 index 000000000..251fc4629 --- /dev/null +++ b/services/embeddings-computer/Dockerfile @@ -0,0 +1,20 @@ +# syntax=docker/dockerfile:1.2 +FROM cnrsinist/ezs-python-server:py3.9-no24-1.0.13 + +WORKDIR /app +# Install all python dependencies +# RUN pip install --no-cache-dir \ +# unidecode==1.3.7 + +# Install all node dependencies +RUN npm install --omit=dev \ + @xenova/transformers@2.17.2 && \ + npm cache clean --force + +WORKDIR /app/public +# Declare files to copy in .dockerignore +COPY --chown=daemon:daemon . /app/public/ +COPY --chown=daemon:daemon ./config.json /app/config.json + +RUN ./download-model.sh && \ + chmod +x ./embed-local.mjs diff --git a/services/embeddings-computer/README.md b/services/embeddings-computer/README.md new file mode 100644 index 000000000..fc1c8c440 --- /dev/null +++ b/services/embeddings-computer/README.md @@ -0,0 +1,5 @@ +# ws-embeddings-computer@0.1.0 + +Vectorisation de texte + +Des services de génération de vecteurs numériques représentant du texte (plongements lexicaux, ou embeddings en anglais). diff --git a/services/embeddings-computer/config.json b/services/embeddings-computer/config.json new file mode 100644 index 000000000..626376699 --- /dev/null +++ b/services/embeddings-computer/config.json @@ -0,0 +1,14 @@ +{ + "environnement": { + "EZS_TITLE": "Vectorisation de texte", + "EZS_DESCRIPTION": "Des services de génération de vecteurs numériques représentant du texte (plongements lexicaux, ou embeddings en anglais).", + "EZS_METRICS": true, + "EZS_CONCURRENCY": 2, + "EZS_CONTINUE_DELAY": 60, + "EZS_NSHARDS": 32, + "EZS_CACHE": true, + "EZS_VERBOSE": false, + "NODE_OPTIONS": "--max_old_space_size=1024", + "NODE_ENV": "production" + } +} \ No newline at end of file diff --git a/services/embeddings-computer/download-model.sh b/services/embeddings-computer/download-model.sh new file mode 100755 index 000000000..2adaf8fb6 --- /dev/null +++ b/services/embeddings-computer/download-model.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail + +MODEL_ID="Xenova/all-MiniLM-L6-v2" +REVISION="main" # ou un hash de commit si tu veux figer la version +BASE_URL="https://huggingface.co/${MODEL_ID}/resolve/${REVISION}" +OUT_DIR="${1:-models/${MODEL_ID}}" + +echo "Downloading ${MODEL_ID} into ${OUT_DIR}" + +mkdir -p "${OUT_DIR}" + +# Liste des fichiers nécessaires pour transformers.js +files=( + "config.json" + "tokenizer_config.json" + "tokenizer.json" + "special_tokens_map.json" + "vocab.txt" + "onnx/model_quantized.onnx" +) + +for file in "${files[@]}"; do + url="${BASE_URL}/${file}?download=true" + dest="${OUT_DIR}/${file}" + dest_dir="$(dirname "${dest}")" + + mkdir -p "${dest_dir}" + + echo "-> ${url}" + curl -fL "${url}" -o "${dest}" || { + echo "Échec du téléchargement de ${url}" >&2 + exit 1 + } +done + +echo "Done." diff --git a/services/embeddings-computer/embed-local.mjs b/services/embeddings-computer/embed-local.mjs new file mode 100644 index 000000000..a05682a4c --- /dev/null +++ b/services/embeddings-computer/embed-local.mjs @@ -0,0 +1,69 @@ +#!/usr/bin/env node + +// embeddings.ts +// Pour Node +// @ts-ignore +globalThis.self = globalThis; + +import { env, pipeline } from "@xenova/transformers"; +import * as readline from "node:readline/promises"; +import { stdin as input, stdout as output } from "node:process"; + +// 1) Forcer les modèles locaux uniquement +env.allowRemoteModels = false; + +// 2) Répertoire racine des modèles pré-téléchargés +// => par exemple monté dans le container à /app/public/models/ +env.localModelPath = "models"; + +// 3) Préchargement (optionnel) au démarrage +/** @type {Promise | null} */ +let embedderPromise = null; + +export function initEmbedder() { + if (!embedderPromise) { + embedderPromise = pipeline( + "feature-extraction", + // IMPORTANT : chemin *logique* du modèle à l’intérieur de models/ + // Si tu as models/Xenova/all-MiniLM-L6-v2, tu mets exactement : + "Xenova/all-MiniLM-L6-v2" + ); + } + return embedderPromise; +} + +/** + * @param texts {string[]} + * @return {Promise} + */ +export async function embedTexts(texts) { + const extractor = await initEmbedder(); + const output = await extractor(texts, { + pooling: "mean", + normalize: true, + }); + + /** @type {Float32Array} */ + const data = output.data; + const [batchSize, dim] = output.dims; + /** @type {number[][]} */ + const vectors = []; + for (let i = 0; i < batchSize; i++) { + const start = i * dim; + const end = start + dim; + vectors.push(Array.from(data.subarray(start, end))); + } + return vectors; +} + + +/////////////////// +const rl = readline.createInterface({ input, output }); +for await (const line of rl) { + const json = JSON.parse(line); + const { value = [] } = json; + const texts = Array.isArray(value) ? value : [value]; + const vectors = await embedTexts(texts); + json.value = vectors; + console.log(JSON.stringify(json)); +} diff --git a/services/embeddings-computer/examples.http b/services/embeddings-computer/examples.http new file mode 100644 index 000000000..1461c1a4c --- /dev/null +++ b/services/embeddings-computer/examples.http @@ -0,0 +1,17 @@ +# These examples can be used directly in VSCode, using HTTPYac extension (anweber.vscode-httpyac) +# They are important, because used to generate the tests.hurl file. + +# Décommenter/commenter les lignes voulues pour tester localement +@host=http://localhost:31976 +# @host=https://embeddings-computer.services.istex.fr + +### +# @name v1embedSmallText +# Calcule les embeddings de petits textes (sans les découper) +POST {{host}}/v1/embed-small-text?indent=true HTTP/1.1 +Content-Type: application/json + +[ + { "value": "This is a test text for the embeddings model. It should be able to handle multiple texts at once." }, + { "value": "Et maintenant, je mets une phrase en français." } +] diff --git a/services/embeddings-computer/package.json b/services/embeddings-computer/package.json new file mode 100644 index 000000000..d3aa5e43d --- /dev/null +++ b/services/embeddings-computer/package.json @@ -0,0 +1,37 @@ +{ + "private": true, + "name": "ws-embeddings-computer", + "version": "0.1.0", + "description": "Vectorisation de texte", + "repository": { + "type": "git", + "url": "git+https://github.com/Inist-CNRS/web-services.git" + }, + "keywords": [ + "ezmaster" + ], + "author": "François Parmentier ", + "license": "MIT", + "bugs": { + "url": "https://github.com/Inist-CNRS/web-services/issues" + }, + "homepage": "https://github.com/Inist-CNRS/web-services/#readme", + "scripts": { + "version:insert:readme": "sed -i \"s#\\(${npm_package_name}.\\)\\([\\.a-z0-9]\\+\\)#\\1${npm_package_version}#g\" README.md && git add README.md", + "version:insert:swagger": "sed -i \"s/\\\"version\\\": \\\"[0-9]\\+.[0-9]\\+.[0-9]\\+\\\"/\\\"version\\\": \\\"${npm_package_version}\\\"/g\" swagger.json && git add swagger.json", + "version:insert": "npm run version:insert:readme && npm run version:insert:swagger", + "version:commit": "git commit -a -m \"release ${npm_package_name}@${npm_package_version}\"", + "version:tag": "git tag \"${npm_package_name}@${npm_package_version}\" -m \"${npm_package_name}@${npm_package_version}\"", + "version:push": "git push && git push --tags", + "version": "npm run version:insert && npm run version:commit && npm run version:tag", + "postversion": "npm run version:push", + "build:check": "DOCKER_BUILDKIT=1 docker build --check -t cnrsinist/${npm_package_name}:latest . && docker run --rm -i hadolint/hadolint hadolint - < Dockerfile ", + "build:dev": "docker build -t cnrsinist/${npm_package_name}:latest .", + "start:dev": "npm run build:dev && docker run --name dev --rm --detach -p 31976:31976 cnrsinist/${npm_package_name}:latest", + "stop:dev": "docker stop dev", + "build": "docker build -t cnrsinist/${npm_package_name}:${npm_package_version} .", + "start": "docker run --rm -p 31976:31976 cnrsinist/${npm_package_name}:${npm_package_version}", + "publish": "docker push cnrsinist/${npm_package_name}:${npm_package_version}" + }, + "avoid-testing": false +} diff --git a/services/embeddings-computer/swagger.json b/services/embeddings-computer/swagger.json new file mode 100644 index 000000000..49942a4d4 --- /dev/null +++ b/services/embeddings-computer/swagger.json @@ -0,0 +1,33 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "embeddings-computer - Vectorisation de texte", + "description": "Des services de génération de vecteurs numériques représentant du texte (plongements lexicaux, ou embeddings en anglais).", + "version": "0.1.0", + "termsOfService": "https://services.istex.fr/", + "contact": { + "name": "Inist-CNRS", + "url": "https://www.inist.fr/nous-contacter/" + } + }, + "servers": [ + { + "x-comment": "Will be automatically completed by the ezs server." + }, + { + "url": "http://vptdmservices.intra.inist.fr:49225/", + "description": "Latest version for production", + "#DISABLED#x-profil": "Standard" + } + ], + "tags": [ + { + "name": "embeddings-computer", + "description": "Vectorisation de texte", + "externalDocs": { + "description": "Plus de documentation", + "url": "https://github.com/inist-cnrs/web-services/tree/main/services/embeddings-computer" + } + } + ] +} \ No newline at end of file diff --git a/services/embeddings-computer/tests.hurl b/services/embeddings-computer/tests.hurl new file mode 100644 index 000000000..5cdbb4722 --- /dev/null +++ b/services/embeddings-computer/tests.hurl @@ -0,0 +1,789 @@ +POST {{host}}/v1/embed-small-text?indent=true +content-type: application/json +[ + { "value": "This is a test text for the embeddings model. It should be able to handle multiple texts at once." }, + { "value": "Et maintenant, je mets une phrase en français." } +] + + +HTTP 200 +[{ + "value": [ + [ + -0.015692012384533882, + -0.023599108681082726, + -0.010133943520486355, + 0.021150317043066025, + 0.0319644995033741, + 0.07492130249738693, + 0.001932966522872448, + -0.040097933262586594, + 0.0688384622335434, + -0.06307569146156311, + 0.0439220629632473, + 0.021676519885659218, + 0.05503205209970474, + 0.032071951776742935, + -0.026506705209612846, + 0.08111781626939774, + 0.014472819864749908, + -0.005403301678597927, + -0.01227337121963501, + -0.006932467687875032, + 0.06849102675914764, + 0.07255568355321884, + 0.07185832411050797, + -0.026757890358567238, + -0.009572461247444153, + 0.06575275957584381, + -0.13665686547756195, + 0.014525004662573338, + 0.04640303552150726, + -0.0056721824221313, + 0.08588230609893799, + 0.01062851957976818, + -0.00945079792290926, + 0.1591542810201645, + 0.023764362558722496, + 0.00951369572430849, + -0.08402018994092941, + 0.05639362707734108, + -0.018711814656853676, + 0.081741563975811, + 0.04270875081419945, + -0.03734738752245903, + -0.009520547464489937, + 0.023538051173090935, + 0.012376395054161549, + -0.06165079399943352, + -0.10168364644050598, + -0.05184600129723549, + 0.06507549434900284, + 0.03009076789021492, + 0.016285128891468048, + -0.043328989297151566, + -0.0316135473549366, + 0.018147701397538185, + 0.03490937128663063, + 0.018689343705773354, + -0.0542386956512928, + -0.05751914530992508, + 0.006857609376311302, + 0.01592378132045269, + 0.021892352029681206, + 0.025151338428258896, + 0.03986135497689247, + 0.03545354679226875, + 0.12056370079517365, + 0.021633083000779152, + -0.05571507290005684, + 0.03157657012343407, + -0.11215361952781677, + -0.0031850787345319986, + -0.04873456060886383, + -0.00003817736796918325, + 0.001888895407319069, + 0.011993873864412308, + 0.006378361955285072, + 0.03956902399659157, + -0.05048573762178421, + -0.10444788634777069, + 0.06864552199840546, + -0.04895978420972824, + -0.026516566053032875, + 0.0016847059596329927, + 0.036045342683792114, + -0.04687892645597458, + 0.06619443744421005, + 0.027965055778622627, + 0.01857483573257923, + -0.06852974742650986, + -0.018779465928673744, + 0.019909391179680824, + 0.0076179588213562965, + -0.05409418046474457, + 0.12330261617898941, + 0.08187174797058105, + -0.11601738631725311, + 0.024498669430613518, + -0.044729430228471756, + 0.02867734245955944, + -0.001158823724836111, + 0.027438683435320854, + -0.011490427888929844, + -0.021521687507629395, + 0.13034449517726898, + 0.0374046191573143, + -0.08418681472539902, + -0.16237840056419373, + 0.007899018004536629, + -0.03843056783080101, + -0.0480625219643116, + -0.11193689703941345, + -0.00017166018369607627, + -0.02265767939388752, + -0.06189427897334099, + -0.0008530904888175428, + -0.05529610440135002, + -0.03938348591327667, + -0.015418276190757751, + 0.019803764298558235, + 0.08721799403429031, + 0.08286018669605255, + -0.020594406872987747, + -0.019772568717598915, + -0.026066141203045845, + -0.036918602883815765, + -0.008898780681192875, + -0.06347330659627914, + 0.050323594361543655, + -4.431690385239316e-34, + 0.0196453295648098, + -0.010675586760044098, + -0.014931196346879005, + 0.09976068139076233, + 0.05761028081178665, + 0.042098235338926315, + -0.12857551872730255, + 0.08945972472429276, + -0.11178305745124817, + -0.06989535689353943, + -0.006482436787337065, + -0.023389365524053574, + 0.027751965448260307, + 0.07912565767765045, + -0.07576091587543488, + -0.015400492586195469, + -0.01822091080248356, + 0.05856692045927048, + 0.05241616070270538, + 0.017656581476330757, + -0.03164387121796608, + 0.00954439491033554, + -0.0007988346042111516, + -0.06893044710159302, + -0.06486772745847702, + 0.04903324320912361, + 0.08987646549940109, + 0.004569139797240496, + -0.044687237590551376, + 0.016851123422384262, + -0.05746123194694519, + 0.010620403103530407, + -0.02587035484611988, + 0.022841045632958412, + 0.02949378453195095, + 0.025031181052327156, + 0.03390727937221527, + 0.02407781220972538, + -0.03904886916279793, + -0.0024614157155156136, + -0.06907342374324799, + -0.050389401614665985, + 0.03913890942931175, + -0.08203703165054321, + -0.037920400500297546, + 0.020786304026842117, + -0.058215174823999405, + 0.03267709165811539, + -0.029144711792469025, + -0.02851570025086403, + 0.0069709778763353825, + 0.05126921832561493, + 0.02402343600988388, + -0.040213048458099365, + 0.09665659815073013, + -0.038193877786397934, + -0.04368320107460022, + 0.020241929218173027, + 0.03380274400115013, + 0.007923675701022148, + -0.07080143690109253, + -0.011667812243103981, + -0.00656502041965723, + 0.010184673592448235, + 0.05125416815280914, + 0.027547702193260193, + -0.03299592435359955, + -0.06090677157044411, + 0.11758647859096527, + 0.02379545383155346, + -0.005605998449027538, + 0.013110984116792679, + -0.01711043156683445, + -0.011726703494787216, + -0.08150805532932281, + -0.04596826806664467, + -0.03950793668627739, + -0.013513350859284401, + -0.013690656051039696, + 0.047067396342754364, + -0.04644995182752609, + -0.11274608969688416, + 0.03818371891975403, + -0.0171826109290123, + -0.037194643169641495, + -0.006445438601076603, + 0.009449632838368416, + -0.14140264689922333, + 0.00016519658674951643, + -0.03314528986811638, + -0.0014568585902452469, + 0.02411671355366707, + -0.03417500481009483, + -0.08372879773378372, + 0.10478942096233368, + 1.386706464052941e-34, + 0.047704197466373444, + 0.023848017677664757, + -0.08751589804887772, + 0.05527852475643158, + 0.11321946978569031, + 0.013065753504633904, + 0.03627217188477516, + 0.10775387287139893, + -0.028165295720100403, + 0.026407010853290558, + 0.04213159158825874, + -0.00047440899652428925, + -0.001064735697582364, + -0.08696269243955612, + -0.02281654253602028, + 0.020983707159757614, + 0.012584476731717587, + -0.0501236729323864, + 0.03733519837260246, + 0.029788827523589134, + -0.033254995942115784, + 0.0018743773689493537, + -0.018176216632127762, + 0.03546600416302681, + 0.03862627595663071, + 0.05462323874235153, + 0.019208688288927078, + -0.006002361420542002, + -0.012850416824221611, + 0.0041762953624129295, + 0.009390093386173248, + -0.0467316135764122, + 0.01535837072879076, + 0.03893163427710533, + -0.01426611002534628, + -0.004016623832285404, + 0.08799943327903748, + -0.02778913639485836, + -0.0007621030672453344, + 0.022479845210909843, + 0.11097020655870438, + 0.02904248796403408, + -0.03562140837311745, + 0.0006608808762393892, + -0.02930385060608387, + 0.019754070788621902, + -0.03629419580101967, + -0.032965287566185, + 0.01871495321393013, + -0.010230325162410736, + -0.05963624641299248, + -0.07638727873563766, + -0.05554071068763733, + -0.0095668975263834, + -0.04069529473781586, + -0.06902948021888733, + -0.017505278810858727, + 0.012385445646941662, + 0.0042882696725428104, + -0.046801984310150146, + -0.013892081566154957, + 0.008337668143212795, + 0.035041555762290955, + 0.006753561552613974, + 0.07213426381349564, + -0.09785440564155579, + 0.00979310180991888, + -0.032136835157871246, + -0.06073647364974022, + 0.001727016642689705, + 0.03651667758822441, + -0.024239525198936462, + -0.03937993571162224, + 0.015351513400673866, + 0.035771045833826065, + -0.011503131128847599, + 0.04321396350860596, + -0.026982899755239487, + -0.035384632647037506, + -0.014156260527670383, + 0.018175171688199043, + -0.05643153563141823, + 0.037090808153152466, + 0.07836572080850601, + 0.013319365680217743, + -0.0009518765145912766, + 0.0019298889674246311, + 0.13136325776576996, + 0.005930537357926369, + 0.04750824719667435, + -0.020097341388463974, + 0.07116672396659851, + 0.013600204139947891, + 0.10299242287874222, + -0.005229951348155737, + -2.5535744185845033e-8, + -0.03748556599020958, + -0.0623357892036438, + -0.05104441940784454, + -0.05686100572347641, + -0.09832031279802322, + -0.02107296511530876, + 0.01289756502956152, + -0.04788001999258995, + -0.018117915838956833, + -0.06603793799877167, + 0.031434834003448486, + 0.04669402167201042, + -0.05137191712856293, + -0.028809290379285812, + 0.01688493974506855, + -0.03144066035747528, + -0.03957105800509453, + 0.0007141577079892159, + -0.003194682765752077, + 0.018191494047641754, + 0.09267061203718185, + 0.0758533924818039, + 0.08920644223690033, + -0.016524000093340874, + -0.06350535154342651, + 0.0734981819987297, + -0.04815369099378586, + 0.07883019000291824, + -0.021182486787438393, + -0.09088641405105591, + 0.060834307223558426, + 0.041483718901872635, + -0.04507807269692421, + -0.018633240833878517, + -0.014482325874269009, + 0.07992859929800034, + 0.023997806012630463, + -0.042415112257003784, + 0.0733204185962677, + 0.03704649582505226, + 0.022554226219654083, + -0.10602323710918427, + -0.05327562987804413, + 0.01160906720906496, + 0.10073119401931763, + 0.022937456145882607, + -0.008392218500375748, + -0.1011073961853981, + 0.0321081168949604, + -0.0012299625668674707, + -0.018685054033994675, + -0.05005154758691788, + -0.021917151287198067, + 0.048468269407749176, + -0.034595366567373276, + -0.04068552330136299, + 0.10239638388156891, + 0.006715099792927504, + 0.030585747212171555, + -0.012826425954699516, + 0.015131937339901924, + 0.01758791320025921, + 0.02707153931260109, + -0.0021181642077863216 + ] + ] +}, +{ + "value": [ + [ + 0.010211953893303871, + 0.05590079724788666, + 0.04263729974627495, + -0.08609593659639359, + 0.057285208255052567, + 0.019115792587399483, + 0.12924768030643463, + 0.10093618184328079, + 0.011831591837108135, + 0.022432249039411545, + 0.020191658288240433, + -0.008136391639709473, + 0.006941348779946566, + -0.042422350496053696, + -0.05130363628268242, + -0.09766674041748047, + 0.017543470486998558, + 0.10345307737588882, + 0.02374557964503765, + 0.028590906411409378, + 0.03822047635912895, + 0.02114521712064743, + 0.06999050825834274, + 0.02675688825547695, + -0.021391738206148148, + 0.0016581583768129349, + -0.045050449669361115, + 0.026267198845744133, + 0.06955720484256744, + 0.005632730666548014, + -0.013330833055078983, + 0.048672497272491455, + 0.012938299216330051, + 0.008341258391737938, + -0.03783183917403221, + 0.036120157688856125, + -0.06577684730291367, + -0.08845817297697067, + 0.03330828621983528, + 0.049389615654945374, + -0.10154664516448975, + 0.029719391837716103, + -0.12340203672647476, + -0.02186063677072525, + 0.0005768121918663383, + 0.0732097253203392, + 0.0012837410904467106, + 0.046242810785770416, + -0.02757783979177475, + -0.0016769738867878914, + -0.0773850753903389, + -0.0037032589316368103, + -0.010460522025823593, + -0.010809059254825115, + 0.04701998829841614, + -0.0556214414536953, + 0.05121377483010292, + -0.00967064406722784, + 0.035708874464035034, + -0.018671683967113495, + 0.059123385697603226, + -0.04226771742105484, + -0.025345908477902412, + 0.048205211758613586, + 0.00514278095215559, + -0.05913405492901802, + 0.08306901901960373, + -0.0013927408726885915, + -0.11931021511554718, + 0.059540264308452606, + -0.020433807745575905, + 0.02342962473630905, + 0.03900209814310074, + -0.0033009902108460665, + 0.024794060736894608, + 0.027524497359991074, + -0.03435215726494789, + 0.0052522579208016396, + -0.020386697724461555, + -0.09087787568569183, + 0.014607387594878674, + 0.014118997380137444, + 0.01822602190077305, + -0.033303193747997284, + 0.06435666978359222, + -0.015526925213634968, + 0.049324654042720795, + 0.04888321831822395, + 0.0526755228638649, + -0.040740691125392914, + -0.1238519549369812, + -0.018406091257929802, + 0.01623981073498726, + 0.05919046700000763, + -0.07569754868745804, + -0.016633115708827972, + -0.04029180854558945, + -0.04986163601279259, + -0.02626093104481697, + 0.05260607600212097, + 0.007180855609476566, + 0.004929105751216412, + 0.09857777506113052, + 0.03546253964304924, + -0.03481423109769821, + -0.03280196711421013, + -0.06493056565523148, + -0.08584022521972656, + 0.032977182418107986, + -0.04165571555495262, + -0.07323396950960159, + -0.03426443040370941, + 0.010765260085463524, + -0.05010337755084038, + 0.03434517979621887, + 0.0387297049164772, + 0.06709086149930954, + -0.10215836763381958, + 0.032074760645627975, + -0.010511738248169422, + 0.07541705667972565, + 0.009132196195423603, + -0.05178719386458397, + -0.05061669275164604, + 0.013050409033894539, + -0.05057387799024582, + 0.09264881908893585, + -2.482397398793991e-33, + -0.02265278249979019, + 0.07592768222093582, + 0.05862845852971077, + 0.02150062285363674, + -0.055899813771247864, + -0.031943388283252716, + -0.09238994866609573, + 0.0445973165333271, + -0.018009647727012634, + 0.04151361808180809, + -0.061548929661512375, + 0.10192837566137314, + 0.0037819822318851948, + 0.044613853096961975, + 0.021674221381545067, + -0.019946260377764702, + 0.09505341947078705, + 0.06388871371746063, + 0.12361954897642136, + -0.03265264257788658, + -0.06372860074043274, + 0.05432569980621338, + 0.03995674103498459, + 0.04668945074081421, + 0.04756292700767517, + -0.0998111218214035, + 0.03003273718059063, + -0.039015986025333405, + -0.07606425136327744, + 0.004801236093044281, + -0.004170286003500223, + -0.006178902927786112, + 0.05087900906801224, + 0.05445103347301483, + 0.017460789531469345, + -0.023133063688874245, + -0.007318664807826281, + 0.09113100916147232, + 0.0008738992037251592, + 0.04393532872200012, + 0.03845246136188507, + -0.02351045235991478, + 0.0021854150108993053, + -0.009130161255598068, + 0.029778150841593742, + -0.0037859915755689144, + -0.02759954333305359, + 0.02951708994805813, + 0.021307192742824554, + 0.0005817993078380823, + -0.03889308124780655, + -0.02741655334830284, + 0.04796877130866051, + -0.041947174817323685, + 0.012421279214322567, + 0.09469267725944519, + -0.11701678484678268, + 0.013801606371998787, + -0.06919188797473907, + -0.07194329798221588, + -0.04391472041606903, + -0.05136536434292793, + -0.0025460419710725546, + 0.08159793168306351, + -0.05791806802153587, + -0.004548690281808376, + 0.033050939440727234, + 0.004741559270769358, + 0.025683032348752022, + -0.017765315249562263, + -0.13150596618652344, + 0.040584713220596313, + 0.06748512387275696, + -0.01944557949900627, + -0.0026489170268177986, + 0.07240130752325058, + -0.035613346844911575, + 0.031065382063388824, + -0.013521708548069, + -0.03980730473995209, + -0.0619039312005043, + -0.09944593161344528, + -0.019722966477274895, + -0.03653806447982788, + -0.0219024159014225, + 0.05910244956612587, + 0.03847363963723183, + 0.017616795375943184, + -0.011034456081688404, + 0.041497644037008286, + -0.025738732889294624, + 0.013284941203892231, + -0.03037301078438759, + -0.02899041585624218, + -0.03441959246993065, + -2.4224844721134943e-34, + -0.008710582740604877, + 0.043821338564157486, + -0.08162684738636017, + 0.049490075558423996, + -0.03196847811341286, + 0.05811871588230133, + 0.01084833312779665, + 0.048190925270318985, + -0.08414512127637863, + -0.04901755228638649, + -0.029366273432970047, + -0.11714664101600647, + 0.08897632360458374, + -0.022111061960458755, + -0.06297563761472702, + 0.05861150845885277, + -0.016224170103669167, + 0.024757632985711098, + -0.03154004365205765, + 0.05397745221853256, + 0.009553425945341587, + -0.0892750546336174, + -0.04138794168829918, + -0.06137412413954735, + -0.04956952854990959, + 0.007033999543637037, + -0.016272587701678276, + 0.04117494449019432, + -0.07002639770507812, + -0.04772721603512764, + 0.03554126247763634, + 0.013636697083711624, + -0.04821866750717163, + 0.011098062619566917, + -0.02471034973859787, + 0.10101950168609619, + 0.037174586206674576, + -0.016457777470350266, + -0.04490131139755249, + 0.09627825021743774, + -0.046188533306121826, + 0.051602624356746674, + 0.03267152979969978, + -0.019053611904382706, + 0.03616560250520706, + -0.022682523354887962, + -0.057854779064655304, + -0.096064992249012, + -0.03680470213294029, + -0.03829933702945709, + -0.038103390485048294, + 0.033008452504873276, + 0.00791300367563963, + -0.06483027338981628, + 0.03717212751507759, + 0.01398780569434166, + 0.029617466032505035, + -0.03986704722046852, + -0.11331740021705627, + -0.008277841843664646, + 0.030140172690153122, + 0.07786533981561661, + -0.006471695378422737, + -0.008601276203989983, + 0.0972871407866478, + -0.040692903101444244, + -0.14173679053783417, + 0.024417651817202568, + 0.05050581693649292, + 0.05358608812093735, + 0.0645323246717453, + 0.0023947847075760365, + -0.05547559633851051, + 0.0327918566763401, + -0.14858992397785187, + 0.009411461651325226, + -0.035811156034469604, + -0.008811315521597862, + -0.041001372039318085, + 0.056821417063474655, + -0.0319264754652977, + 0.00534266559407115, + -0.024083320051431656, + -0.013388140127062798, + -0.05306541174650192, + -0.03556394204497337, + -0.0453655980527401, + -0.03163968026638031, + 0.04523839056491852, + -0.06295470893383026, + 0.004213219042867422, + 0.019324664026498795, + -0.0051198392175138, + 0.0013682761928066611, + 0.08710192143917084, + -2.0922588106486728e-8, + -0.008018972352147102, + -0.014521948993206024, + -0.04076200723648071, + -0.035537850111722946, + 0.08206577599048615, + -0.12447647005319595, + -0.019448500126600266, + -0.021465448662638664, + -0.04688676446676254, + 0.018583277240395546, + -0.018877407535910606, + 0.05892346799373627, + -0.044145241379737854, + 0.02138117328286171, + -0.02180454321205616, + 0.09954677522182465, + -0.006020975764840841, + -0.021154023706912994, + 0.024655470624566078, + -0.0176086463034153, + -0.01212289184331894, + 0.010649388656020164, + 0.0073579587042331696, + -0.0826379731297493, + -0.00858239270746708, + -0.0006440654979087412, + 0.007081685587763786, + -0.036166392266750336, + -0.031844448298215866, + -0.06733717024326324, + -0.011997392401099205, + 0.07074916362762451, + -0.02658378705382347, + -0.06877199560403824, + -0.018146153539419174, + 0.08417797088623047, + 0.014696449972689152, + 0.05662623420357704, + 0.028450431302189827, + 0.047739580273628235, + 0.13768304884433746, + 0.07496992498636246, + -0.051314692944288254, + -0.016336409375071526, + 0.0326714962720871, + 0.0014536010567098856, + 0.02535213530063629, + 0.03485077992081642, + 0.04305409640073776, + -0.01761631853878498, + -0.00025872423429973423, + 0.06999087333679199, + -0.024332454428076744, + 0.01962215267121792, + 0.03364282473921776, + 0.07560396939516068, + -0.061235133558511734, + 0.009905686601996422, + 0.0028607500717043877, + -0.033605143427848816, + 0.008457325398921967, + 0.13417485356330872, + 0.0236456748098135, + -0.037430424243211746 + ] + ] +}] \ No newline at end of file diff --git a/services/embeddings-computer/v1/embed-small-text.ini b/services/embeddings-computer/v1/embed-small-text.ini new file mode 100644 index 000000000..ed10575d2 --- /dev/null +++ b/services/embeddings-computer/v1/embed-small-text.ini @@ -0,0 +1,38 @@ +# OpenAPI Documentation - JSON format (dot notation) +mimeType = application/json + +post.operationId = post-v1-embed-small-text +post.requestBody.content.application/json.schema.$ref = #/components/schemas/JSONStream +post.requestBody.required = true +post.responses.default.content.application/json.schema.$ref = #/components/schemas/JSONStream +post.responses.default.description = Embeddings du texte envoyé (tableau de nombres) +post.summary = Calcule les embeddings de petits textes (sans les découper) +post.description = Calcule les embeddings de petits textes (sans les découper). Utilise le modèle [`all-MiniLM-L6-v2`](https://huggingface.co/Xenova/all-MiniLM-L6-v2). +post.tags.0: embeddings-computer +post.parameters.0.description = Indenter le JSON résultant +post.parameters.0.in = query +post.parameters.0.name = indent +post.parameters.0.required = false +post.parameters.0.schema.type = boolean + +# Examples +post.requestBody.content.application/json.example.0.value = This is a test text for the embeddings model. It should be able to handle multiple texts at once. +post.requestBody.content.application/json.example.2.value = Et maintenant, je mets une phrase en français. + +[use] +plugin = @ezs/basics +plugin = @ezs/spawn + +[JSONParse] +separator = * + +[expand] +path = value +size = 100 + +[expand/exec] +# command should be executable ! +command = ./embed-local.mjs + +[dump] +indent = env('indent', false)