From 63e793a58b7f191c20a73fcc8f4a8f4d0e4f00f7 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Mon, 3 Aug 2026 16:02:55 +0200 Subject: [PATCH 01/59] =?UTF-8?q?=E2=99=BB=EF=B8=8F(collaboration)=20switc?= =?UTF-8?q?h=20collaboration=20server=20from=20hocuspocus=20to=20yhub?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Kevin Jahns --- .gitignore | 3 + CHANGELOG.md | 8 + Makefile | 15 +- compose-e2e.yml | 15 +- compose.yml | 67 +- docker/files/yhub/initdb/01-yhub.sql | 14 + documentation/system-requirements.md | 2 +- env.d/development/common | 4 +- env.d/development/common.e2e | 1 - env.d/production.dist/common | 1 + .../management/commands/clean_document.py | 8 + .../core/services/collaboration_services.py | 101 +-- .../documents/test_api_documents_can_edit.py | 123 +-- .../test_api_documents_content_update.py | 121 +-- .../documents/test_api_documents_update.py | 262 +------ .../test_external_api_documents_accesses.py | 44 +- ...ternal_api_documents_link_configuration.py | 2 - .../test_services_collaboration_services.py | 342 +------- src/backend/core/tests/test_tasks_access.py | 17 - src/backend/impress/settings.py | 4 + src/frontend/apps/e2e/.env | 2 +- src/frontend/apps/e2e/.env.example | 2 +- .../e2e/__tests__/app-impress/config.spec.ts | 4 +- .../app-impress/doc-collaboration.spec.ts | 75 +- src/frontend/apps/impress/package.json | 2 +- .../core/config/hooks/useCollaborationUrl.tsx | 11 +- .../docs/doc-comments/hooks/useComments.ts | 4 +- .../doc-editor/__tests__/DocEditor.spec.tsx | 4 +- .../doc-editor/components/BlockNoteEditor.tsx | 11 +- .../docs/doc-editor/components/DocEditor.tsx | 12 +- .../docs/doc-editor/hook/useCollaboration.tsx | 7 +- .../docs/doc-editor/hook/useSaveDoc.tsx | 18 +- .../doc-management/api/useDuplicateDoc.tsx | 6 +- .../docs/doc-management/api/useUpdateDoc.tsx | 13 +- .../stores/useProviderStore.tsx | 167 ++-- .../components/ModalConfirmationVersion.tsx | 6 +- .../right-panel/components/RightPanel.tsx | 3 +- .../impress/src/stores/useBroadcastStore.tsx | 31 +- src/frontend/package.json | 1 + .../__tests__/collaborationBackend.test.ts | 66 -- .../collaborationResetConnections.test.ts | 63 -- .../getDocumentConnectionInfoHandler.test.ts | 275 ------- .../y-provider/__tests__/hocuspocusWS.test.ts | 388 --------- src/frontend/servers/y-provider/package.json | 11 +- .../src/api/collaborationBackend.ts | 88 --- src/frontend/servers/y-provider/src/env.ts | 2 - .../collaborationResetConnectionsHandler.ts | 48 -- .../src/handlers/collaborationWSHandler.ts | 13 - .../getDocumentConnectionInfoHandler.ts | 48 -- .../servers/y-provider/src/handlers/index.ts | 3 - .../servers/y-provider/src/middlewares.ts | 28 - src/frontend/servers/y-provider/src/routes.ts | 3 - .../y-provider/src/servers/appServer.ts | 39 +- .../src/servers/hocuspocusServer.ts | 95 --- .../servers/y-provider/src/servers/index.ts | 1 - src/frontend/yarn.lock | 170 +--- src/yhub-server/Dockerfile | 13 + src/yhub-server/package-lock.json | 738 ++++++++++++++++++ src/yhub-server/package.json | 14 + src/yhub-server/server.js | 98 +++ 60 files changed, 1214 insertions(+), 2523 deletions(-) create mode 100644 docker/files/yhub/initdb/01-yhub.sql delete mode 100644 src/frontend/servers/y-provider/__tests__/collaborationBackend.test.ts delete mode 100644 src/frontend/servers/y-provider/__tests__/collaborationResetConnections.test.ts delete mode 100644 src/frontend/servers/y-provider/__tests__/getDocumentConnectionInfoHandler.test.ts delete mode 100644 src/frontend/servers/y-provider/__tests__/hocuspocusWS.test.ts delete mode 100644 src/frontend/servers/y-provider/src/api/collaborationBackend.ts delete mode 100644 src/frontend/servers/y-provider/src/handlers/collaborationResetConnectionsHandler.ts delete mode 100644 src/frontend/servers/y-provider/src/handlers/collaborationWSHandler.ts delete mode 100644 src/frontend/servers/y-provider/src/handlers/getDocumentConnectionInfoHandler.ts delete mode 100644 src/frontend/servers/y-provider/src/servers/hocuspocusServer.ts create mode 100644 src/yhub-server/Dockerfile create mode 100644 src/yhub-server/package-lock.json create mode 100644 src/yhub-server/package.json create mode 100644 src/yhub-server/server.js diff --git a/.gitignore b/.gitignore index 61a7f1ce22..08dfcc4b13 100644 --- a/.gitignore +++ b/.gitignore @@ -47,6 +47,9 @@ env.d/terraform compose.override.yml docker/auth/*.local +# yhub server local install +src/yhub-server/node_modules/ + # npm node_modules diff --git a/CHANGELOG.md b/CHANGELOG.md index defbd42f84..e188d43e4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,14 @@ and this project adheres to - 🐛(frontend) export images embedded with a relative url #2573 - 🐛(y-provider) fix sentry init #2579 - 🐛(helm) show the database error while jobs wait for it to be ready #2578 +- ♻️(collaboration) migrate the collaboration server from hocuspocus to yhub: + the dev stack gains dedicated valkey and postgres services for yhub, and + the kick (reset-connections) and get-connections APIs have no yhub + equivalent yet — they are deferred with TODO(yhub) stubs +- 💥(y-provider) the published `lasuite/impress-y-provider` image becomes + converter-only and no longer serves `/collaboration/ws/`; deployments using + the existing helm values lose collaboration until the helm chart routes + collaboration to yhub (follow-up) ## [v5.4.1] - 2026-07-09 diff --git a/Makefile b/Makefile index 54dd50b4d8..fcac8eff2b 100644 --- a/Makefile +++ b/Makefile @@ -190,6 +190,7 @@ bootstrap-e2e: \ build: cache ?= build: ## build the project containers @$(MAKE) build-backend cache=$(cache) + @$(MAKE) build-yhub cache=$(cache) @$(MAKE) build-yjs-provider cache=$(cache) @$(MAKE) build-frontend cache=$(cache) .PHONY: build @@ -199,9 +200,14 @@ build-backend: ## build the app-dev container @$(COMPOSE) build app-dev $(cache) .PHONY: build-backend +build-yhub: cache ?= +build-yhub: ## build the yhub collaboration server container + @$(COMPOSE) build yhub $(cache) +.PHONY: build-yhub + build-yjs-provider: cache ?= build-yjs-provider: ## build the y-provider container - @$(COMPOSE) build y-provider-development $(cache) + @$(COMPOSE) build y-provider-development-converter $(cache) .PHONY: build-yjs-provider build-frontend: cache ?= @@ -212,8 +218,9 @@ build-frontend: ## build the frontend container build-e2e: cache ?= build-e2e: ## build the e2e container @$(MAKE) build-backend cache=$(cache) + @$(MAKE) build-yhub cache=$(cache) @$(COMPOSE_E2E) build frontend $(cache) - @$(COMPOSE_E2E) build y-provider $(cache) + @$(COMPOSE_E2E) build y-provider-converter $(cache) .PHONY: build-e2e nginx-frontend: ## build the nginx-frontend container @@ -232,8 +239,8 @@ run-backend: ## Start only the backend application and all needed services @$(MAKE) create-docker-network @$(COMPOSE) up --force-recreate -d docspec @$(COMPOSE) up --force-recreate -d celery-dev - @$(COMPOSE) up --force-recreate -d y-provider-development @$(COMPOSE) up --force-recreate -d y-provider-development-converter + @$(COMPOSE) up --force-recreate -d yhub @$(COMPOSE) up --force-recreate -d nginx .PHONY: run-backend @@ -246,9 +253,7 @@ run: run-e2e: ## start the e2e server run-e2e: @$(MAKE) run-backend - @$(COMPOSE_E2E) stop y-provider-development @$(COMPOSE_E2E) up --force-recreate -d frontend - @$(COMPOSE_E2E) up --force-recreate -d y-provider @$(COMPOSE_E2E) up --force-recreate -d y-provider-converter .PHONY: run-e2e diff --git a/compose-e2e.yml b/compose-e2e.yml index f918b5cc1e..e081b1df39 100644 --- a/compose-e2e.yml +++ b/compose-e2e.yml @@ -13,7 +13,7 @@ services: ports: - "3000:3000" - y-provider: + y-provider-converter: user: ${DOCKER_USER:-1000} build: context: . @@ -24,16 +24,3 @@ services: env_file: - env.d/development/common - env.d/development/common.local - ports: - - "4444:4444" - - y-provider-converter: - user: ${DOCKER_USER:-1000} - image: impress:y-provider-production - restart: unless-stopped - env_file: - - env.d/development/common - - env.d/development/common.local - depends_on: - y-provider: - condition: service_started diff --git a/compose.yml b/compose.yml index fe5732f51b..b8ac0bf474 100644 --- a/compose.yml +++ b/compose.yml @@ -181,7 +181,7 @@ services: volumes: - ".:/app" - y-provider-development: + y-provider-development-converter: user: ${DOCKER_USER:-1000} build: context: . @@ -192,27 +192,66 @@ services: env_file: - env.d/development/common - env.d/development/common.local - ports: - - "4444:4444" volumes: - ./src/frontend/:/home/frontend - /home/frontend/node_modules - /home/frontend/servers/y-provider/node_modules - y-provider-development-converter: + yhub-valkey: + image: valkey/valkey:alpine + # volatile-lru per yhub DEPLOYMENT.md; AOF because valkey is the authoritative store + # for updates the worker hasn't persisted yet (up to taskDebounce+minMessageLifetime) + command: ["valkey-server", "--maxmemory-policy", "volatile-lru", + "--appendonly", "yes", "--appendfsync", "everysec"] + volumes: + - yhub-valkey-data:/data + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 1s + timeout: 2s + retries: 60 + + yhub-postgres: + image: postgres:16-alpine + environment: + POSTGRES_USER: yhub + POSTGRES_PASSWORD: yhub + POSTGRES_DB: yhub + volumes: + - yhub-pgdata:/var/lib/postgresql/data + # NOTE: initdb.d only runs on a FRESH volume; schema changes need `podman volume rm` + - ./docker/files/yhub/initdb:/docker-entrypoint-initdb.d:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U yhub"] + interval: 1s + timeout: 2s + retries: 60 + # no published port (Django's postgres already publishes) + + yhub: user: ${DOCKER_USER:-1000} - image: impress:y-provider-development - restart: unless-stopped + build: + context: ./src/yhub-server + dockerfile: Dockerfile + target: yhub + image: impress:yhub + environment: + HOME: /tmp # same reason as node-based services above (unmapped uid) + PORT: 3002 + REDIS: redis://yhub-valkey:6379 + POSTGRES: postgres://yhub:yhub@yhub-postgres:5432/yhub + REDIS_PREFIX: yhub env_file: - env.d/development/common - env.d/development/common.local - volumes: - - ./src/frontend/:/home/frontend - - /home/frontend/node_modules - - /home/frontend/servers/y-provider/node_modules + restart: unless-stopped + ports: + - "3002:3002" depends_on: - y-provider-development: - condition: service_started + yhub-valkey: + condition: service_healthy + yhub-postgres: + condition: service_healthy kc_postgresql: image: postgres:14.3 @@ -268,3 +307,7 @@ networks: name: lasuite-network driver: bridge external: true + +volumes: + yhub-pgdata: {} + yhub-valkey-data: {} diff --git a/docker/files/yhub/initdb/01-yhub.sql b/docker/files/yhub/initdb/01-yhub.sql new file mode 100644 index 0000000000..0a28fb3c06 --- /dev/null +++ b/docker/files/yhub/initdb/01-yhub.sql @@ -0,0 +1,14 @@ +-- Column-for-column from yhub bin/init-db.js (unquoted identifiers so +-- case-folding matches yhub's persistence.js queries). +CREATE TABLE IF NOT EXISTS yhub_ydoc_v1 ( + org text, + docid text, + branch text, + t text, + created INT8, + gcDoc bytea, + nongcDoc bytea, + contentmap bytea, + contentids bytea, + PRIMARY KEY (org,docid,branch,t) +); diff --git a/documentation/system-requirements.md b/documentation/system-requirements.md index db337d9b23..db36deea83 100644 --- a/documentation/system-requirements.md +++ b/documentation/system-requirements.md @@ -89,7 +89,7 @@ Production deployments differ significantly from development environments. The t | --------- | --------------------- | | 3000 | Next.js | | 8071 | Django | -| 4444 | Y-Provider | +| 3002 | yhub (collaboration WS) | | 8080 | Keycloak | | 8083 | Nginx proxy | | 9000/9001 | MinIO | diff --git a/env.d/development/common b/env.d/development/common index b0aea4e690..ef3a8010bf 100644 --- a/env.d/development/common +++ b/env.d/development/common @@ -72,12 +72,12 @@ OIDC_RS_ALLOWED_AUDIENCES="" USER_RECONCILIATION_FORM_URL=http://localhost:3000 # Collaboration -COLLABORATION_API_URL=http://y-provider-development:4444/collaboration/api/ +# TODO(yhub): no management API yet COLLABORATION_BACKEND_BASE_URL=http://app-dev:8000 COLLABORATION_SERVER_ORIGIN=http://localhost:3000 COLLABORATION_SERVER_SECRET=my-secret COLLABORATION_WS_NOT_CONNECTED_READ_ONLY=true -COLLABORATION_WS_URL=ws://localhost:4444/collaboration/ws/ +COLLABORATION_WS_URL=ws://localhost:3002/ws/docs COLLABORATION_WS_INACTIVITY_TIMEOUT=15 # Seconds DJANGO_SERVER_TO_SERVER_API_TOKENS=server-api-token diff --git a/env.d/development/common.e2e b/env.d/development/common.e2e index 6a2131c78b..5ad8fd0d8c 100644 --- a/env.d/development/common.e2e +++ b/env.d/development/common.e2e @@ -1,6 +1,5 @@ # For the CI job test-e2e BURST_THROTTLE_RATES="1000/minute" -COLLABORATION_API_URL=http://y-provider:4444/collaboration/api/ SUSTAINED_THROTTLE_RATES="1000/minute" Y_PROVIDER_API_BASE_URL=http://y-provider-converter:4444/api/ diff --git a/env.d/production.dist/common b/env.d/production.dist/common index bb289de71d..f54f54a37c 100644 --- a/env.d/production.dist/common +++ b/env.d/production.dist/common @@ -6,4 +6,5 @@ FRONTEND_HOST=frontend YPROVIDER_HOST=y-provider BUCKET_NAME=docs-media-storage REALM_NAME=docs +# TODO(yhub): route is /ws/docs once prod ingress is migrated #COLLABORATION_WS_URL=wss://${DOCS_HOST}/collaboration/ws/ \ No newline at end of file diff --git a/src/backend/core/management/commands/clean_document.py b/src/backend/core/management/commands/clean_document.py index e7a006ad51..5e041a80c8 100644 --- a/src/backend/core/management/commands/clean_document.py +++ b/src/backend/core/management/commands/clean_document.py @@ -1,5 +1,13 @@ """Clean a document by resetting it (keeping its title) and deleting all descendants.""" +# TODO(yhub): this sandbox reset no longer erases the document content. It purges +# the S3 versions, but yhub durably retains the Yjs document in its own Postgres +# and re-serves it on the next websocket connect (CRDT merge with the empty +# seed resurrects the purged content). yhub has no delete API; until it grows +# one, the interim remediation is to run, against yhub's stores: +# DELETE FROM yhub_ydoc_v1 WHERE org='docs' AND docid=''; +# and drop the `yhub:room:docs::*` redis keys. + import logging from django.conf import settings diff --git a/src/backend/core/services/collaboration_services.py b/src/backend/core/services/collaboration_services.py index fa1e1e867a..0bf891580a 100644 --- a/src/backend/core/services/collaboration_services.py +++ b/src/backend/core/services/collaboration_services.py @@ -2,102 +2,39 @@ from logging import getLogger -from django.conf import settings -from django.core.exceptions import ImproperlyConfigured - -import requests - -from core import models - logger = getLogger(__name__) class CollaborationService: """Service class for Collaboration related operations.""" - def __init__(self): - """Ensure that the collaboration configuration is set properly.""" - if settings.COLLABORATION_API_URL is None: - raise ImproperlyConfigured("Collaboration configuration not set") - def reset_connections(self, document_id, user_id=None): """ Reset the connections of a document and all its descendants in the collaboration server. - Resetting a connection means that the user will be disconnected and will - have to reconnect to the collaboration server, with updated rights. + TODO(yhub): yhub exposes no kick API, so this is a no-op. The regression + is stronger than losing the hocuspocus disconnect: a revoked user keeps + their already-authorized websocket until it closes on its own, and the + edits they push in the meantime are durably persisted and re-served by + yhub (hocuspocus lost them with the room). Until yhub grows a kick API, + the manual remediation is yhub's rollback endpoint — per document: + `POST /rollback/{org}/{docid}` with a lib0-encoded body containing + `{"by": ""}` (see yhub API.md "Rollback"), authenticated as a + user with update ability on the document. """ - try: - document = models.Document.objects.get(pk=document_id) - except models.Document.DoesNotExist: - logger.error("Document %s does not exists anymore", document_id) - return - - documents = models.Document.objects.filter( - path__startswith=document.path, depth__gte=document.depth - ).order_by("path") - - for doc in documents: - try: - self._reset_connection(doc.id, user_id) - except requests.HTTPError: - logger.error("impossible to reset connections for document %s", doc.id) - - def _reset_connection(self, room, user_id=None): - """ - Reset connections of a single room in the collaboration server. - """ - endpoint = "reset-connections" - - # room is necessary as a parameter, it is easier to stick to the - # same pod thanks to a parameter - endpoint_url = f"{settings.COLLABORATION_API_URL}{endpoint}/?room={room}" - - # Note: Collaboration microservice accepts only raw token, which is not recommended - headers = {"Authorization": settings.COLLABORATION_SERVER_SECRET} - if user_id: - headers["X-User-Id"] = user_id - - try: - response = requests.post(endpoint_url, headers=headers, timeout=10) - except requests.RequestException as e: - raise requests.HTTPError("Failed to notify WebSocket server.") from e - - if response.status_code != 200: - raise requests.HTTPError( - f"Failed to notify WebSocket server. Status code: {response.status_code}, " - f"Response: {response.text}" - ) + logger.info( + "reset_connections is a no-op (no yhub kick API), document %s, user %s", + document_id, + user_id, + ) + # pylint: disable=unused-argument def get_document_connection_info(self, room, session_key): """ Get the connection info for a document. - """ - endpoint = "get-connections" - querystring = { - "room": room, - "sessionKey": session_key, - } - endpoint_url = f"{settings.COLLABORATION_API_URL}{endpoint}/" - - headers = {"Authorization": settings.COLLABORATION_SERVER_SECRET} - try: - response = requests.get( - endpoint_url, headers=headers, params=querystring, timeout=10 - ) - except requests.RequestException as e: - raise requests.HTTPError("Failed to get document connection info.") from e - - if response.status_code == 200: - result = response.json() - return result.get("count", 0), result.get("exists", False) - - if response.status_code == 404: - return 0, False - - raise requests.HTTPError( - f"Failed to get document connection info. Status code: {response.status_code}, " - f"Response: {response.text}" - ) + TODO(yhub): yhub exposes no connection-info API, so pretend nobody is + connected. Callers fall back to the cache-lock no-websocket path. + """ + return 0, False diff --git a/src/backend/core/tests/documents/test_api_documents_can_edit.py b/src/backend/core/tests/documents/test_api_documents_can_edit.py index f167f033a2..4755bd93ba 100644 --- a/src/backend/core/tests/documents/test_api_documents_can_edit.py +++ b/src/backend/core/tests/documents/test_api_documents_can_edit.py @@ -3,7 +3,6 @@ from django.core.cache import cache import pytest -import responses from rest_framework.test import APIClient from core import factories @@ -11,22 +10,13 @@ pytestmark = pytest.mark.django_db -@responses.activate @pytest.mark.parametrize("ws_not_connected_ready_only", [True, False]) @pytest.mark.parametrize("role", ["editor", "reader"]) def test_api_documents_can_edit_anonymous(settings, ws_not_connected_ready_only, role): """Anonymous users can edit documents when link_role is editor.""" document = factories.DocumentFactory(link_reach="public", link_role=role) client = APIClient() - session_key = client.session.session_key - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = ws_not_connected_ready_only - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, json={"count": 0, "exists": False}) response = client.get(f"/api/v1.0/documents/{document.id!s}/can-edit/") @@ -35,10 +25,8 @@ def test_api_documents_can_edit_anonymous(settings, ws_not_connected_ready_only, else: assert response.status_code == 200 assert response.json() == {"can_edit": True} - assert ws_resp.call_count == (1 if ws_not_connected_ready_only else 0) -@responses.activate @pytest.mark.parametrize("ws_not_connected_ready_only", [True, False]) def test_api_documents_can_edit_authenticated_no_websocket( settings, ws_not_connected_ready_only @@ -50,19 +38,10 @@ def test_api_documents_can_edit_authenticated_no_websocket( user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = ws_not_connected_ready_only - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - - ws_resp = responses.get(endpoint_url, json={"count": 0, "exists": False}) assert cache.get(f"docs:no-websocket:{document.id}") is None @@ -72,10 +51,8 @@ def test_api_documents_can_edit_authenticated_no_websocket( assert response.status_code == 200 assert response.json() == {"can_edit": True} - assert ws_resp.call_count == (1 if ws_not_connected_ready_only else 0) -@responses.activate def test_api_documents_can_edit_authenticated_no_websocket_user_already_editing( settings, ): @@ -86,18 +63,10 @@ def test_api_documents_can_edit_authenticated_no_websocket_user_already_editing( user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, json={"count": 0, "exists": False}) cache.set(f"docs:no-websocket:{document.id}", "other_session_key") @@ -107,45 +76,13 @@ def test_api_documents_can_edit_authenticated_no_websocket_user_already_editing( assert response.status_code == 200 assert response.json() == {"can_edit": False} - assert ws_resp.call_count == 1 - - -@responses.activate -def test_api_documents_can_edit_no_websocket_other_user_connected_to_websocket( - settings, -): - """ - A user not connected to the websocket and another user is connected to the websocket, - the document can not be updated. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - session_key = client.session.session_key - document = factories.DocumentFactory(users=[(user, "editor")]) +# TODO(yhub): removed test_api_documents_can_edit_no_websocket_other_user_connected_to_websocket +# here. yhub has no connection-info API: get_document_connection_info is stubbed to report +# nobody connected, so another user connected to the websocket can no longer block edition. +# Re-add the test once yhub exposes a connection-info API. - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, json={"count": 3, "exists": False}) - assert cache.get(f"docs:no-websocket:{document.id}") is None - - response = client.get( - f"/api/v1.0/documents/{document.id!s}/can-edit/", - ) - assert response.status_code == 200 - assert response.json() == {"can_edit": False} - assert cache.get(f"docs:no-websocket:{document.id}") is None - assert ws_resp.call_count == 1 - - -@responses.activate def test_api_documents_can_edit_user_connected_to_websocket(settings): """ A user connected to the websocket, the document can be updated. @@ -153,18 +90,10 @@ def test_api_documents_can_edit_user_connected_to_websocket(settings): user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, json={"count": 3, "exists": True}) assert cache.get(f"docs:no-websocket:{document.id}") is None @@ -174,10 +103,8 @@ def test_api_documents_can_edit_user_connected_to_websocket(settings): assert response.status_code == 200 assert response.json() == {"can_edit": True} assert cache.get(f"docs:no-websocket:{document.id}") is None - assert ws_resp.call_count == 1 -@responses.activate def test_api_documents_can_edit_websocket_server_unreachable_fallback_to_no_websocket( settings, ): @@ -188,18 +115,10 @@ def test_api_documents_can_edit_websocket_server_unreachable_fallback_to_no_webs user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, status=500) assert cache.get(f"docs:no-websocket:{document.id}") is None @@ -209,10 +128,7 @@ def test_api_documents_can_edit_websocket_server_unreachable_fallback_to_no_webs assert response.status_code == 200 assert response.json() == {"can_edit": True} - assert ws_resp.call_count == 1 - -@responses.activate def test_api_documents_can_edit_websocket_server_unreachable_fallback_to_no_websocket_other_users( settings, ): @@ -223,18 +139,10 @@ def test_api_documents_can_edit_websocket_server_unreachable_fallback_to_no_webs user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, status=500) cache.set(f"docs:no-websocket:{document.id}", "other_session_key") @@ -245,10 +153,8 @@ def test_api_documents_can_edit_websocket_server_unreachable_fallback_to_no_webs assert response.json() == {"can_edit": False} assert cache.get(f"docs:no-websocket:{document.id}") == "other_session_key" - assert ws_resp.call_count == 1 -@responses.activate def test_api_documents_can_edit_websocket_server_room_not_found( settings, ): @@ -259,18 +165,10 @@ def test_api_documents_can_edit_websocket_server_room_not_found( user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, status=404) assert cache.get(f"docs:no-websocket:{document.id}") is None @@ -280,10 +178,7 @@ def test_api_documents_can_edit_websocket_server_room_not_found( assert response.status_code == 200 assert response.json() == {"can_edit": True} - assert ws_resp.call_count == 1 - -@responses.activate def test_api_documents_can_edit_websocket_server_room_not_found_other_already_editing( settings, ): @@ -294,18 +189,10 @@ def test_api_documents_can_edit_websocket_server_room_not_found_other_already_ed user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, status=404) cache.set(f"docs:no-websocket:{document.id}", "other_session_key") @@ -314,5 +201,3 @@ def test_api_documents_can_edit_websocket_server_room_not_found_other_already_ed ) assert response.status_code == 200 assert response.json() == {"can_edit": False} - - assert ws_resp.call_count == 1 diff --git a/src/backend/core/tests/documents/test_api_documents_content_update.py b/src/backend/core/tests/documents/test_api_documents_content_update.py index b7b8761476..5184df4118 100644 --- a/src/backend/core/tests/documents/test_api_documents_content_update.py +++ b/src/backend/core/tests/documents/test_api_documents_content_update.py @@ -11,7 +11,6 @@ import pycrdt import pytest -import responses from rest_framework import status from rest_framework.test import APIClient @@ -254,7 +253,6 @@ def test_api_documents_content_update_link_editor(): assert models.Document.objects.filter(id=document.id).exists() -@responses.activate def test_api_documents_content_update_authenticated_no_websocket(settings): """ When a user updates the document content, not connected to the websocket and is the first @@ -267,14 +265,7 @@ def test_api_documents_content_update_authenticated_no_websocket(settings): document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, json={"count": 0, "exists": False}) assert django_cache.get(f"docs:no-websocket:{document.id}") is None @@ -285,10 +276,8 @@ def test_api_documents_content_update_authenticated_no_websocket(settings): assert response.status_code == status.HTTP_204_NO_CONTENT assert get_s3_content(document) == get_sample_ydoc() assert django_cache.get(f"docs:no-websocket:{document.id}") == session_key - assert ws_resp.call_count == 1 -@responses.activate def test_api_documents_content_update_authenticated_no_websocket_user_already_editing( settings, ): @@ -299,18 +288,10 @@ def test_api_documents_content_update_authenticated_no_websocket_user_already_ed user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, json={"count": 0, "exists": False}) django_cache.set(f"docs:no-websocket:{document.id}", "other_session_key") @@ -320,46 +301,15 @@ def test_api_documents_content_update_authenticated_no_websocket_user_already_ed ) assert response.status_code == status.HTTP_403_FORBIDDEN assert response.json() == {"detail": "You are not allowed to edit this document."} - assert ws_resp.call_count == 1 -@responses.activate -def test_api_documents_content_update_no_websocket_other_user_connected_to_websocket( - settings, -): - """ - When a user updates document content without websocket and another user is connected - to the websocket, the update should be denied. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - session_key = client.session.session_key - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, json={"count": 3, "exists": False}) - - assert django_cache.get(f"docs:no-websocket:{document.id}") is None - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_sample_ydoc(), "websocket": False}, - ) - assert response.status_code == status.HTTP_403_FORBIDDEN - assert response.json() == {"detail": "You are not allowed to edit this document."} - assert django_cache.get(f"docs:no-websocket:{document.id}") is None - assert ws_resp.call_count == 1 +# TODO(yhub): removed +# test_api_documents_content_update_no_websocket_other_user_connected_to_websocket +# here. yhub has no connection-info API: get_document_connection_info is stubbed to report +# nobody connected, so another user connected to the websocket can no longer block the update. +# Re-add the test once yhub exposes a connection-info API. -@responses.activate def test_api_documents_content_update_user_connected_to_websocket(settings): """ When a user updates document content and is connected to the websocket, @@ -372,14 +322,7 @@ def test_api_documents_content_update_user_connected_to_websocket(settings): document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, json={"count": 3, "exists": True}) assert django_cache.get(f"docs:no-websocket:{document.id}") is None @@ -389,11 +332,11 @@ def test_api_documents_content_update_user_connected_to_websocket(settings): ) assert response.status_code == status.HTTP_204_NO_CONTENT assert get_s3_content(document) == get_sample_ydoc() - assert django_cache.get(f"docs:no-websocket:{document.id}") is None - assert ws_resp.call_count == 1 + # TODO(yhub): the stubbed connection info reports nobody connected, so the + # no-websocket cache lock is taken even though the user is connected. + assert django_cache.get(f"docs:no-websocket:{document.id}") == session_key -@responses.activate def test_api_documents_content_update_websocket_server_unreachable_fallback_to_no_websocket( settings, ): @@ -408,14 +351,7 @@ def test_api_documents_content_update_websocket_server_unreachable_fallback_to_n document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, status=500) assert django_cache.get(f"docs:no-websocket:{document.id}") is None @@ -426,10 +362,8 @@ def test_api_documents_content_update_websocket_server_unreachable_fallback_to_n assert response.status_code == status.HTTP_204_NO_CONTENT assert get_s3_content(document) == get_sample_ydoc() assert django_cache.get(f"docs:no-websocket:{document.id}") == session_key - assert ws_resp.call_count == 1 -@responses.activate def test_api_content_update_websocket_server_unreachable_fallback_to_no_websocket_other_users( settings, ): @@ -440,18 +374,10 @@ def test_api_content_update_websocket_server_unreachable_fallback_to_no_websocke user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, status=500) django_cache.set(f"docs:no-websocket:{document.id}", "other_session_key") @@ -461,10 +387,8 @@ def test_api_content_update_websocket_server_unreachable_fallback_to_no_websocke ) assert response.status_code == status.HTTP_403_FORBIDDEN assert django_cache.get(f"docs:no-websocket:{document.id}") == "other_session_key" - assert ws_resp.call_count == 1 -@responses.activate def test_api_content_update_websocket_server_room_not_found_fallback_to_no_websocket_other_users( settings, ): @@ -475,18 +399,10 @@ def test_api_content_update_websocket_server_room_not_found_fallback_to_no_webso user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, status=404) django_cache.set(f"docs:no-websocket:{document.id}", "other_session_key") @@ -496,10 +412,8 @@ def test_api_content_update_websocket_server_room_not_found_fallback_to_no_webso ) assert response.status_code == status.HTTP_403_FORBIDDEN assert django_cache.get(f"docs:no-websocket:{document.id}") == "other_session_key" - assert ws_resp.call_count == 1 -@responses.activate def test_api_documents_content_update_force_websocket_param_to_true(settings): """ When the websocket parameter is set to true, the content should be updated without any check. @@ -507,18 +421,10 @@ def test_api_documents_content_update_force_websocket_param_to_true(settings): user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, status=500) assert django_cache.get(f"docs:no-websocket:{document.id}") is None @@ -529,10 +435,8 @@ def test_api_documents_content_update_force_websocket_param_to_true(settings): assert response.status_code == status.HTTP_204_NO_CONTENT assert get_s3_content(document) == get_sample_ydoc() assert django_cache.get(f"docs:no-websocket:{document.id}") is None - assert ws_resp.call_count == 0 -@responses.activate def test_api_documents_content_update_feature_flag_disabled(settings): """ When the feature flag is disabled, the content should be updated without any check. @@ -540,18 +444,10 @@ def test_api_documents_content_update_feature_flag_disabled(settings): user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = False - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, status=500) assert django_cache.get(f"docs:no-websocket:{document.id}") is None @@ -562,7 +458,6 @@ def test_api_documents_content_update_feature_flag_disabled(settings): assert response.status_code == status.HTTP_204_NO_CONTENT assert get_s3_content(document) == get_sample_ydoc() assert django_cache.get(f"docs:no-websocket:{document.id}") is None - assert ws_resp.call_count == 0 def test_api_documents_content_upadte_invalid_yjs_doc(): diff --git a/src/backend/core/tests/documents/test_api_documents_update.py b/src/backend/core/tests/documents/test_api_documents_update.py index 29c6d72cfc..358753b496 100644 --- a/src/backend/core/tests/documents/test_api_documents_update.py +++ b/src/backend/core/tests/documents/test_api_documents_update.py @@ -10,7 +10,6 @@ from django.core.cache import cache import pytest -import responses from rest_framework.test import APIClient from core import factories, models @@ -304,7 +303,6 @@ def test_api_documents_update_authenticated_editor_administrator_or_owner( assert value == new_document_values[key] -@responses.activate def test_api_documents_update_authenticated_no_websocket(settings): """ When a user updates the document, not connected to the websocket and is the first to update, @@ -321,15 +319,7 @@ def test_api_documents_update_authenticated_no_websocket(settings): instance=factories.DocumentFactory() ).data new_document_values["websocket"] = False - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - - ws_resp = responses.get(endpoint_url, json={"count": 0, "exists": False}) assert cache.get(f"docs:no-websocket:{document.id}") is None old_path = document.path @@ -344,10 +334,8 @@ def test_api_documents_update_authenticated_no_websocket(settings): document.refresh_from_db() assert document.path == old_path assert cache.get(f"docs:no-websocket:{document.id}") == session_key - assert ws_resp.call_count == 1 -@responses.activate def test_api_documents_update_authenticated_no_websocket_user_already_editing(settings): """ When a user updates the document, not connected to the websocket and is not the first to update, @@ -356,7 +344,6 @@ def test_api_documents_update_authenticated_no_websocket_user_already_editing(se user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) @@ -364,14 +351,7 @@ def test_api_documents_update_authenticated_no_websocket_user_already_editing(se instance=factories.DocumentFactory() ).data new_document_values["websocket"] = False - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, json={"count": 0, "exists": False}) cache.set(f"docs:no-websocket:{document.id}", "other_session_key") @@ -383,49 +363,13 @@ def test_api_documents_update_authenticated_no_websocket_user_already_editing(se assert response.status_code == 403 assert response.json() == {"detail": "You are not allowed to edit this document."} - assert ws_resp.call_count == 1 - - -@responses.activate -def test_api_documents_update_no_websocket_other_user_connected_to_websocket(settings): - """ - When a user updates the document, not connected to the websocket and another user is connected - to the websocket, the document should not be updated. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - session_key = client.session.session_key - - document = factories.DocumentFactory(users=[(user, "editor")]) - new_document_values = serializers.DocumentSerializer( - instance=factories.DocumentFactory() - ).data - new_document_values["websocket"] = False - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, json={"count": 3, "exists": False}) +# TODO(yhub): removed test_api_documents_update_no_websocket_other_user_connected_to_websocket +# here. yhub has no connection-info API: get_document_connection_info is stubbed to report +# nobody connected, so another user connected to the websocket can no longer block the update. +# Re-add the test once yhub exposes a connection-info API. - assert cache.get(f"docs:no-websocket:{document.id}") is None - response = client.put( - f"/api/v1.0/documents/{document.id!s}/", - new_document_values, - format="json", - ) - assert response.status_code == 403 - assert response.json() == {"detail": "You are not allowed to edit this document."} - assert cache.get(f"docs:no-websocket:{document.id}") is None - assert ws_resp.call_count == 1 - - -@responses.activate def test_api_documents_update_user_connected_to_websocket(settings): """ When a user updates the document, connected to the websocket, the document should be updated. @@ -441,14 +385,7 @@ def test_api_documents_update_user_connected_to_websocket(settings): instance=factories.DocumentFactory() ).data new_document_values["websocket"] = False - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, json={"count": 3, "exists": True}) assert cache.get(f"docs:no-websocket:{document.id}") is None old_path = document.path @@ -462,11 +399,11 @@ def test_api_documents_update_user_connected_to_websocket(settings): document.refresh_from_db() assert document.path == old_path - assert cache.get(f"docs:no-websocket:{document.id}") is None - assert ws_resp.call_count == 1 + # TODO(yhub): the stubbed connection info reports nobody connected, so the + # no-websocket cache lock is taken even though the user is connected. + assert cache.get(f"docs:no-websocket:{document.id}") == session_key -@responses.activate def test_api_documents_update_websocket_server_unreachable_fallback_to_no_websocket( settings, ): @@ -485,14 +422,7 @@ def test_api_documents_update_websocket_server_unreachable_fallback_to_no_websoc instance=factories.DocumentFactory() ).data new_document_values["websocket"] = False - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, status=500) assert cache.get(f"docs:no-websocket:{document.id}") is None old_path = document.path @@ -507,10 +437,8 @@ def test_api_documents_update_websocket_server_unreachable_fallback_to_no_websoc document.refresh_from_db() assert document.path == old_path assert cache.get(f"docs:no-websocket:{document.id}") == session_key - assert ws_resp.call_count == 1 -@responses.activate def test_api_documents_update_websocket_server_unreachable_fallback_to_no_websocket_other_users( settings, ): @@ -521,7 +449,6 @@ def test_api_documents_update_websocket_server_unreachable_fallback_to_no_websoc user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) @@ -529,14 +456,7 @@ def test_api_documents_update_websocket_server_unreachable_fallback_to_no_websoc instance=factories.DocumentFactory() ).data new_document_values["websocket"] = False - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, status=500) cache.set(f"docs:no-websocket:{document.id}", "other_session_key") @@ -548,10 +468,8 @@ def test_api_documents_update_websocket_server_unreachable_fallback_to_no_websoc assert response.status_code == 403 assert cache.get(f"docs:no-websocket:{document.id}") == "other_session_key" - assert ws_resp.call_count == 1 -@responses.activate def test_api_documents_update_websocket_server_room_not_found_fallback_to_no_websocket_other_users( settings, ): @@ -562,7 +480,6 @@ def test_api_documents_update_websocket_server_room_not_found_fallback_to_no_web user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) @@ -570,14 +487,7 @@ def test_api_documents_update_websocket_server_room_not_found_fallback_to_no_web instance=factories.DocumentFactory() ).data new_document_values["websocket"] = False - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, status=404) cache.set(f"docs:no-websocket:{document.id}", "other_session_key") @@ -589,18 +499,15 @@ def test_api_documents_update_websocket_server_room_not_found_fallback_to_no_web assert response.status_code == 403 assert cache.get(f"docs:no-websocket:{document.id}") == "other_session_key" - assert ws_resp.call_count == 1 -@responses.activate -def test_api_documents_update_force_websocket_param_to_true(settings): +def test_api_documents_update_force_websocket_param_to_true(): """ When the websocket parameter is set to true, the document should be updated without any check. """ user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) @@ -608,13 +515,6 @@ def test_api_documents_update_force_websocket_param_to_true(settings): instance=factories.DocumentFactory() ).data new_document_values["websocket"] = True - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, status=500) assert cache.get(f"docs:no-websocket:{document.id}") is None old_path = document.path @@ -629,10 +529,8 @@ def test_api_documents_update_force_websocket_param_to_true(settings): document.refresh_from_db() assert document.path == old_path assert cache.get(f"docs:no-websocket:{document.id}") is None - assert ws_resp.call_count == 0 -@responses.activate def test_api_documents_update_feature_flag_disabled(settings): """ When the feature flag is disabled, the document should be updated without any check. @@ -640,7 +538,6 @@ def test_api_documents_update_feature_flag_disabled(settings): user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) @@ -648,14 +545,7 @@ def test_api_documents_update_feature_flag_disabled(settings): instance=factories.DocumentFactory() ).data new_document_values["websocket"] = False - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = False - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, status=500) assert cache.get(f"docs:no-websocket:{document.id}") is None old_path = document.path @@ -670,7 +560,6 @@ def test_api_documents_update_feature_flag_disabled(settings): document.refresh_from_db() assert document.path == old_path assert cache.get(f"docs:no-websocket:{document.id}") is None - assert ws_resp.call_count == 0 @pytest.mark.parametrize("via", VIA) @@ -968,7 +857,6 @@ def test_api_documents_patch_authenticated_editor_administrator_or_owner( assert document_values[key] == old_document_values[key] -@responses.activate def test_api_documents_patch_authenticated_no_websocket(settings): """ When a user patches the document, not connected to the websocket and is the first to update, @@ -981,14 +869,7 @@ def test_api_documents_patch_authenticated_no_websocket(settings): document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, json={"count": 0, "exists": False}) assert cache.get(f"docs:no-websocket:{document.id}") is None old_path = document.path @@ -1006,10 +887,8 @@ def test_api_documents_patch_authenticated_no_websocket(settings): assert document.path == old_path assert document.title == "new title" assert cache.get(f"docs:no-websocket:{document.id}") == session_key - assert ws_resp.call_count == 1 -@responses.activate def test_api_documents_patch_authenticated_no_websocket_user_already_editing(settings): """ When a user patches the document, not connected to the websocket and is not the first to @@ -1018,18 +897,10 @@ def test_api_documents_patch_authenticated_no_websocket_user_already_editing(set user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, json={"count": 0, "exists": False}) cache.set(f"docs:no-websocket:{document.id}", "other_session_key") @@ -1041,45 +912,13 @@ def test_api_documents_patch_authenticated_no_websocket_user_already_editing(set assert response.status_code == 403 assert response.json() == {"detail": "You are not allowed to edit this document."} - assert ws_resp.call_count == 1 - - -@responses.activate -def test_api_documents_patch_no_websocket_other_user_connected_to_websocket(settings): - """ - When a user patches the document, not connected to the websocket and another user is connected - to the websocket, the document should not be updated. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - session_key = client.session.session_key - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, json={"count": 3, "exists": False}) - - assert cache.get(f"docs:no-websocket:{document.id}") is None - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/", - {"title": "new title"}, - format="json", - ) - assert response.status_code == 403 - assert response.json() == {"detail": "You are not allowed to edit this document."} - assert cache.get(f"docs:no-websocket:{document.id}") is None - assert ws_resp.call_count == 1 +# TODO(yhub): removed test_api_documents_patch_no_websocket_other_user_connected_to_websocket +# here. yhub has no connection-info API: get_document_connection_info is stubbed to report +# nobody connected, so another user connected to the websocket can no longer block the patch. +# Re-add the test once yhub exposes a connection-info API. -@responses.activate def test_api_documents_patch_user_connected_to_websocket(settings): """ When a user patches the document while connected to the websocket, the document should be @@ -1092,14 +931,7 @@ def test_api_documents_patch_user_connected_to_websocket(settings): document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, json={"count": 3, "exists": True}) assert cache.get(f"docs:no-websocket:{document.id}") is None old_path = document.path @@ -1116,11 +948,11 @@ def test_api_documents_patch_user_connected_to_websocket(settings): document = models.Document.objects.get(id=document.id) assert document.path == old_path assert document.title == "new title" - assert cache.get(f"docs:no-websocket:{document.id}") is None - assert ws_resp.call_count == 1 + # TODO(yhub): the stubbed connection info reports nobody connected, so the + # no-websocket cache lock is taken even though the user is connected. + assert cache.get(f"docs:no-websocket:{document.id}") == session_key -@responses.activate def test_api_documents_patch_websocket_server_unreachable_fallback_to_no_websocket( settings, ): @@ -1135,14 +967,7 @@ def test_api_documents_patch_websocket_server_unreachable_fallback_to_no_websock document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, status=500) assert cache.get(f"docs:no-websocket:{document.id}") is None old_path = document.path @@ -1160,10 +985,8 @@ def test_api_documents_patch_websocket_server_unreachable_fallback_to_no_websock assert document.path == old_path assert document.title == "new title" assert cache.get(f"docs:no-websocket:{document.id}") == session_key - assert ws_resp.call_count == 1 -@responses.activate def test_api_documents_patch_websocket_server_unreachable_fallback_to_no_websocket_other_users( settings, ): @@ -1174,18 +997,10 @@ def test_api_documents_patch_websocket_server_unreachable_fallback_to_no_websock user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, status=500) cache.set(f"docs:no-websocket:{document.id}", "other_session_key") @@ -1197,10 +1012,8 @@ def test_api_documents_patch_websocket_server_unreachable_fallback_to_no_websock assert response.status_code == 403 assert cache.get(f"docs:no-websocket:{document.id}") == "other_session_key" - assert ws_resp.call_count == 1 -@responses.activate def test_api_documents_patch_websocket_server_room_not_found_fallback_to_no_websocket_other_users( settings, ): @@ -1211,18 +1024,10 @@ def test_api_documents_patch_websocket_server_room_not_found_fallback_to_no_webs user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, status=404) cache.set(f"docs:no-websocket:{document.id}", "other_session_key") @@ -1234,29 +1039,18 @@ def test_api_documents_patch_websocket_server_room_not_found_fallback_to_no_webs assert response.status_code == 403 assert cache.get(f"docs:no-websocket:{document.id}") == "other_session_key" - assert ws_resp.call_count == 1 -@responses.activate -def test_api_documents_patch_force_websocket_param_to_true(settings): +def test_api_documents_patch_force_websocket_param_to_true(): """ When the websocket parameter is set to true, the patch should be applied without any check. """ user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, status=500) - assert cache.get(f"docs:no-websocket:{document.id}") is None old_path = document.path @@ -1273,10 +1067,8 @@ def test_api_documents_patch_force_websocket_param_to_true(settings): assert document.path == old_path assert document.title == "new title" assert cache.get(f"docs:no-websocket:{document.id}") is None - assert ws_resp.call_count == 0 -@responses.activate def test_api_documents_patch_feature_flag_disabled(settings): """ When the feature flag is disabled, the patch should be applied without any check. @@ -1284,18 +1076,10 @@ def test_api_documents_patch_feature_flag_disabled(settings): user = factories.UserFactory(with_owned_document=True) client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "editor")]) - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = False - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, status=500) assert cache.get(f"docs:no-websocket:{document.id}") is None old_path = document.path @@ -1313,7 +1097,6 @@ def test_api_documents_patch_feature_flag_disabled(settings): assert document.path == old_path assert document.title == "new title" assert cache.get(f"docs:no-websocket:{document.id}") is None - assert ws_resp.call_count == 0 @pytest.mark.parametrize("via", VIA) @@ -1358,7 +1141,6 @@ def test_api_documents_patch_administrator_or_owner_of_another(via, mock_user_te ) -@responses.activate def test_api_documents_patch_empty_body(settings): """ Test when data is empty the document should not be updated. @@ -1373,14 +1155,7 @@ def test_api_documents_patch_empty_body(settings): document = factories.DocumentFactory(users=[(user, "owner")], creator=user) document_updated_at = document.updated_at - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}get-connections/" - f"?room={document.id}&sessionKey={session_key}" - ) - ws_resp = responses.get(endpoint_url, json={"count": 3, "exists": True}) assert cache.get(f"docs:no-websocket:{document.id}") is None @@ -1398,5 +1173,6 @@ def test_api_documents_patch_empty_body(settings): new_document_values = serializers.DocumentSerializer(instance=document).data assert new_document_values == old_document_values assert document_updated_at == document.updated_at - assert cache.get(f"docs:no-websocket:{document.id}") is None - assert ws_resp.call_count == 1 + # TODO(yhub): the stubbed connection info reports nobody connected, so the + # no-websocket cache lock is taken even for an empty body. + assert cache.get(f"docs:no-websocket:{document.id}") == session_key diff --git a/src/backend/core/tests/external_api/test_external_api_documents_accesses.py b/src/backend/core/tests/external_api/test_external_api_documents_accesses.py index 957b308291..1a26f34563 100644 --- a/src/backend/core/tests/external_api/test_external_api_documents_accesses.py +++ b/src/backend/core/tests/external_api/test_external_api_documents_accesses.py @@ -9,7 +9,6 @@ from django.test import override_settings import pytest -import responses from rest_framework.test import APIClient from core import factories, models @@ -504,7 +503,6 @@ def test_external_api_document_accesses_update_can_be_allowed( user_token, resource_server_backend, user_specific_sub, - settings, ): """ A user who is related to a document SHOULD be allowed to update @@ -525,19 +523,6 @@ def test_external_api_document_accesses_update_can_be_allowed( document=document, user=other_user, role=models.RoleChoices.READER ) - # Add the reset-connections endpoint to the existing mock - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}reset-connections/?room={document.id}" - ) - resource_server_backend.add( - responses.POST, - endpoint_url, - json={}, - status=200, - ) - old_values = serializers.DocumentAccessSerializer(instance=access).data # Update only the role field @@ -573,7 +558,6 @@ def test_external_api_document_accesses_partial_update_can_be_allowed( user_token, resource_server_backend, user_specific_sub, - settings, ): """ A user who is related to a document SHOULD be allowed to update @@ -594,19 +578,6 @@ def test_external_api_document_accesses_partial_update_can_be_allowed( document=document, user=other_user, role=models.RoleChoices.READER ) - # Add the reset-connections endpoint to the existing mock - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}reset-connections/?room={document.id}" - ) - resource_server_backend.add( - responses.POST, - endpoint_url, - json={}, - status=200, - ) - response = client.patch( f"/external_api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", data={"role": models.RoleChoices.EDITOR}, @@ -635,7 +606,7 @@ def test_external_api_document_accesses_partial_update_can_be_allowed( } ) def test_external_api_documents_accesses_delete_can_be_allowed( - user_token, resource_server_backend, user_specific_sub, settings + user_token, resource_server_backend, user_specific_sub ): """ Connected users SHOULD be allowed to delete an access for @@ -661,19 +632,6 @@ def test_external_api_documents_accesses_delete_can_be_allowed( document=document, user=other_user, role=models.RoleChoices.READER ) - # Add the reset-connections endpoint to the existing mock - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}reset-connections/?room={document.id}" - ) - resource_server_backend.add( - responses.POST, - endpoint_url, - json={}, - status=200, - ) - response = client.delete( f"/external_api/v1.0/documents/{document.id!s}/accesses/{other_access.id!s}/", ) diff --git a/src/backend/core/tests/external_api/test_external_api_documents_link_configuration.py b/src/backend/core/tests/external_api/test_external_api_documents_link_configuration.py index 885862f0f8..7c1e6a3086 100644 --- a/src/backend/core/tests/external_api/test_external_api_documents_link_configuration.py +++ b/src/backend/core/tests/external_api/test_external_api_documents_link_configuration.py @@ -60,8 +60,6 @@ def test_external_api_documents_link_configuration_not_allowed( ], }, }, - COLLABORATION_API_URL="http://example.com/", - COLLABORATION_SERVER_SECRET="secret-token", ) @patch("core.api.viewsets.reset_service_connections_in_cascade.delay") def test_external_api_documents_link_configuration_can_be_allowed( diff --git a/src/backend/core/tests/test_services_collaboration_services.py b/src/backend/core/tests/test_services_collaboration_services.py index 35e607d21f..1eec2f1772 100644 --- a/src/backend/core/tests/test_services_collaboration_services.py +++ b/src/backend/core/tests/test_services_collaboration_services.py @@ -3,342 +3,28 @@ core.services.collaboration_services module. """ -import json -import logging -import re -from contextlib import contextmanager -from unittest import mock -from uuid import uuid4 - -from django.core.exceptions import ImproperlyConfigured - -import pytest -import requests import responses -from core import factories, models from core.services.collaboration_services import CollaborationService -# pylint: disable=protected-access - -@pytest.fixture(name="mock_reset_connections") -def mock_reset_connections_fixture(settings): +def test_reset_connections_makes_no_http_call(): """ - Creates a context manager to mock the reset-connections endpoint for collaboration services. - Args: - settings: A settings object that contains the configuration for the collaboration API. - Returns: - A context manager function that mocks the reset-connections endpoint. - The context manager function takes the following parameters: - document_id (str): The ID of the document for which connections are being reset. - user_id (str, optional): The ID of the user making the request. Defaults to None. - Usage: - with mock_reset_connections(settings)(document_id, user_id) as mock: - # Your test code here - The context manager performs the following actions: - - Mocks the reset-connections endpoint using responses.RequestsMock. - - Sets the COLLABORATION_API_URL and COLLABORATION_SERVER_SECRET in the settings. - - Verifies that the reset-connections endpoint is called exactly once. - - Checks that the request URL and headers are correct. - - If user_id is provided, checks that the X-User-Id header is correct. + TODO(yhub): yhub has no kick API, so reset_connections is a no-op. It must + neither make any HTTP call nor raise, even without any collaboration + settings configured. """ + with responses.RequestsMock(): + CollaborationService().reset_connections("document-id") + CollaborationService().reset_connections("document-id", user_id="user-id") - @contextmanager - def _mock_reset_connections(document_id, user_id=None): - with responses.RequestsMock() as rsps: - # Mock the reset-connections endpoint - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" - endpoint_url = ( - f"{settings.COLLABORATION_API_URL}reset-connections/?room={document_id}" - ) - rsps.add( - responses.POST, - endpoint_url, - json={}, - status=200, - ) - yield - - assert len(rsps.calls) == 1, ( - "Expected one call to reset-connections endpoint" - ) - request = rsps.calls[0].request - assert request.url == endpoint_url, f"Unexpected URL called: {request.url}" - assert ( - request.headers.get("Authorization") - == settings.COLLABORATION_SERVER_SECRET - ), "Incorrect Authorization header" - - if user_id: - assert request.headers.get("X-User-Id") == user_id, ( - "Incorrect X-User-Id header" - ) - - return _mock_reset_connections - - -def test_init_without_api_url(settings): - """Test that ImproperlyConfigured is raised when COLLABORATION_API_URL is None.""" - settings.COLLABORATION_API_URL = None - with pytest.raises(ImproperlyConfigured): - CollaborationService() - - -def test_init_with_api_url(settings): - """Test that the service initializes correctly when COLLABORATION_API_URL is set.""" - settings.COLLABORATION_API_URL = "http://example.com/" - service = CollaborationService() - assert isinstance(service, CollaborationService) - - -@responses.activate -def test_reset_connection_with_user_id(settings): - """Test _reset_connection with a provided user_id.""" - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" - service = CollaborationService() - - room = "room1" - user_id = "user123" - endpoint_url = "http://example.com/reset-connections/?room=" + room - - responses.add(responses.POST, endpoint_url, json={}, status=200) - - service._reset_connection(room, user_id) - - assert len(responses.calls) == 1 - request = responses.calls[0].request - - assert request.url == endpoint_url - assert request.headers.get("Authorization") == "secret-token" - assert request.headers.get("X-User-Id") == "user123" - - -@responses.activate -def test_reset_connection_without_user_id(settings): - """Test _reset_connection without a user_id.""" - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" - service = CollaborationService() - - room = "room1" - user_id = None - endpoint_url = "http://example.com/reset-connections/?room=" + room - - responses.add( - responses.POST, - endpoint_url, - json={}, - status=200, - ) - - service._reset_connection(room, user_id) - - assert len(responses.calls) == 1 - request = responses.calls[0].request - - assert request.url == endpoint_url - assert request.headers.get("Authorization") == "secret-token" - assert request.headers.get("X-User-Id") is None - - -@responses.activate -def test_reset_connection_non_200_response(settings): - """Test that an HTTPError is raised when the response status is not 200.""" - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" - service = CollaborationService() - - room = "room1" - user_id = "user123" - endpoint_url = "http://example.com/reset-connections/?room=" + room - response_body = {"error": "Internal Server Error"} - - responses.add(responses.POST, endpoint_url, json=response_body, status=500) - - expected_exception_message = re.escape( - "Failed to notify WebSocket server. Status code: 500, Response: " - ) + re.escape(json.dumps(response_body)) - - with pytest.raises(requests.HTTPError, match=expected_exception_message): - service._reset_connection(room, user_id) - - assert len(responses.calls) == 1 - - -@responses.activate -def test_reset_connection_request_exception(settings): - """Test that an HTTPError is raised when a RequestException occurs.""" - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" - service = CollaborationService() - - room = "room1" - user_id = "user123" - endpoint_url = "http://example.com/reset-connections?room=" + room - - responses.add( - responses.POST, - endpoint_url, - body=requests.exceptions.ConnectionError("Network error"), - ) - - with pytest.raises(requests.HTTPError, match="Failed to notify WebSocket server."): - service._reset_connection(room, user_id) - - assert len(responses.calls) == 1 - -@pytest.fixture(name="collaboration_service") -def collaboration_service_fixture(settings): - """Return a configured CollaborationService instance.""" - settings.COLLABORATION_API_URL = "http://example.com/" - settings.COLLABORATION_SERVER_SECRET = "secret-token" - return CollaborationService() - - -@pytest.mark.django_db -@mock.patch.object(CollaborationService, "_reset_connection") -def test_reset_connections_document_does_not_exist( - mock_reset_connection, - collaboration_service, - caplog, -): - """ - When the document does not exist anymore, an error is logged and no - connection is reset. - """ - unknown_id = uuid4() - - with caplog.at_level(logging.ERROR, logger="core.services.collaboration_services"): - collaboration_service.reset_connections(unknown_id) - - mock_reset_connection.assert_not_called() - assert f"Document {unknown_id} does not exists anymore" in caplog.text - - -@pytest.mark.django_db -@mock.patch.object(CollaborationService, "_reset_connection") -def test_reset_connections_single_document( - mock_reset_connection, - collaboration_service, -): - """A document without descendants should have its own connections reset.""" - document = factories.DocumentFactory() - - collaboration_service.reset_connections(document.id) - - mock_reset_connection.assert_called_once_with(document.id, None) - - -@pytest.mark.django_db -@mock.patch.object(CollaborationService, "_reset_connection") -def test_reset_connections_cascade_on_document_and_descendants( - mock_reset_connection, - collaboration_service, -): - """ - The document itself and every one of its descendants should be reset, - ordered by path. - """ - root = factories.DocumentFactory() - child1 = factories.DocumentFactory(parent=root) - child2 = factories.DocumentFactory(parent=root) - grandchild = factories.DocumentFactory(parent=child1) - - collaboration_service.reset_connections(root.id) - - expected_ids = [ - doc.id - for doc in models.Document.objects.filter( - path__startswith=root.path, depth__gte=root.depth - ).order_by("path") - ] - assert set(expected_ids) == {root.id, child1.id, child2.id, grandchild.id} - - called_ids = [call.args[0] for call in mock_reset_connection.call_args_list] - assert called_ids == expected_ids - assert mock_reset_connection.call_count == 4 - - -@pytest.mark.django_db -@mock.patch.object(CollaborationService, "_reset_connection") -def test_reset_connections_starts_from_a_sub_document( - mock_reset_connection, - collaboration_service, -): - """ - When called on a sub-document, only that sub-document and its own - descendants should be reset, not its ancestors or siblings. - """ - root = factories.DocumentFactory() - child = factories.DocumentFactory(parent=root) - sibling = factories.DocumentFactory(parent=root) - grandchild = factories.DocumentFactory(parent=child) - - collaboration_service.reset_connections(child.id) - - called_ids = {call.args[0] for call in mock_reset_connection.call_args_list} - assert called_ids == {child.id, grandchild.id} - assert root.id not in called_ids - assert sibling.id not in called_ids - - -@pytest.mark.django_db -@mock.patch.object(CollaborationService, "_reset_connection") -def test_reset_connections_forwards_user_id( - mock_reset_connection, - collaboration_service, -): - """The provided user_id should be forwarded to every reset call.""" - root = factories.DocumentFactory() - factories.DocumentFactory(parent=root) - user_id = str(uuid4()) - - collaboration_service.reset_connections(root.id, user_id=user_id) - - assert mock_reset_connection.call_count == 2 - for call in mock_reset_connection.call_args_list: - assert call.args[1] == user_id - - -@pytest.mark.django_db -@mock.patch.object(CollaborationService, "_reset_connection") -def test_reset_connections_continues_on_http_error( - mock_reset_connection, - collaboration_service, - caplog, -): +def test_get_document_connection_info_makes_no_http_call(): """ - An HTTPError raised while resetting one document should be logged and must - not prevent the remaining documents from being processed. + TODO(yhub): yhub has no connection-info API, so get_document_connection_info + always reports nobody connected, without making any HTTP call. """ - root = factories.DocumentFactory() - child1 = factories.DocumentFactory(parent=root) - child2 = factories.DocumentFactory(parent=root) - - ordered_docs = list( - models.Document.objects.filter( - path__startswith=root.path, depth__gte=root.depth - ).order_by("path") - ) - failing_doc = ordered_docs[1] - - def _side_effect(room, _user_id=None): - if room == failing_doc.id: - raise requests.HTTPError("boom") - - mock_reset_connection.side_effect = _side_effect - - with caplog.at_level(logging.ERROR, logger="core.services.collaboration_services"): - collaboration_service.reset_connections(root.id) - - assert mock_reset_connection.call_count == 3 - called_ids = [call.args[0] for call in mock_reset_connection.call_args_list] - assert set(called_ids) == {root.id, child1.id, child2.id} - - assert ( - f"impossible to reset connections for document {failing_doc.id}" in caplog.text - ) + with responses.RequestsMock(): + assert CollaborationService().get_document_connection_info( + "room", "session-key" + ) == (0, False) diff --git a/src/backend/core/tests/test_tasks_access.py b/src/backend/core/tests/test_tasks_access.py index d794c6b248..b5368c453e 100644 --- a/src/backend/core/tests/test_tasks_access.py +++ b/src/backend/core/tests/test_tasks_access.py @@ -5,10 +5,6 @@ from unittest import mock -from django.core.exceptions import ImproperlyConfigured - -import pytest - from core.tasks.access import reset_service_connections_in_cascade @@ -33,16 +29,3 @@ def test_reset_service_connections_defaults_user_id_to_none(mock_service): mock_service.return_value.reset_connections.assert_called_once_with( "document-id", None ) - - -@mock.patch( - "core.tasks.access.CollaborationService", - side_effect=ImproperlyConfigured("Collaboration configuration not set"), -) -def test_reset_service_connections_propagates_improperly_configured(mock_service): # pylint: disable=unused-argument - """ - If the collaboration service is not configured, instantiating it raises - ImproperlyConfigured, which should propagate out of the task. - """ - with pytest.raises(ImproperlyConfigured): - reset_service_connections_in_cascade("document-id") diff --git a/src/backend/impress/settings.py b/src/backend/impress/settings.py index bce3ea8153..a6a2f79ff1 100755 --- a/src/backend/impress/settings.py +++ b/src/backend/impress/settings.py @@ -512,9 +512,13 @@ class Base(Configuration): SENTRY_DSN = values.Value(None, environ_name="SENTRY_DSN", environ_prefix=None) # Collaboration + # TODO(yhub): unused since the yhub migration — yhub has no management API + # (reset-connections / get-connections). Kept until a yhub kick and + # connection-info API exist and CollaborationService is reinstated. COLLABORATION_API_URL = values.Value( None, environ_name="COLLABORATION_API_URL", environ_prefix=None ) + # TODO(yhub): unused since the yhub migration, see COLLABORATION_API_URL. COLLABORATION_SERVER_SECRET = SecretFileValue( None, environ_name="COLLABORATION_SERVER_SECRET", environ_prefix=None ) diff --git a/src/frontend/apps/e2e/.env b/src/frontend/apps/e2e/.env index 1da7cdfed1..a0bd90c655 100644 --- a/src/frontend/apps/e2e/.env +++ b/src/frontend/apps/e2e/.env @@ -1,7 +1,7 @@ PORT=3000 BASE_URL=http://localhost:3000 BASE_API_URL=http://localhost:8071/api/v1.0 -COLLABORATION_WS_URL=ws://localhost:4444/collaboration/ws/ +COLLABORATION_WS_URL=ws://localhost:3002/ws/docs MEDIA_BASE_URL=http://localhost:8083 CUSTOM_SIGN_IN=false IS_INSTANCE=false diff --git a/src/frontend/apps/e2e/.env.example b/src/frontend/apps/e2e/.env.example index 52f7745dac..45272cc115 100644 --- a/src/frontend/apps/e2e/.env.example +++ b/src/frontend/apps/e2e/.env.example @@ -1,7 +1,7 @@ PORT=3000 BASE_URL=http://localhost:3000 BASE_API_URL=http://localhost:8071/api/v1.0 -COLLABORATION_WS_URL=ws://localhost:4444/collaboration/ws/ +COLLABORATION_WS_URL=ws://localhost:3002/ws/docs MEDIA_BASE_URL=http://localhost:8083 IS_INSTANCE=false CUSTOM_SIGN_IN=false diff --git a/src/frontend/apps/e2e/__tests__/app-impress/config.spec.ts b/src/frontend/apps/e2e/__tests__/app-impress/config.spec.ts index e8dcb31cac..0cca72cbb5 100644 --- a/src/frontend/apps/e2e/__tests__/app-impress/config.spec.ts +++ b/src/frontend/apps/e2e/__tests__/app-impress/config.spec.ts @@ -82,9 +82,9 @@ test.describe('Config', () => { .click(); const webSocket = await page.waitForEvent('websocket', (webSocket) => { - return webSocket.url().includes(`${process.env.COLLABORATION_WS_URL}`); + return webSocket.url().includes(`${process.env.COLLABORATION_WS_URL}/`); }); - expect(webSocket.url()).toContain(`${process.env.COLLABORATION_WS_URL}`); + expect(webSocket.url()).toContain(`${process.env.COLLABORATION_WS_URL}/`); }); test('it checks FRONTEND_CSS_URL config', async ({ page }) => { diff --git a/src/frontend/apps/e2e/__tests__/app-impress/doc-collaboration.spec.ts b/src/frontend/apps/e2e/__tests__/app-impress/doc-collaboration.spec.ts index 9ea0fbd5d8..526e25a747 100644 --- a/src/frontend/apps/e2e/__tests__/app-impress/doc-collaboration.spec.ts +++ b/src/frontend/apps/e2e/__tests__/app-impress/doc-collaboration.spec.ts @@ -15,14 +15,10 @@ test.describe('Doc Collaboration', () => { /** * We check: * - connection to the collaborative server - * - signal of the backend to the collaborative server (connection should close) - * - reconnection to the collaborative server */ test('checks the connection with collaborative server', async ({ page }) => { - let webSocketPromise = page.waitForEvent('websocket', (webSocket) => { - return webSocket - .url() - .includes(`${process.env.COLLABORATION_WS_URL}?room=`); + const webSocketPromise = page.waitForEvent('websocket', (webSocket) => { + return webSocket.url().includes(`${process.env.COLLABORATION_WS_URL}/`); }); await page @@ -32,42 +28,21 @@ test.describe('Doc Collaboration', () => { }) .click(); - let webSocket = await webSocketPromise; - expect(webSocket.url()).toContain( - `${process.env.COLLABORATION_WS_URL}?room=`, - ); + const webSocket = await webSocketPromise; + expect(webSocket.url()).toContain(`${process.env.COLLABORATION_WS_URL}/`); // Is connected - let framesentPromise = webSocket.waitForEvent('framesent'); + const framesentPromise = webSocket.waitForEvent('framesent'); await writeInEditor({ page, text: 'Hello World' }); - let framesent = await framesentPromise; + const framesent = await framesentPromise; expect(framesent.payload).not.toBeNull(); - await page.getByRole('button', { name: 'Share' }).click(); - - const selectVisibility = page.getByTestId('doc-visibility'); - - // When the visibility is changed, the ws should close the connection (backend signal) - const wsClosePromise = webSocket.waitForEvent('close'); - - await selectVisibility.click(); - await page.getByRole('menuitemradio', { name: 'Connected' }).click(); - - // Assert that the doc reconnects to the ws - const wsClose = await wsClosePromise; - expect(wsClose.isClosed()).toBeTruthy(); - - // Check the ws is connected again - webSocket = await page.waitForEvent('websocket', (webSocket) => { - return webSocket - .url() - .includes(`${process.env.COLLABORATION_WS_URL}?room=`); - }); - framesentPromise = webSocket.waitForEvent('framesent'); - framesent = await framesentPromise; - expect(framesent.payload).not.toBeNull(); + // TODO(yhub): re-add the close/reconnect check (the backend closed the + // connection when the doc visibility changed) once yhub exposes a kick + // API - `reset_connections` is currently a no-op so the server never + // closes the connection. }); test('it cannot edit if viewer but see and can get resources', async ({ @@ -136,20 +111,24 @@ test.describe('Doc Collaboration', () => { await cleanup(); }); - test('it checks block editing when not connected to collab server', async ({ + // TODO(yhub): re-enable when yhub exposes a connection-info API - the test + // asserts `can_edit=false` while another user is connected to the + // collaborative server, but `get_document_connection_info` is currently + // stubbed to report no connections. + test.skip('it checks block editing when not connected to collab server', async ({ page, browserName, }) => { test.slow(); /** - * The good port is 4444, but we want to simulate a not connected + * The good port is 3002, but we want to simulate a not connected * collaborative server. * So we use a port that is not used by the collaborative server. * The server will not be able to connect to the collaborative server. */ await overrideConfig(page, { - COLLABORATION_WS_URL: 'ws://localhost:5555/collaboration/ws/', + COLLABORATION_WS_URL: 'ws://localhost:5555/ws/docs', COLLABORATION_WS_NOT_CONNECTED_READ_ONLY: true, }); @@ -211,18 +190,14 @@ test.describe('Doc Collaboration', () => { const webSocketPromise = otherPage.waitForEvent( 'websocket', (webSocket) => { - return webSocket - .url() - .includes(`${process.env.COLLABORATION_WS_URL}?room=`); + return webSocket.url().includes(`${process.env.COLLABORATION_WS_URL}/`); }, ); await otherPage.goto(urlChildDoc); const webSocket = await webSocketPromise; - expect(webSocket.url()).toContain( - `${process.env.COLLABORATION_WS_URL}?room=`, - ); + expect(webSocket.url()).toContain(`${process.env.COLLABORATION_WS_URL}/`); await verifyDocName(otherPage, childTitle); @@ -288,9 +263,7 @@ test.describe('Doc Collaboration', () => { await page.goto('/'); let webSocketPromise = page.waitForEvent('websocket', (webSocket) => { - return webSocket - .url() - .includes(`${process.env.COLLABORATION_WS_URL}?room=`); + return webSocket.url().includes(`${process.env.COLLABORATION_WS_URL}/`); }); await page @@ -301,9 +274,7 @@ test.describe('Doc Collaboration', () => { .click(); let webSocket = await webSocketPromise; - expect(webSocket.url()).toContain( - `${process.env.COLLABORATION_WS_URL}?room=`, - ); + expect(webSocket.url()).toContain(`${process.env.COLLABORATION_WS_URL}/`); // Is connected let framesentPromise = webSocket.waitForEvent('framesent'); @@ -332,9 +303,7 @@ test.describe('Doc Collaboration', () => { // Check the ws is connected again webSocketPromise = page.waitForEvent('websocket', (webSocket) => { - return webSocket - .url() - .includes(`${process.env.COLLABORATION_WS_URL}?room=`); + return webSocket.url().includes(`${process.env.COLLABORATION_WS_URL}/`); }); // Simulate the tab becoming visible again diff --git a/src/frontend/apps/impress/package.json b/src/frontend/apps/impress/package.json index 030ee6eecb..b86be1cf9d 100644 --- a/src/frontend/apps/impress/package.json +++ b/src/frontend/apps/impress/package.json @@ -43,7 +43,6 @@ "@gouvfr-lasuite/cunningham-react": "*", "@gouvfr-lasuite/integration": "1.0.3", "@gouvfr-lasuite/ui-kit": "0.28.0", - "@hocuspocus/provider": "3.4.4", "@lottiefiles/dotlottie-react": "^0.19.6", "@mantine/core": "9.5.0", "@mantine/hooks": "9.5.0", @@ -78,6 +77,7 @@ "use-debounce": "10.1.1", "uuid": "14.0.1", "y-protocols": "1.0.7", + "y-websocket": "3.0.0", "yjs": "*", "zod": "4.4.3", "zustand": "5.0.14" diff --git a/src/frontend/apps/impress/src/core/config/hooks/useCollaborationUrl.tsx b/src/frontend/apps/impress/src/core/config/hooks/useCollaborationUrl.tsx index b06683729b..feee4ab031 100644 --- a/src/frontend/apps/impress/src/core/config/hooks/useCollaborationUrl.tsx +++ b/src/frontend/apps/impress/src/core/config/hooks/useCollaborationUrl.tsx @@ -7,11 +7,12 @@ export const useCollaborationUrl = (room?: string) => { return; } - const base = + // The room is appended to the base URL by the provider (y-websocket) + return ( conf?.COLLABORATION_WS_URL || (typeof window !== 'undefined' - ? `wss://${window.location.host}/collaboration/ws/` - : ''); - - return `${base}?room=${room}`; + ? // TODO(yhub): no prod ingress route yet + `wss://${window.location.host}/ws/docs` + : '') + ); }; diff --git a/src/frontend/apps/impress/src/features/docs/doc-comments/hooks/useComments.ts b/src/frontend/apps/impress/src/features/docs/doc-comments/hooks/useComments.ts index 1a65022413..5903d39768 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-comments/hooks/useComments.ts +++ b/src/frontend/apps/impress/src/features/docs/doc-comments/hooks/useComments.ts @@ -30,13 +30,13 @@ export function useComments( canComment, config?.REACTIONS_MAX_PER_COMMENT ?? 0, ), - provider?.document, + provider?.doc, ); }, [ docId, canComment, provider?.awareness, - provider?.document, + provider?.doc, user?.full_name, config?.REACTIONS_MAX_PER_COMMENT, ]); diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/__tests__/DocEditor.spec.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/__tests__/DocEditor.spec.tsx index 3dbbb33fbe..e27870b94d 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-editor/__tests__/DocEditor.spec.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/__tests__/DocEditor.spec.tsx @@ -28,8 +28,8 @@ vi.mock('../../doc-management', async () => { useIsCollaborativeEditable: () => ({ isEditable: true, isLoading: false }), useProviderStore: () => ({ provider: { - configuration: { name: 'test-doc-id' }, - document: { + roomname: 'test-doc-id', + doc: { getXmlFragment: () => null, }, }, diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteEditor.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteEditor.tsx index caa8a91bb0..894cf006b3 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteEditor.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteEditor.tsx @@ -17,11 +17,10 @@ import { ThreadsSidebar, useCreateBlockNote, } from '@blocknote/react'; -import { HocuspocusProvider } from '@hocuspocus/provider'; import { useEffect, useMemo, useRef } from 'react'; import { createPortal } from 'react-dom'; import { useTranslation } from 'react-i18next'; -import type { Awareness } from 'y-protocols/awareness'; +import { WebsocketProvider } from 'y-websocket'; import * as Y from 'yjs'; import { Box, TextErrors } from '@/components'; @@ -86,7 +85,7 @@ export const blockNoteSchema = (withMultiColumn?.(baseBlockNoteSchema) || interface BlockNoteEditorProps { doc: Doc; - provider: HocuspocusProvider; + provider: WebsocketProvider; } export const BlockNoteEditor = ({ doc, provider }: BlockNoteEditorProps) => { @@ -94,7 +93,7 @@ export const BlockNoteEditor = ({ doc, provider }: BlockNoteEditorProps) => { const { setEditor } = useEditorStore(); const { themeTokens } = useCunninghamTheme(); const refEditorContainer = useRef(null); - useSaveDoc(doc.id, provider.document); + useSaveDoc(doc.id, provider.doc); const { i18n, t } = useTranslation(); const langLocalesBN = @@ -151,8 +150,8 @@ export const BlockNoteEditor = ({ doc, provider }: BlockNoteEditorProps) => { const editor: DocsBlockNoteEditor = useCreateBlockNote( { collaboration: { - provider: provider as { awareness?: Awareness | undefined }, - fragment: provider.document.getXmlFragment('document-store'), + provider, + fragment: provider.doc.getXmlFragment('document-store'), user: { name: cursorName, color: randomColor(), diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/components/DocEditor.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/components/DocEditor.tsx index 96626f5c3a..f24db166bc 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-editor/components/DocEditor.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/components/DocEditor.tsx @@ -142,16 +142,10 @@ interface DocCoreEditorProps { export const DocCoreEditor = ({ doc, readOnly }: DocCoreEditorProps) => { const { provider, isReady } = useProviderStore(); const isProviderReady = isReady && provider; - const showContent = !!( - isProviderReady && provider?.configuration.name === doc.id - ); + const showContent = !!(isProviderReady && provider?.roomname === doc.id); const { skeletonVisible, isFadingOut } = useSkeletonFadeOut(showContent); - if ( - skeletonVisible || - !isProviderReady || - provider?.configuration.name !== doc.id - ) { + if (skeletonVisible || !isProviderReady || provider?.roomname !== doc.id) { return ( { if (readOnly) { return ( ); diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useCollaboration.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useCollaboration.tsx index e32c41515f..150b7c07bf 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useCollaboration.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useCollaboration.tsx @@ -56,6 +56,9 @@ export const useCollaboration = (room: string) => { * When the provider detects a lost connection, we invalidate the document query to trigger a refetch. * Because it can be because the user has access to the document that are modified * (e.g., permissions changed, document deleted, user removed) + * TODO(yhub): this invalidation used to ride on the server-side kick + * (reset-connections); without a kick API a permission change no longer + * triggers a refetch until the connection drops for another reason. */ useEffect(() => { if (hasLostConnection && room) { @@ -71,7 +74,7 @@ export const useCollaboration = (room: string) => { * when the document visibility changes. */ useEffect(() => { - if (!room || broadcastProvider?.document?.guid !== room) { + if (!room || broadcastProvider?.doc.guid !== room) { return; } @@ -80,7 +83,7 @@ export const useCollaboration = (room: string) => { queryKey: [KEY_DOC, { id: room }], }); }); - }, [addTask, room, queryClient, broadcastProvider?.document?.guid]); + }, [addTask, room, queryClient, broadcastProvider?.doc.guid]); /** * Set the provider when the collaboration URL and the document content are available. diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useSaveDoc.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useSaveDoc.tsx index b6ceb0230b..2edd76424a 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useSaveDoc.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useSaveDoc.tsx @@ -1,5 +1,6 @@ import { useRouter } from 'next/router'; import { useCallback, useEffect, useRef, useState } from 'react'; +import { WebsocketProvider } from 'y-websocket'; import * as Y from 'yjs'; import { useDocContentUpdate } from '@/docs/doc-management/api/useDocContentUpdate'; @@ -49,22 +50,19 @@ export const useSaveDoc = (docId: string, yDoc: Y.Doc) => { ) => { /** * When the AI edit the doc transaction.local is false, - * so we check if the origin constructor to know where + * so we check the transaction origin to know where * the transaction comes from. - * "PluginKey" constructor comes from the current user, but transaction.local is more reliable - * "HocuspocusProvider" constructor comes from other users from the collaboration server, - * it seems quite reliable too. - * The AI constructor name seems to not be reliable enough, but by deduction if it's not local + * "PluginKey" origin comes from the current user, but transaction.local is more reliable + * Updates from other users are applied by the collaboration server with + * the provider instance as origin, it seems quite reliable too. + * The AI origin seems to not be reliable enough, but by deduction if it's not local * and not from other users, it has to be from the AI. * * TODO: see if we can get the local changes from the AI */ - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - const transactionOrigin = transaction?.origin?.constructor?.name; - const PROVIDER_ORIGIN_CONSTRUCTOR = 'HocuspocusProvider'; - const isAIChange = - !transaction.local && transactionOrigin !== PROVIDER_ORIGIN_CONSTRUCTOR; + !transaction.local && + !(transaction.origin instanceof WebsocketProvider); /** * notifySubscribers generate a transaction that can be diff --git a/src/frontend/apps/impress/src/features/docs/doc-management/api/useDuplicateDoc.tsx b/src/frontend/apps/impress/src/features/docs/doc-management/api/useDuplicateDoc.tsx index 96ff439431..10f6165467 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-management/api/useDuplicateDoc.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-management/api/useDuplicateDoc.tsx @@ -70,14 +70,12 @@ export function useDuplicateDoc(options?: DuplicateDocOptions) { mutationFn: async (variables) => { // Save the document if we can first, to ensure the latest state is duplicated const canSave = - variables.canSave && - provider && - provider.document.guid === variables.docId; + variables.canSave && provider && provider.doc.guid === variables.docId; if (canSave) { await updateDocContent({ id: variables.docId, - content: toBase64(Y.encodeStateAsUpdate(provider.document)), + content: toBase64(Y.encodeStateAsUpdate(provider.doc)), }); } diff --git a/src/frontend/apps/impress/src/features/docs/doc-management/api/useUpdateDoc.tsx b/src/frontend/apps/impress/src/features/docs/doc-management/api/useUpdateDoc.tsx index 63791ea889..5a9cf397ed 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-management/api/useUpdateDoc.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-management/api/useUpdateDoc.tsx @@ -6,11 +6,13 @@ import { import { APIError, errorCauses, fetchAPI } from '@/api'; +import { useProviderStore } from '../stores'; import { Doc } from '../types'; export interface UpdateDocParams { id: Doc['id']; title?: string; + websocket?: boolean; } export const updateDoc = async ({ @@ -38,7 +40,16 @@ type UseUpdateDoc = UseMutationOptions & { export function useUpdateDoc(queryConfig?: UseUpdateDoc) { const queryClient = useQueryClient(); return useMutation({ - mutationFn: updateDoc, + /** + * Tell the backend when we hold a live collaboration connection, + * otherwise its no-websocket cache lock blocks the update while + * another user is connected. + */ + mutationFn: (params) => + updateDoc({ + ...(useProviderStore.getState().isSynced ? { websocket: true } : {}), + ...params, + }), ...queryConfig, onSuccess: (data, variables, onMutateResult, context) => { queryConfig?.listInvalidQueries?.forEach((queryKey) => { diff --git a/src/frontend/apps/impress/src/features/docs/doc-management/stores/useProviderStore.tsx b/src/frontend/apps/impress/src/features/docs/doc-management/stores/useProviderStore.tsx index 5411cee6fb..cba10a6388 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-management/stores/useProviderStore.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-management/stores/useProviderStore.tsx @@ -1,5 +1,4 @@ -import { CloseEvent } from '@hocuspocus/common'; -import { HocuspocusProvider, WebSocketStatus } from '@hocuspocus/provider'; +import { WebsocketProvider } from 'y-websocket'; import * as Y from 'yjs'; import { create } from 'zustand'; @@ -10,12 +9,12 @@ export interface UseCollaborationStore { providerUrl: string, storeId: string, initialDoc?: Base64, - ) => HocuspocusProvider; + ) => WebsocketProvider; destroyProvider: () => void; setReady: (value: boolean) => void; pauseForInactivity: () => void; resumeFromInactivity: () => void; - provider: HocuspocusProvider | undefined; + provider: WebsocketProvider | undefined; isConnected: boolean; isReady: boolean; isSynced: boolean; @@ -33,18 +32,14 @@ const defaultValues = { isPausedForInactivity: false, }; -type ExtendedCloseEvent = CloseEvent & { wasClean: boolean }; - /** * When a massive simultaneous disconnection occurs (e.g. infra restart), all * clients would reconnect and invalidate their queries at exactly the same * time, causing a possible DB spike. Adding random jitter spreads these events over a * time window so the load is absorbed gradually. */ -const RECONNECT_BASE_DELAY_MS = 1000; const RECONNECT_JITTER_MAX_MS = 3000; -let reconnectTimeout: ReturnType | undefined; let lostConnectionTimeout: ReturnType | undefined; export const useProviderStore = create((set, get) => ({ @@ -58,104 +53,54 @@ export const useProviderStore = create((set, get) => ({ Y.applyUpdate(doc, Buffer.from(initialDoc, 'base64')); } - const provider = new HocuspocusProvider({ - url: wsUrl, - name: storeId, - document: doc, - onDisconnect(data) { - // Skip reconnect when the disconnect was triggered by inactivity: - // reconnection only happens once the user becomes active again. - if (get().isPausedForInactivity) { - return; - } - - // Attempt to reconnect if the disconnection was clean (initiated by the client or server) - if ((data.event as ExtendedCloseEvent).wasClean) { - if (data.event.reason === 'No cookies' && data.event.code === 4001) { - console.error( - 'Disconnection due to missing cookies. Not attempting to reconnect.', - ); - void provider.disconnect(); - set({ - isReady: true, - isConnected: false, - }); - return; - } - - clearTimeout(reconnectTimeout); - - // Jitter spreading for reconnection attempts - // Math.random() generates a random delay to avoid all clients - // reconnecting at the same time - reconnectTimeout = setTimeout( - () => void provider.connect(), - RECONNECT_BASE_DELAY_MS + Math.random() * RECONNECT_JITTER_MAX_MS, - ); - } - }, - onAuthenticationFailed() { - set({ isReady: true, isConnected: false }); - }, - onAuthenticated() { - set({ isReady: true, isConnected: true }); - }, - onStatus: ({ status }) => { - const isConnected = status === WebSocketStatus.Connected; - const wasConnected = get().isConnected; - - if (isConnected) { - clearTimeout(lostConnectionTimeout); - } - // If we were previously connected and now we're not, - // we might have lost the connection - else if (wasConnected && !get().isPausedForInactivity) { - clearTimeout(lostConnectionTimeout); - // Jitter spreading for reconnection attempts - // Math.random() generates a random delay to avoid all clients - // reconnecting at the same time - lostConnectionTimeout = setTimeout( - () => set({ hasLostConnection: true }), - Math.random() * RECONNECT_JITTER_MAX_MS, - ); - } - - set((state) => { - /** - * status === WebSocketStatus.Connected does not mean we are totally connected - * because authentication can still be in progress and failed - * So we only update isConnected when we lose the connection - */ - const connected = - status !== WebSocketStatus.Connected - ? { - isConnected: false, - } - : undefined; - - return { - ...connected, - isReady: state.isReady || status === WebSocketStatus.Disconnected, - }; - }); - }, - onSynced: ({ state }) => { - set({ isSynced: state, isReady: true }); - }, - onClose(data) { - /** - * Handle the "Reset Connection" event from the server - * This is triggered when the server wants to reset the connection - * for clients in the room. - * A disconnect is made automatically but it takes time to be triggered, - * so we force the disconnection here. - */ - if (data.event.code === 1000) { - provider.disconnect(); - } - }, + const provider = new WebsocketProvider(wsUrl, storeId, doc, { + // BroadcastChannel would bypass server auth + disableBc: true, + // The default 2.5s backoff would hammer the backend with auth fetches + // on permanently-failing sockets + maxBackoffTime: 30000, + // Guarantees inbound traffic for y-websocket's 30s no-traffic watchdog + resyncInterval: 20000, + }); + + provider.on('status', ({ status }) => { + // 'connecting' must be ignored: it fires on every backoff retry. + // 'disconnected' is handled via 'connection-close' (it never fires + // for sockets that failed to open). + if (status === 'connected') { + clearTimeout(lostConnectionTimeout); + // An open socket means we are authenticated (auth happens at upgrade) + set({ isConnected: true, isReady: true }); + } + }); + + provider.on('sync', (isSynced: boolean) => { + set({ isSynced, isReady: true }); }); + // Fires on every close AND every failed connection attempt + // (an auth failure surfaces as an upgrade-level 401, close code 1006). + provider.on('connection-close', () => { + // Skip when the disconnect was triggered by inactivity: + // reconnection only happens once the user becomes active again. + if (get().isPausedForInactivity) { + return; + } + + // The editor renders from the last snapshot while y-websocket retries + set({ isConnected: false, isReady: true }); + + clearTimeout(lostConnectionTimeout); + // Jitter spreading: Math.random() generates a random delay to avoid + // all clients invalidating their queries at the same time + lostConnectionTimeout = setTimeout( + () => set({ hasLostConnection: true }), + Math.random() * RECONNECT_JITTER_MAX_MS, + ); + }); + + // TODO(yhub): re-add kick handling when yhub exposes a kick API (was onClose code 1000). + set({ provider, }); @@ -163,12 +108,19 @@ export const useProviderStore = create((set, get) => ({ return provider; }, destroyProvider: () => { - clearTimeout(reconnectTimeout); - clearTimeout(lostConnectionTimeout); const provider = get().provider; if (provider) { + /** + * destroy() emits 'connection-close' synchronously before removing + * listeners, which re-arms lostConnectionTimeout: it must be cleared + * after, or a stale "connection lost" banner flashes on the next doc. + */ provider.destroy(); + // y-websocket never destroys the awareness: its interval would leak + provider.awareness.destroy(); + provider.doc.destroy(); } + clearTimeout(lostConnectionTimeout); set(defaultValues); }, @@ -177,7 +129,6 @@ export const useProviderStore = create((set, get) => ({ if (get().isPausedForInactivity) { return; } - clearTimeout(reconnectTimeout); clearTimeout(lostConnectionTimeout); set({ isPausedForInactivity: true, hasLostConnection: false }); get().provider?.disconnect(); @@ -188,7 +139,7 @@ export const useProviderStore = create((set, get) => ({ } clearTimeout(lostConnectionTimeout); set({ isPausedForInactivity: false }); - void get().provider?.connect(); + get().provider?.connect(); }, resetLostConnection: () => set({ hasLostConnection: false }), })); diff --git a/src/frontend/apps/impress/src/features/docs/doc-versioning/components/ModalConfirmationVersion.tsx b/src/frontend/apps/impress/src/features/docs/doc-versioning/components/ModalConfirmationVersion.tsx index 851c541eb1..73b2fb9745 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-versioning/components/ModalConfirmationVersion.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-versioning/components/ModalConfirmationVersion.tsx @@ -58,11 +58,7 @@ export const ModalConfirmationVersion = ({ return; } - revertUpdate( - provider.document, - provider.document, - base64ToYDoc(version.content), - ); + revertUpdate(provider.doc, provider.doc, base64ToYDoc(version.content)); threadStore?.refreshThreads(); diff --git a/src/frontend/apps/impress/src/features/right-panel/components/RightPanel.tsx b/src/frontend/apps/impress/src/features/right-panel/components/RightPanel.tsx index c4a265ac2d..085b41303a 100644 --- a/src/frontend/apps/impress/src/features/right-panel/components/RightPanel.tsx +++ b/src/frontend/apps/impress/src/features/right-panel/components/RightPanel.tsx @@ -19,8 +19,7 @@ export const RightPanel = () => { const { setIsPanelOpen, isPanelOpen, activePanel } = useRightPanelStore(); const { isMobile } = useResponsiveStore(); const { provider, isReady } = useProviderStore(); - const isProviderReady = - isReady && provider && provider?.configuration.name === doc?.id; + const isProviderReady = isReady && provider && provider?.roomname === doc?.id; const { restoreFocus } = useFocusStore(); /** diff --git a/src/frontend/apps/impress/src/stores/useBroadcastStore.tsx b/src/frontend/apps/impress/src/stores/useBroadcastStore.tsx index 3876dcd023..94864410ed 100644 --- a/src/frontend/apps/impress/src/stores/useBroadcastStore.tsx +++ b/src/frontend/apps/impress/src/stores/useBroadcastStore.tsx @@ -1,4 +1,4 @@ -import { HocuspocusProvider } from '@hocuspocus/provider'; +import { WebsocketProvider } from 'y-websocket'; import * as Y from 'yjs'; import { create } from 'zustand'; @@ -6,10 +6,10 @@ interface BroadcastState { addTask: (taskLabel: string, action: () => void) => void; broadcast: (taskLabel: string) => void; cleanupBroadcast: () => void; - getBroadcastProvider: () => HocuspocusProvider | undefined; - handleProviderSync: () => void; - provider?: HocuspocusProvider; - setBroadcastProvider: (provider: HocuspocusProvider) => void; + getBroadcastProvider: () => WebsocketProvider | undefined; + handleProviderSync: (isSynced: boolean) => void; + provider?: WebsocketProvider; + setBroadcastProvider: (provider: WebsocketProvider) => void; setTask: ( taskLabel: string, task: Y.Array, @@ -34,13 +34,18 @@ export const useBroadcastStore = create((set, get) => ({ // Clean up old provider listeners const oldProvider = get().provider; if (oldProvider) { - oldProvider.off('synced', get().handleProviderSync); + oldProvider.off('sync', get().handleProviderSync); } - provider.on('synced', get().handleProviderSync); + provider.on('sync', get().handleProviderSync); set({ provider }); }, - handleProviderSync: () => { + handleProviderSync: (isSynced) => { + // 'sync' fires on both edges; only re-register the tasks once synced + if (!isSynced) { + return; + } + const tasks = get().tasks; Object.entries(tasks).forEach(([taskLabel, { action }]) => { get().addTask(taskLabel, action); @@ -61,10 +66,16 @@ export const useBroadcastStore = create((set, get) => ({ return; } - const task = provider.document.getArray(taskLabel); + const task = provider.doc.getArray(taskLabel); get().setTask(taskLabel, task, action); }, setTask: (taskLabel: string, task: Y.Array, action: () => void) => { + // Unobserve the previous observer to avoid leaking one per re-registration + const previousTask = get().tasks[taskLabel]; + if (previousTask) { + previousTask.task.unobserve(previousTask.observer); + } + let isInitializing = true; const observer = ( _event: Y.YArrayEvent, @@ -102,7 +113,7 @@ export const useBroadcastStore = create((set, get) => ({ cleanupBroadcast: () => { const provider = get().provider; if (provider) { - provider.off('synced', get().handleProviderSync); + provider.off('sync', get().handleProviderSync); } // Unobserve all document-specific tasks diff --git a/src/frontend/package.json b/src/frontend/package.json index 5cf7495633..1891a2a2f4 100644 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -47,6 +47,7 @@ "sharp": "0.35.0", "typescript": "6.0.3", "wrap-ansi": "10.0.0", + "y-protocols": "1.0.7", "yjs": "13.6.31" }, "packageManager": "yarn@1.22.22" diff --git a/src/frontend/servers/y-provider/__tests__/collaborationBackend.test.ts b/src/frontend/servers/y-provider/__tests__/collaborationBackend.test.ts deleted file mode 100644 index 17c88cf0be..0000000000 --- a/src/frontend/servers/y-provider/__tests__/collaborationBackend.test.ts +++ /dev/null @@ -1,66 +0,0 @@ -import axios from 'axios'; -import { describe, expect, test, vi } from 'vitest'; - -vi.mock('../src/env', () => ({ - COLLABORATION_BACKEND_BASE_URL: 'http://app-dev:8000', - Y_PROVIDER_API_KEY: 'test-yprovider-key', -})); - -describe('CollaborationBackend', () => { - test('fetchDocument sends X-Y-Provider-Key header', async () => { - const axiosGetSpy = vi.spyOn(axios, 'get').mockResolvedValue({ - status: 200, - data: { - id: 'test-doc-id', - abilities: { retrieve: true, update: true }, - }, - }); - - const { fetchDocument } = await import('@/api/collaborationBackend'); - const documentId = 'test-document-123'; - - await fetchDocument({ name: documentId }, { cookie: 'test-cookie' }); - - expect(axiosGetSpy).toHaveBeenCalledWith( - `http://app-dev:8000/api/v1.0/documents/${documentId}/`, - expect.objectContaining({ - headers: expect.objectContaining({ - 'X-Y-Provider-Key': 'test-yprovider-key', - cookie: 'test-cookie', - }), - }), - ); - - axiosGetSpy.mockRestore(); - }); - - test('fetchCurrentUser sends X-Y-Provider-Key header', async () => { - const axiosGetSpy = vi.spyOn(axios, 'get').mockResolvedValue({ - status: 200, - data: { - id: 'test-user-id', - email: 'test@example.com', - }, - }); - - const { fetchCurrentUser } = await import('@/api/collaborationBackend'); - - await fetchCurrentUser({ - cookie: 'test-cookie', - origin: 'http://localhost:3000', - }); - - expect(axiosGetSpy).toHaveBeenCalledWith( - 'http://app-dev:8000/api/v1.0/users/me/', - expect.objectContaining({ - headers: expect.objectContaining({ - 'X-Y-Provider-Key': 'test-yprovider-key', - cookie: 'test-cookie', - origin: 'http://localhost:3000', - }), - }), - ); - - axiosGetSpy.mockRestore(); - }); -}); diff --git a/src/frontend/servers/y-provider/__tests__/collaborationResetConnections.test.ts b/src/frontend/servers/y-provider/__tests__/collaborationResetConnections.test.ts deleted file mode 100644 index da11b023c9..0000000000 --- a/src/frontend/servers/y-provider/__tests__/collaborationResetConnections.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import request from 'supertest'; -import { describe, expect, test, vi } from 'vitest'; - -vi.mock('../src/env', async (importOriginal) => { - return { - ...(await importOriginal()), - PORT: 5555, - COLLABORATION_SERVER_ORIGIN: 'http://localhost:3000', - COLLABORATION_SERVER_SECRET: 'test-secret-api-key', - }; -}); - -console.error = vi.fn(); - -import { COLLABORATION_SERVER_ORIGIN as origin } from '@/env'; -import { hocuspocusServer, initApp } from '@/servers'; - -describe('Server Tests', () => { - test('POST /collaboration/api/reset-connections?room=[ROOM_ID] with incorrect API key should return 403', async () => { - const app = initApp(); - - const response = await request(app) - .post('/collaboration/api/reset-connections/?room=test-room') - .set('Origin', origin) - .set('Authorization', 'wrong-api-key'); - - expect(response.status).toBe(401); - expect(response.body).toStrictEqual({ - error: 'Unauthorized: Invalid API Key', - }); - }); - - test('POST /collaboration/api/reset-connections?room=[ROOM_ID] failed if room not indicated', async () => { - const app = initApp(); - - const response = await request(app) - .post('/collaboration/api/reset-connections/') - .set('Origin', origin) - .set('Authorization', 'test-secret-api-key') - .send({ document_id: 'test-document' }); - - expect(response.status).toBe(400); - expect(response.body).toStrictEqual({ error: 'Room name not provided' }); - }); - - test('POST /collaboration/api/reset-connections?room=[ROOM_ID] with correct API key should reset connections', async () => { - const closeConnectionsMock = vi - .spyOn(hocuspocusServer.hocuspocus, 'closeConnections') - .mockResolvedValue(); - - const app = initApp(); - - const response = await request(app) - .post('/collaboration/api/reset-connections?room=test-room') - .set('Origin', origin) - .set('Authorization', 'test-secret-api-key'); - - expect(response.status).toBe(200); - expect(response.body).toStrictEqual({ message: 'Connections reset' }); - - expect(closeConnectionsMock).toHaveBeenCalledOnce(); - }); -}); diff --git a/src/frontend/servers/y-provider/__tests__/getDocumentConnectionInfoHandler.test.ts b/src/frontend/servers/y-provider/__tests__/getDocumentConnectionInfoHandler.test.ts deleted file mode 100644 index 7efe46c3d8..0000000000 --- a/src/frontend/servers/y-provider/__tests__/getDocumentConnectionInfoHandler.test.ts +++ /dev/null @@ -1,275 +0,0 @@ -import request from 'supertest'; -import { v4 as uuid } from 'uuid'; -import { describe, expect, test, vi } from 'vitest'; - -vi.mock('../src/env', async (importOriginal) => { - return { - ...(await importOriginal()), - PORT: 5556, - COLLABORATION_SERVER_ORIGIN: 'http://localhost:3000', - COLLABORATION_SERVER_SECRET: 'test-secret-api-key', - }; -}); - -console.error = vi.fn(); - -import { COLLABORATION_SERVER_ORIGIN as origin } from '@/env'; -import { hocuspocusServer, initApp } from '@/servers'; - -const apiEndpoint = '/collaboration/api/get-connections/'; - -describe('Server Tests', () => { - test('POST /collaboration/api/get-connections?room=[ROOM_ID] with incorrect API key should return 403', async () => { - const app = initApp(); - - const response = await request(app) - .get(`${apiEndpoint}?room=test-room`) - .set('Origin', origin) - .set('Authorization', 'wrong-api-key'); - - expect(response.status).toBe(401); - expect(response.body.error).toBe('Unauthorized: Invalid API Key'); - }); - - test('POST /collaboration/api/get-connections?room=[ROOM_ID] failed if room not indicated', async () => { - const app = initApp(); - - const response = await request(app) - .get(`${apiEndpoint}`) - .set('Origin', origin) - .set('Authorization', 'test-secret-api-key') - .send({ document_id: 'test-document' }); - - expect(response.status).toBe(400); - expect(response.body.error).toBe('Room name not provided'); - }); - - test('POST /collaboration/api/get-connections?room=[ROOM_ID] failed if session key not indicated', async () => { - const app = initApp(); - - const response = await request(app) - .get(`${apiEndpoint}?room=test-room`) - .set('Origin', origin) - .set('Authorization', 'test-secret-api-key') - .send({ document_id: 'test-document' }); - - expect(response.status).toBe(400); - expect(response.body.error).toBe('Session key not provided'); - }); - - test('POST /collaboration/api/get-connections?room=[ROOM_ID] return a 404 if room not found', async () => { - const app = initApp(); - - const response = await request(app) - .get(`${apiEndpoint}?room=test-room&sessionKey=test-session-key`) - .set('Origin', origin) - .set('Authorization', 'test-secret-api-key'); - - expect(response.status).toBe(404); - expect(response.body.error).toBe('Room not found'); - }); - - test('POST /collaboration/api/get-connections?room=[ROOM_ID] returns connection info, session key existing', async () => { - const document = await hocuspocusServer.hocuspocus.createDocument( - 'test-room', - {}, - uuid(), - { isAuthenticated: true, readOnly: false }, - {}, - ); - - document.addConnection({ - webSocket: 1, - context: { sessionKey: 'test-session-key' }, - document: document, - pongReceived: false, - readOnly: false, - request: null, - timeout: 0, - socketId: uuid(), - lock: null, - } as any); - document.addConnection({ - webSocket: 2, - context: { sessionKey: 'other-session-key' }, - document: document, - pongReceived: false, - readOnly: false, - request: null, - timeout: 0, - socketId: uuid(), - lock: null, - } as any); - document.addConnection({ - webSocket: 3, - context: { sessionKey: 'last-session-key' }, - document: document, - pongReceived: false, - readOnly: false, - request: null, - timeout: 0, - socketId: uuid(), - lock: null, - } as any); - document.addConnection({ - webSocket: 4, - context: { sessionKey: 'session-read-only' }, - document: document, - pongReceived: false, - readOnly: true, - request: null, - timeout: 0, - socketId: uuid(), - lock: null, - } as any); - - const app = initApp(); - - const response = await request(app) - .get(`${apiEndpoint}?room=test-room&sessionKey=test-session-key`) - .set('Origin', origin) - .set('Authorization', 'test-secret-api-key'); - - expect(response.status).toBe(200); - expect(response.body).toEqual({ - count: 3, - exists: true, - }); - }); - - test('POST /collaboration/api/get-connections?room=[ROOM_ID] returns connection info, session key not existing', async () => { - const document = await hocuspocusServer.hocuspocus.createDocument( - 'test-room', - {}, - uuid(), - { isAuthenticated: true, readOnly: false }, - {}, - ); - - document.addConnection({ - webSocket: 1, - context: { sessionKey: 'test-session-key' }, - document: document, - pongReceived: false, - readOnly: false, - request: null, - timeout: 0, - socketId: uuid(), - lock: null, - } as any); - document.addConnection({ - webSocket: 2, - context: { sessionKey: 'other-session-key' }, - document: document, - pongReceived: false, - readOnly: false, - request: null, - timeout: 0, - socketId: uuid(), - lock: null, - } as any); - document.addConnection({ - webSocket: 3, - context: { sessionKey: 'last-session-key' }, - document: document, - pongReceived: false, - readOnly: false, - request: null, - timeout: 0, - socketId: uuid(), - lock: null, - } as any); - document.addConnection({ - webSocket: 4, - context: { sessionKey: 'session-read-only' }, - document: document, - pongReceived: false, - readOnly: true, - request: null, - timeout: 0, - socketId: uuid(), - lock: null, - } as any); - - const app = initApp(); - - const response = await request(app) - .get(`${apiEndpoint}?room=test-room&sessionKey=non-existing-session-key`) - .set('Origin', origin) - .set('Authorization', 'test-secret-api-key'); - - expect(response.status).toBe(200); - expect(response.body).toEqual({ - count: 3, - exists: false, - }); - }); - - test('POST /collaboration/api/get-connections?room=[ROOM_ID] returns connection info, session key not existing, read only connection', async () => { - const document = await hocuspocusServer.hocuspocus.createDocument( - 'test-room', - {}, - uuid(), - { isAuthenticated: true, readOnly: false }, - {}, - ); - - document.addConnection({ - webSocket: 1, - context: { sessionKey: 'test-session-key' }, - document: document, - pongReceived: false, - readOnly: false, - request: null, - timeout: 0, - socketId: uuid(), - lock: null, - } as any); - document.addConnection({ - webSocket: 2, - context: { sessionKey: 'other-session-key' }, - document: document, - pongReceived: false, - readOnly: false, - request: null, - timeout: 0, - socketId: uuid(), - lock: null, - } as any); - document.addConnection({ - webSocket: 3, - context: { sessionKey: 'last-session-key' }, - document: document, - pongReceived: false, - readOnly: false, - request: null, - timeout: 0, - socketId: uuid(), - lock: null, - } as any); - document.addConnection({ - webSocket: 4, - context: { sessionKey: 'session-read-only' }, - document: document, - pongReceived: false, - readOnly: true, - request: null, - timeout: 0, - socketId: uuid(), - lock: null, - } as any); - - const app = initApp(); - - const response = await request(app) - .get(`${apiEndpoint}?room=test-room&sessionKey=session-read-only`) - .set('Origin', origin) - .set('Authorization', 'test-secret-api-key'); - - expect(response.status).toBe(200); - expect(response.body).toEqual({ - count: 3, - exists: false, - }); - }); -}); diff --git a/src/frontend/servers/y-provider/__tests__/hocuspocusWS.test.ts b/src/frontend/servers/y-provider/__tests__/hocuspocusWS.test.ts deleted file mode 100644 index 16d6c929d8..0000000000 --- a/src/frontend/servers/y-provider/__tests__/hocuspocusWS.test.ts +++ /dev/null @@ -1,388 +0,0 @@ -import { Server } from 'node:net'; - -import { - HocuspocusProvider, - HocuspocusProviderWebsocket, -} from '@hocuspocus/provider'; -import { v1 as uuidv1, v4 as uuidv4 } from 'uuid'; -import { - afterAll, - afterEach, - beforeAll, - describe, - expect, - test, - vi, -} from 'vitest'; -import WebSocket from 'ws'; - -const portWS = 6666; - -vi.mock('../src/env', async (importOriginal) => { - return { - ...(await importOriginal()), - PORT: 5559, - COLLABORATION_SERVER_ORIGIN: 'http://localhost:3000', - COLLABORATION_SERVER_SECRET: 'test-secret-api-key', - COLLABORATION_BACKEND_BASE_URL: 'http://app-dev:8000', - COLLABORATION_LOGGING: 'true', - }; -}); - -vi.mock('../src/api/collaborationBackend', () => ({ - fetchCurrentUser: vi.fn(), - fetchDocument: vi.fn(), -})); - -console.error = vi.fn(); -console.log = vi.fn(); - -import * as CollaborationBackend from '@/api/collaborationBackend'; -import { COLLABORATION_SERVER_ORIGIN as origin, PORT as port } from '@/env'; -import { promiseDone } from '@/helpers'; -import { hocuspocusServer, initApp } from '@/servers'; - -describe('Server Tests', () => { - let server: Server; - - afterEach(() => { - vi.clearAllMocks(); - vi.restoreAllMocks(); - }); - - beforeAll(async () => { - server = initApp().listen(port); - await hocuspocusServer.listen(portWS); - }); - - afterAll(() => { - void hocuspocusServer.destroy(); - server.close(); - }); - - test('WebSocket connection with bad origin should be closed', () => { - const { promise, done } = promiseDone(); - const room = uuidv4(); - const ws = new WebSocket(`ws://localhost:${port}/?room=${room}`, { - headers: { - Origin: 'http://bad-origin.com', - }, - }); - - ws.onclose = () => { - expect(ws.readyState).toBe(ws.CLOSED); - done(); - }; - - return promise; - }); - - test('WebSocket connection without cookies header should be closed', () => { - const { promise, done } = promiseDone(); - const room = uuidv4(); - const ws = new WebSocket(`ws://localhost:${port}/?room=${room}`, { - headers: { - Origin: origin, - }, - }); - - ws.onclose = () => { - expect(ws.readyState).toBe(ws.CLOSED); - done(); - }; - - return promise; - }); - - test('WebSocket connection not allowed if room not matching provider name', () => { - const { promise, done } = promiseDone(); - const room = uuidv4(); - const wsHocus = new HocuspocusProviderWebsocket({ - url: `ws://localhost:${portWS}/?room=${room}`, - WebSocketPolyfill: WebSocket, - maxAttempts: 1, - }); - - const providerName = uuidv4(); - const provider = new HocuspocusProvider({ - websocketProvider: wsHocus, - name: providerName, - onAuthenticationFailed(data) { - expect(console.log).toHaveBeenCalledWith( - expect.any(String), - ' --- ', - 'Invalid room name - Probable hacking attempt:', - providerName, - room, - ); - - wsHocus.stopConnectionAttempt(); - expect(data.reason).toBe('permission-denied'); - wsHocus.webSocket?.close(); - wsHocus.disconnect(); - provider.destroy(); - wsHocus.destroy(); - done(); - }, - }); - - provider.attach(); - - return promise; - }); - - test('WebSocket connection not allowed if room is not a valid uuid v4', () => { - const { promise, done } = promiseDone(); - const room = uuidv1(); - const wsHocus = new HocuspocusProviderWebsocket({ - url: `ws://localhost:${portWS}/?room=${room}`, - WebSocketPolyfill: WebSocket, - maxAttempts: 1, - }); - - const provider = new HocuspocusProvider({ - websocketProvider: wsHocus, - name: room, - onAuthenticationFailed: (data) => { - expect(console.log).toHaveBeenLastCalledWith( - expect.any(String), - ' --- ', - 'Room name is not a valid uuid:', - room, - ); - - wsHocus.stopConnectionAttempt(); - expect(data.reason).toBe('permission-denied'); - wsHocus.webSocket?.close(); - wsHocus.disconnect(); - provider.destroy(); - wsHocus.destroy(); - done(); - }, - }); - - provider.attach(); - - return promise; - }); - - test('WebSocket connection not allowed if room is not a valid uuid', () => { - const { promise, done } = promiseDone(); - const room = 'not-a-valid-uuid'; - const wsHocus = new HocuspocusProviderWebsocket({ - url: `ws://localhost:${portWS}/?room=${room}`, - WebSocketPolyfill: WebSocket, - maxAttempts: 1, - }); - - const provider = new HocuspocusProvider({ - websocketProvider: wsHocus, - name: room, - onAuthenticationFailed: (data) => { - expect(console.log).toHaveBeenLastCalledWith( - expect.any(String), - ' --- ', - 'Room name is not a valid uuid:', - room, - ); - - wsHocus.stopConnectionAttempt(); - expect(data.reason).toBe('permission-denied'); - wsHocus.webSocket?.close(); - wsHocus.disconnect(); - provider.destroy(); - wsHocus.destroy(); - done(); - }, - }); - - provider.attach(); - - return promise; - }); - - test('WebSocket connection fails if user can not access document', () => { - const { promise, done } = promiseDone(); - - const room = uuidv4(); - - const fetchDocumentMock = vi - .spyOn(CollaborationBackend, 'fetchDocument') - .mockRejectedValue(new Error('some error')); - - const wsHocus = new HocuspocusProviderWebsocket({ - url: `ws://localhost:${portWS}/?room=${room}`, - WebSocketPolyfill: WebSocket, - maxAttempts: 1, - }); - - const provider = new HocuspocusProvider({ - websocketProvider: wsHocus, - name: room, - onAuthenticationFailed: (data) => { - expect(console.error).toHaveBeenLastCalledWith( - '[onConnect]', - 'Backend error: Unauthorized', - ); - - wsHocus.stopConnectionAttempt(); - expect(data.reason).toBe('permission-denied'); - expect(fetchDocumentMock).toHaveBeenCalledExactlyOnceWith( - { name: room }, - expect.any(Object), - ); - wsHocus.webSocket?.close(); - wsHocus.disconnect(); - provider.destroy(); - wsHocus.destroy(); - done(); - }, - }); - - provider.attach(); - - return promise; - }); - - test('WebSocket connection fails if user do not have correct retrieve ability', () => { - const { promise, done } = promiseDone(); - - const room = uuidv4(); - - const fetchDocumentMock = vi - .spyOn(CollaborationBackend, 'fetchDocument') - .mockResolvedValue({ abilities: { retrieve: false } } as any); - - const wsHocus = new HocuspocusProviderWebsocket({ - url: `ws://localhost:${portWS}/?room=${room}`, - WebSocketPolyfill: WebSocket, - maxAttempts: 1, - }); - - const provider = new HocuspocusProvider({ - websocketProvider: wsHocus, - name: room, - onAuthenticationFailed: (data) => { - expect(console.log).toHaveBeenLastCalledWith( - expect.any(String), - ' --- ', - 'onConnect: Unauthorized to retrieve this document', - room, - ); - - wsHocus.stopConnectionAttempt(); - expect(data.reason).toBe('permission-denied'); - expect(fetchDocumentMock).toHaveBeenCalledExactlyOnceWith( - { name: room }, - expect.any(Object), - ); - wsHocus.webSocket?.close(); - wsHocus.disconnect(); - provider.destroy(); - wsHocus.destroy(); - done(); - }, - }); - - provider.attach(); - - return promise; - }); - - [true, false].forEach((canEdit) => { - test(`WebSocket connection ${canEdit ? 'can' : 'can not'} edit document`, () => { - const { promise, done } = promiseDone(); - - const fetchDocumentMock = vi - .spyOn(CollaborationBackend, 'fetchDocument') - .mockResolvedValue({ - abilities: { retrieve: true, update: canEdit }, - } as any); - - const room = uuidv4(); - const wsHocus = new HocuspocusProviderWebsocket({ - url: `ws://localhost:${portWS}/?room=${room}`, - WebSocketPolyfill: WebSocket, - }); - - const provider = new HocuspocusProvider({ - websocketProvider: wsHocus, - name: room, - onConnect: () => { - void hocuspocusServer.hocuspocus - .openDirectConnection(room) - .then((connection) => { - connection.document?.getConnections().forEach((connection) => { - expect(connection.readOnly).toBe(!canEdit); - }); - - void connection.disconnect(); - - provider.destroy(); - wsHocus.destroy(); - - expect(fetchDocumentMock).toHaveBeenCalledWith( - { name: room }, - expect.any(Object), - ); - - done(); - }); - }, - }); - - provider.attach(); - - return promise; - }); - }); - - test('Add request header x-user-id if found', () => { - const { promise, done } = promiseDone(); - - const fetchDocumentMock = vi - .spyOn(CollaborationBackend, 'fetchDocument') - .mockResolvedValue({ - abilities: { retrieve: true, update: true }, - } as any); - - const fetchCurrentUserMock = vi - .spyOn(CollaborationBackend, 'fetchCurrentUser') - .mockResolvedValue({ id: 'test-user-id' } as any); - - const room = uuidv4(); - const wsHocus = new HocuspocusProviderWebsocket({ - url: `ws://localhost:${portWS}/?room=${room}`, - WebSocketPolyfill: WebSocket, - }); - - const provider = new HocuspocusProvider({ - websocketProvider: wsHocus, - name: room, - onConnect: () => { - const document = hocuspocusServer.hocuspocus.documents.get(room); - if (document) { - document.getConnections().forEach((connection) => { - expect(connection.context.userId).toBe('test-user-id'); - }); - } - - provider.destroy(); - wsHocus.destroy(); - - expect(fetchDocumentMock).toHaveBeenCalledWith( - { name: room }, - expect.any(Object), - ); - - expect(fetchCurrentUserMock).toHaveBeenCalled(); - - done(); - }, - }); - - provider.attach(); - - return promise; - }); -}); diff --git a/src/frontend/servers/y-provider/package.json b/src/frontend/servers/y-provider/package.json index cee7738f2c..f11df4b152 100644 --- a/src/frontend/servers/y-provider/package.json +++ b/src/frontend/servers/y-provider/package.json @@ -18,26 +18,18 @@ "dependencies": { "@blocknote/core": "0.51.4", "@blocknote/server-util": "0.51.4", - "@hocuspocus/server": "3.4.4", "@sentry/node": "10.69.0", "@sentry/profiling-node": "10.69.0", "@tiptap/extensions": "*", - "axios": "1.18.1", "cors": "2.8.6", "express": "5.2.1", - "express-ws": "5.0.2", - "uuid": "14.0.1", - "y-protocols": "1.0.7", "yjs": "*" }, "devDependencies": { - "@hocuspocus/provider": "3.4.4", "@types/cors": "2.8.19", "@types/express": "5.0.6", - "@types/express-ws": "3.0.6", "@types/node": "*", "@types/supertest": "7.2.1", - "@types/ws": "8.18.1", "cross-env": "10.1.0", "eslint-plugin-docs": "*", "nodemon": "3.1.14", @@ -46,8 +38,7 @@ "tsc-alias": "1.9.1", "typescript": "*", "vitest": "4.1.10", - "vitest-mock-extended": "5.1.0", - "ws": "8.21.1" + "vitest-mock-extended": "5.1.0" }, "packageManager": "yarn@1.22.22" } diff --git a/src/frontend/servers/y-provider/src/api/collaborationBackend.ts b/src/frontend/servers/y-provider/src/api/collaborationBackend.ts deleted file mode 100644 index a9ae76b247..0000000000 --- a/src/frontend/servers/y-provider/src/api/collaborationBackend.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { IncomingHttpHeaders } from 'http'; - -import axios from 'axios'; - -import { COLLABORATION_BACKEND_BASE_URL, Y_PROVIDER_API_KEY } from '@/env'; - -export interface User { - id: string; - email: string; - full_name: string; - short_name: string; - language: string; -} - -type Base64 = string; - -interface Doc { - id: string; - title?: string; - content?: Base64; - creator: string; - is_favorite: boolean; - link_reach: 'restricted' | 'public' | 'authenticated'; - link_role: 'reader' | 'editor'; - nb_accesses_ancestors: number; - nb_accesses_direct: number; - created_at: string; - updated_at: string; - abilities: { - accesses_manage: boolean; - accesses_view: boolean; - ai_proxy: boolean; - ai_transform: boolean; - ai_translate: boolean; - attachment_upload: boolean; - children_create: boolean; - children_list: boolean; - collaboration_auth: boolean; - destroy: boolean; - favorite: boolean; - invite_owner: boolean; - link_configuration: boolean; - media_auth: boolean; - move: boolean; - partial_update: boolean; - restore: boolean; - retrieve: boolean; - update: boolean; - versions_destroy: boolean; - versions_list: boolean; - versions_retrieve: boolean; - }; -} - -async function fetch( - path: string, - requestHeaders: IncomingHttpHeaders, -): Promise { - const response = await axios.get( - `${COLLABORATION_BACKEND_BASE_URL}${path}`, - { - headers: { - cookie: requestHeaders['cookie'], - origin: requestHeaders['origin'], - 'X-Y-Provider-Key': Y_PROVIDER_API_KEY, - }, - }, - ); - - if (response.status !== 200) { - throw new Error(`Failed to fetch ${path}: ${response.statusText}`); - } - - return response.data; -} - -export function fetchDocument( - { name }: { name: string }, - requestHeaders: IncomingHttpHeaders, -): Promise { - return fetch(`/api/v1.0/documents/${name}/`, requestHeaders); -} - -export function fetchCurrentUser( - requestHeaders: IncomingHttpHeaders, -): Promise { - return fetch('/api/v1.0/users/me/', requestHeaders); -} diff --git a/src/frontend/servers/y-provider/src/env.ts b/src/frontend/servers/y-provider/src/env.ts index e125edd905..13dfcb577f 100644 --- a/src/frontend/servers/y-provider/src/env.ts +++ b/src/frontend/servers/y-provider/src/env.ts @@ -16,5 +16,3 @@ export const Y_PROVIDER_API_KEY = process.env.Y_PROVIDER_API_KEY_FILE : process.env.Y_PROVIDER_API_KEY || 'yprovider-api-key'; export const PORT = Number(process.env.PORT || 4444); export const SENTRY_DSN = process.env.SENTRY_DSN || ''; -export const COLLABORATION_BACKEND_BASE_URL = - process.env.COLLABORATION_BACKEND_BASE_URL || 'http://app-dev:8000'; diff --git a/src/frontend/servers/y-provider/src/handlers/collaborationResetConnectionsHandler.ts b/src/frontend/servers/y-provider/src/handlers/collaborationResetConnectionsHandler.ts deleted file mode 100644 index 41dfcee0c5..0000000000 --- a/src/frontend/servers/y-provider/src/handlers/collaborationResetConnectionsHandler.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { Request, Response } from 'express'; - -import { hocuspocusServer } from '@/servers'; -import { logger } from '@/utils'; - -type ResetConnectionsRequestQuery = { - room?: string; -}; - -export const collaborationResetConnectionsHandler = ( - req: Request, - res: Response, -) => { - const room = req.query.room; - const userId = req.headers['x-user-id']; - - logger('Resetting connections in room:', room, 'for user:', userId); - - if (!room) { - res.status(400).json({ error: 'Room name not provided' }); - return; - } - - /** - * If no user ID is provided, close all connections in the room - */ - if (!userId) { - hocuspocusServer.hocuspocus.closeConnections(room); - } else { - /** - * Close connections for the user in the room - */ - hocuspocusServer.hocuspocus.documents.forEach((doc) => { - if (doc.name !== room) { - return; - } - - doc.getConnections().forEach((connection) => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - if (connection.context.userId === userId) { - connection.close(); - } - }); - }); - } - - res.status(200).json({ message: 'Connections reset' }); -}; diff --git a/src/frontend/servers/y-provider/src/handlers/collaborationWSHandler.ts b/src/frontend/servers/y-provider/src/handlers/collaborationWSHandler.ts deleted file mode 100644 index 8890ad0b45..0000000000 --- a/src/frontend/servers/y-provider/src/handlers/collaborationWSHandler.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Request } from 'express'; -import * as ws from 'ws'; - -import { hocuspocusServer } from '@/servers/hocuspocusServer'; - -export const collaborationWSHandler = (ws: ws.WebSocket, req: Request) => { - try { - hocuspocusServer.hocuspocus.handleConnection(ws, req); - } catch (error) { - console.error('Failed to handle WebSocket connection:', error); - ws.close(); - } -}; diff --git a/src/frontend/servers/y-provider/src/handlers/getDocumentConnectionInfoHandler.ts b/src/frontend/servers/y-provider/src/handlers/getDocumentConnectionInfoHandler.ts deleted file mode 100644 index d015e89861..0000000000 --- a/src/frontend/servers/y-provider/src/handlers/getDocumentConnectionInfoHandler.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { Request, Response } from 'express'; - -import { hocuspocusServer } from '@/servers'; -import { logger } from '@/utils'; - -type getDocumentConnectionInfoRequestQuery = { - room?: string; - sessionKey?: string; -}; - -export const getDocumentConnectionInfoHandler = ( - req: Request, - res: Response, -) => { - const room = req.query.room; - const sessionKey = req.query.sessionKey; - - if (!room) { - res.status(400).json({ error: 'Room name not provided' }); - return; - } - - if (!req.query.sessionKey) { - res.status(400).json({ error: 'Session key not provided' }); - return; - } - - logger('Getting document connection info for room:', room); - - const roomInfo = hocuspocusServer.hocuspocus.documents.get(room); - - if (!roomInfo) { - logger('Room not found:', room); - res.status(404).json({ error: 'Room not found' }); - return; - } - const connections = roomInfo - .getConnections() - .filter((connection) => connection.readOnly === false); - - res.status(200).json({ - count: connections.length, - exists: connections.some( - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - (connection) => connection.context.sessionKey === sessionKey, - ), - }); -}; diff --git a/src/frontend/servers/y-provider/src/handlers/index.ts b/src/frontend/servers/y-provider/src/handlers/index.ts index 26b0ebedab..c8d08f6794 100644 --- a/src/frontend/servers/y-provider/src/handlers/index.ts +++ b/src/frontend/servers/y-provider/src/handlers/index.ts @@ -1,4 +1 @@ -export * from './collaborationResetConnectionsHandler'; -export * from './collaborationWSHandler'; export * from './convertHandler'; -export * from './getDocumentConnectionInfoHandler'; diff --git a/src/frontend/servers/y-provider/src/middlewares.ts b/src/frontend/servers/y-provider/src/middlewares.ts index f62678885a..11a9e1df56 100644 --- a/src/frontend/servers/y-provider/src/middlewares.ts +++ b/src/frontend/servers/y-provider/src/middlewares.ts @@ -1,6 +1,5 @@ import cors from 'cors'; import { NextFunction, Request, Response } from 'express'; -import * as ws from 'ws'; import { COLLABORATION_SERVER_ORIGIN, @@ -8,8 +7,6 @@ import { Y_PROVIDER_API_KEY, } from '@/env'; -import { logger } from './utils'; - const VALID_API_KEYS = [COLLABORATION_SERVER_SECRET, Y_PROVIDER_API_KEY]; const allowedOrigins = COLLABORATION_SERVER_ORIGIN.split(','); @@ -42,28 +39,3 @@ export const httpSecurity = ( next(); }; - -export const wsSecurity = ( - ws: ws.WebSocket, - req: Request, - next: NextFunction, -): void => { - // Origin check - const origin = req.headers['origin']; - if (!origin || !allowedOrigins.includes(origin)) { - ws.close(4001, 'Origin not allowed'); - logger('CORS policy violation: Invalid Origin', origin); - return; - } - - const cookies = req.headers['cookie']; - if (!cookies) { - ws.close(4001, 'No cookies'); - logger('CORS policy violation: No cookies'); - logger('UA:', req.headers['user-agent']); - logger('URL:', req.url); - return; - } - - next(); -}; diff --git a/src/frontend/servers/y-provider/src/routes.ts b/src/frontend/servers/y-provider/src/routes.ts index 5bb73365fb..f0a000990f 100644 --- a/src/frontend/servers/y-provider/src/routes.ts +++ b/src/frontend/servers/y-provider/src/routes.ts @@ -1,6 +1,3 @@ export const routes = { - COLLABORATION_WS: '/collaboration/ws/', - COLLABORATION_RESET_CONNECTIONS: '/collaboration/api/reset-connections/', CONVERT: '/api/convert/', - COLLABORATION_GET_CONNECTIONS: '/collaboration/api/get-connections/', }; diff --git a/src/frontend/servers/y-provider/src/servers/appServer.ts b/src/frontend/servers/y-provider/src/servers/appServer.ts index 87334cec71..14541cb48b 100644 --- a/src/frontend/servers/y-provider/src/servers/appServer.ts +++ b/src/frontend/servers/y-provider/src/servers/appServer.ts @@ -1,51 +1,22 @@ import * as Sentry from '@sentry/node'; import express from 'express'; -import expressWebsockets from 'express-ws'; import { CONVERSION_FILE_MAX_SIZE } from '@/env'; -import { - collaborationResetConnectionsHandler, - collaborationWSHandler, - convertHandler, - getDocumentConnectionInfoHandler, -} from '@/handlers'; -import { corsMiddleware, httpSecurity, wsSecurity } from '@/middlewares'; +import { convertHandler } from '@/handlers'; +import { corsMiddleware, httpSecurity } from '@/middlewares'; import { routes } from '@/routes'; import { logger } from '@/utils'; /** - * init the collaboration server. + * init the conversion server. * - * @returns An object containing the Express app, Hocuspocus server, and HTTP server instance. + * @returns The Express app instance. */ export const initApp = () => { - const { app } = expressWebsockets(express()); + const app = express(); app.use(corsMiddleware); - /** - * Route to handle WebSocket connections - */ - app.ws(routes.COLLABORATION_WS, wsSecurity, collaborationWSHandler); - - /** - * Route to reset connections in a room: - * - If no user ID is provided, close all connections in the room - * - If a user ID is provided, close connections for the user in the room - */ - app.post( - routes.COLLABORATION_RESET_CONNECTIONS, - httpSecurity, - express.json(), - collaborationResetConnectionsHandler, - ); - - app.get( - routes.COLLABORATION_GET_CONNECTIONS, - httpSecurity, - getDocumentConnectionInfoHandler, - ); - /** * Route to convert Markdown or BlockNote blocks and Yjs content */ diff --git a/src/frontend/servers/y-provider/src/servers/hocuspocusServer.ts b/src/frontend/servers/y-provider/src/servers/hocuspocusServer.ts deleted file mode 100644 index d60ec1947f..0000000000 --- a/src/frontend/servers/y-provider/src/servers/hocuspocusServer.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { Server } from '@hocuspocus/server'; -import { validate as uuidValidate, version as uuidVersion } from 'uuid'; - -import { fetchCurrentUser, fetchDocument } from '@/api/collaborationBackend'; -import { logger } from '@/utils'; - -export const hocuspocusServer = new Server({ - name: 'docs-collaboration', - timeout: 30000, - quiet: true, - async onConnect({ - requestHeaders, - connectionConfig, - documentName, - requestParameters, - context, - request, - }) { - const roomParam = requestParameters.get('room'); - - if (documentName !== roomParam) { - logger( - 'Invalid room name - Probable hacking attempt:', - documentName, - requestParameters.get('room'), - ); - logger('UA:', request.headers['user-agent']); - logger('URL:', request.url); - - return Promise.reject(new Error('Wrong room name: Unauthorized')); - } - - if (!uuidValidate(documentName) || uuidVersion(documentName) !== 4) { - logger('Room name is not a valid uuid:', documentName); - - return Promise.reject(new Error('Wrong room name: Unauthorized')); - } - - let canEdit; - - try { - const document = await fetchDocument( - { name: documentName }, - requestHeaders, - ); - - if (!document.abilities.retrieve) { - logger( - 'onConnect: Unauthorized to retrieve this document', - documentName, - ); - return Promise.reject(new Error('Wrong abilities:Unauthorized')); - } - - canEdit = document.abilities.update; - } catch (error: unknown) { - if (error instanceof Error) { - logger('onConnect: backend error', error.message); - } - - return Promise.reject(new Error('Backend error: Unauthorized')); - } - - connectionConfig.readOnly = !canEdit; - - const session = requestHeaders['cookie'] - ?.split('; ') - .find((cookie) => cookie.startsWith('docs_sessionid=')); - if (session) { - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - context.sessionKey = session.split('=')[1]; - } - - /* - * Unauthenticated users can be allowed to connect - * so we flag only authenticated users - */ - try { - const user = await fetchCurrentUser(requestHeaders); - - // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access - context.userId = user.id; - } catch { - /* empty */ - } - - logger( - 'Connection established on room:', - documentName, - 'canEdit:', - canEdit, - ); - return Promise.resolve(); - }, -}); diff --git a/src/frontend/servers/y-provider/src/servers/index.ts b/src/frontend/servers/y-provider/src/servers/index.ts index 4908530f4b..a926364356 100644 --- a/src/frontend/servers/y-provider/src/servers/index.ts +++ b/src/frontend/servers/y-provider/src/servers/index.ts @@ -1,2 +1 @@ export * from './appServer'; -export * from './hocuspocusServer'; diff --git a/src/frontend/yarn.lock b/src/frontend/yarn.lock index a71f1c28c9..8f776ad9f6 100644 --- a/src/frontend/yarn.lock +++ b/src/frontend/yarn.lock @@ -2191,35 +2191,6 @@ resolved "https://registry.yarnpkg.com/@handlewithcare/prosemirror-suggest-changes/-/prosemirror-suggest-changes-0.1.8.tgz#707d432376718d4618065b22aafbc55b9ce4ea5b" integrity sha512-ewrJl4a8dTpPJNhqYySE2ZCjTRpXulWlUmFy3sbyJgPnGtN/zx7+8tbQ1OhHfMzZWfdmA8VjP9ecy+KO4HdOpA== -"@hocuspocus/common@^3.4.4": - version "3.4.4" - resolved "https://registry.yarnpkg.com/@hocuspocus/common/-/common-3.4.4.tgz#a888fbd6dff2f0b8947c76b7841bddb89eb4d795" - integrity sha512-RykIJ0tsHHMP4Xk+4UCbc7SO5LgGxGUSTdbh6anJEsaALAyqinf1Nn5HYuMjLPolAmsar1v++m9zufR09NLpXA== - dependencies: - lib0 "^0.2.87" - -"@hocuspocus/provider@3.4.4": - version "3.4.4" - resolved "https://registry.yarnpkg.com/@hocuspocus/provider/-/provider-3.4.4.tgz#ab4ff0b55f9faf848ddbc5775956afee440a4e97" - integrity sha512-KbsMAfdYcIJD8eMU/5QnpXcSOvIWAcCNI33FSRSaKCIpYBFtAwkYIwWnZJmPZ8a1BMAtqQc+uvy9+UQf7GHnGQ== - dependencies: - "@hocuspocus/common" "^3.4.4" - "@lifeomic/attempt" "^3.0.2" - lib0 "^0.2.87" - ws "^8.17.1" - -"@hocuspocus/server@3.4.4": - version "3.4.4" - resolved "https://registry.yarnpkg.com/@hocuspocus/server/-/server-3.4.4.tgz#b44ad0aea9bdcc32d166e598278a4d5609cf03e9" - integrity sha512-UV+oaONAejOzeYgUygNcgsc8RdZvSokVvAxluZJIisLACpRO/VsseQ5lWKDRwLd7Fn6+rHWDH3hGuQ1fdX1Ycg== - dependencies: - "@hocuspocus/common" "^3.4.4" - async-lock "^1.3.1" - async-mutex "^0.5.0" - kleur "^4.1.4" - lib0 "^0.2.47" - ws "^8.5.0" - "@humanfs/core@^0.19.2": version "0.19.2" resolved "https://registry.yarnpkg.com/@humanfs/core/-/core-0.19.2.tgz#a8272ca03b2acf492670222b2320b6c421bfde60" @@ -2819,11 +2790,6 @@ resolved "https://registry.yarnpkg.com/@keyv/serialize/-/serialize-1.1.1.tgz#0c01dd3a3483882af7cf3878d4e71d505c81fc4a" integrity sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA== -"@lifeomic/attempt@^3.0.2": - version "3.1.0" - resolved "https://registry.yarnpkg.com/@lifeomic/attempt/-/attempt-3.1.0.tgz#7fc703559177b81a008b9d263e3d9a001d11d08a" - integrity sha512-QZqem4QuAnAyzfz+Gj5/+SLxqwCAw2qmt7732ZXodr6VDWGeYLG6w1i/vYLa55JQM9wRuBKLmXmiZ2P0LtE5rw== - "@lottiefiles/dotlottie-react@^0.19.6": version "0.19.10" resolved "https://registry.yarnpkg.com/@lottiefiles/dotlottie-react/-/dotlottie-react-0.19.10.tgz#0f445a83eab1d83ec9b5aeab3daf4ce13c0f2adc" @@ -6263,7 +6229,7 @@ resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.9.tgz#cf3f0e876d7bee15a93ab925b82bf570a3904a24" integrity sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg== -"@types/express-serve-static-core@*", "@types/express-serve-static-core@^5.0.0": +"@types/express-serve-static-core@^5.0.0": version "5.1.0" resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-5.1.0.tgz#74f47555b3d804b54cb7030e6f9aa0c7485cfc5b" integrity sha512-jnHMsrd0Mwa9Cf4IdOzbz543y4XJepXrbia2T4b6+spXC2We3t1y6K44D3mR8XMFSXMCf3/l7rCgddfx7UNVBA== @@ -6273,24 +6239,6 @@ "@types/range-parser" "*" "@types/send" "*" -"@types/express-ws@3.0.6": - version "3.0.6" - resolved "https://registry.yarnpkg.com/@types/express-ws/-/express-ws-3.0.6.tgz#b38cee8f84db1c9aaf11a53964db07d58c90909c" - integrity sha512-6ZDt+tMEQgM4RC1sMX1fIO7kHQkfUDlWfxoPddXUeeDjmc+Yt/fCzqXfp8rFahNr5eIxdomrWphLEWDkB2q3UQ== - dependencies: - "@types/express" "*" - "@types/express-serve-static-core" "*" - "@types/ws" "*" - -"@types/express@*": - version "5.0.3" - resolved "https://registry.yarnpkg.com/@types/express/-/express-5.0.3.tgz#6c4bc6acddc2e2a587142e1d8be0bce20757e956" - integrity sha512-wGA0NX93b19/dZC1J18tKWVIYWyyF2ZjT9vin/NRu0qzzvfVzWjs04iq2rQ3H65vCTQYlRqs3YHfY7zjdV+9Kw== - dependencies: - "@types/body-parser" "*" - "@types/express-serve-static-core" "^5.0.0" - "@types/serve-static" "*" - "@types/express@5.0.6": version "5.0.6" resolved "https://registry.yarnpkg.com/@types/express/-/express-5.0.6.tgz#2d724b2c990dcb8c8444063f3580a903f6d500cc" @@ -6394,11 +6342,6 @@ resolved "https://registry.yarnpkg.com/@types/methods/-/methods-1.1.4.tgz#d3b7ac30ac47c91054ea951ce9eed07b1051e547" integrity sha512-ymXWVrDiCxTBE3+RIrrP533E70eA+9qu7zdWoHuOmGujkYtzf4HQF96b8nwHLqhuf4ykX61IGRIB38CC6/sImQ== -"@types/mime@^1": - version "1.3.5" - resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.5.tgz#1ef302e01cf7d2b5a0fa526790c9123bf1d06690" - integrity sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w== - "@types/minimatch@^3.0.3": version "3.0.5" resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.5.tgz#1001cc5e6a3704b83c236027e77f2f58ea010f40" @@ -6481,23 +6424,6 @@ dependencies: "@types/node" "*" -"@types/send@<1": - version "0.17.5" - resolved "https://registry.yarnpkg.com/@types/send/-/send-0.17.5.tgz#d991d4f2b16f2b1ef497131f00a9114290791e74" - integrity sha512-z6F2D3cOStZvuk2SaP6YrwkNO65iTZcwA2ZkSABegdkAh/lf+Aa/YQndZVfmEXT5vgAp6zv06VQ3ejSVjAny4w== - dependencies: - "@types/mime" "^1" - "@types/node" "*" - -"@types/serve-static@*": - version "1.15.9" - resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.15.9.tgz#f9b08ab7dd8bbb076f06f5f983b683654fe0a025" - integrity sha512-dOTIuqpWLyl3BBXU3maNQsS4A3zuuoYRNIvYSxxhebPfXg2mzWQEPne/nlJ37yOse6uGgR386uTpdsx4D0QZWA== - dependencies: - "@types/http-errors" "*" - "@types/node" "*" - "@types/send" "<1" - "@types/serve-static@^2": version "2.2.0" resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-2.2.0.tgz#d4a447503ead0d1671132d1ab6bd58b805d8de6a" @@ -6554,13 +6480,6 @@ resolved "https://registry.yarnpkg.com/@types/use-sync-external-store/-/use-sync-external-store-0.0.6.tgz#60be8d21baab8c305132eb9cb912ed497852aadc" integrity sha512-zFDAD+tlpf2r4asuHEj0XH6pY6i0g5NeAHPn+15wk3BV6JA69eERFXC1gyGThDkVa1zCyKr5jox1+2LbV/AMLg== -"@types/ws@*", "@types/ws@8.18.1": - version "8.18.1" - resolved "https://registry.yarnpkg.com/@types/ws/-/ws-8.18.1.tgz#48464e4bf2ddfd17db13d845467f6070ffea4aa9" - integrity sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg== - dependencies: - "@types/node" "*" - "@types/yargs-parser@*": version "21.0.3" resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-21.0.3.tgz#815e30b786d2e8f0dcd85fd5bcf5e1a04d008f15" @@ -7581,18 +7500,6 @@ async-function@^1.0.0: resolved "https://registry.yarnpkg.com/async-function/-/async-function-1.0.0.tgz#509c9fca60eaf85034c6829838188e4e4c8ffb2b" integrity sha512-hsU18Ae8CDTR6Kgu9DYf0EbCr/a5iGL0rytQDobUcdpYOKokk8LEjVphnXkDkgpi0wYVsqrXuP0bZxJaTqdgoA== -async-lock@^1.3.1: - version "1.4.1" - resolved "https://registry.yarnpkg.com/async-lock/-/async-lock-1.4.1.tgz#56b8718915a9b68b10fce2f2a9a3dddf765ef53f" - integrity sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ== - -async-mutex@^0.5.0: - version "0.5.0" - resolved "https://registry.yarnpkg.com/async-mutex/-/async-mutex-0.5.0.tgz#353c69a0b9e75250971a64ac203b0ebfddd75482" - integrity sha512-1A94B18jkJ3DYq284ohPxoXbfTA5HsQ7/Mf4DEhcyLx3Bz27Rh59iScbB6EPiP+B+joue6YCxcMXSbFC1tZKwA== - dependencies: - tslib "^2.4.0" - async@^3.2.6: version "3.2.6" resolved "https://registry.yarnpkg.com/async/-/async-3.2.6.tgz#1b0728e14929d51b85b449b7f06e27c1145e38ce" @@ -7625,16 +7532,6 @@ axe-core@^4.10.0: resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.11.0.tgz#16f74d6482e343ff263d4f4503829e9ee91a86b6" integrity sha512-ilYanEU8vxxBexpJd8cWM4ElSQq4QctCLKih0TSfjIfCQTeyH/6zVrmIJfLPrKTKJRbiG+cfnZbQIjAlJmF1jQ== -axios@1.18.1: - version "1.18.1" - resolved "https://registry.yarnpkg.com/axios/-/axios-1.18.1.tgz#d63f9863bcd8938815c86f9e2abd380189d96dfe" - integrity sha512-3nTvFlvpn9Zu/RkHUqtc7/+al4UpRW5az71ap5zccp6e8RAYEzhMTecX8Dz1wWDYrPpUoB1HAQEGEAEvUr7S9g== - dependencies: - follow-redirects "^1.16.0" - form-data "^4.0.5" - https-proxy-agent "^5.0.1" - proxy-from-env "^2.1.0" - axobject-query@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-4.1.0.tgz#28768c76d0e3cff21bc62a9e2d0b6ac30042a1ee" @@ -9538,13 +9435,6 @@ expect@^30.0.0: jest-mock "30.2.0" jest-util "30.2.0" -express-ws@5.0.2: - version "5.0.2" - resolved "https://registry.yarnpkg.com/express-ws/-/express-ws-5.0.2.tgz#5b02d41b937d05199c6c266d7cc931c823bda8eb" - integrity sha512-0uvmuk61O9HXgLhGl3QhNSEtRsQevtmbL94/eILaliEADZBHZOQUAiHFrGPrgsjikohyrmSG5g+sCfASTt0lkQ== - dependencies: - ws "^7.4.6" - express@5.2.1: version "5.2.1" resolved "https://registry.yarnpkg.com/express/-/express-5.2.1.tgz#8f21d15b6d327f92b4794ecf8cb08a72f956ac04" @@ -9778,11 +9668,6 @@ flatted@^3.2.9, flatted@^3.3.3: resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.4.2.tgz#f5c23c107f0f37de8dbdf24f13722b3b98d52726" integrity sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA== -follow-redirects@^1.16.0: - version "1.16.0" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.16.0.tgz#28474a159d3b9d11ef62050a14ed60e4df6d61bc" - integrity sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw== - fontkit@^2.0.2: version "2.0.4" resolved "https://registry.yarnpkg.com/fontkit/-/fontkit-2.0.4.tgz#4765d664c68b49b5d6feb6bd1051ee49d8ec5ab0" @@ -10395,7 +10280,7 @@ http-proxy-agent@^7.0.2: agent-base "^7.1.0" debug "^4.3.4" -https-proxy-agent@^5.0.0, https-proxy-agent@^5.0.1: +https-proxy-agent@^5.0.0: version "5.0.1" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6" integrity sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA== @@ -11579,11 +11464,6 @@ kind-of@^6.0.2: resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.3.tgz#07c05034a6c349fa06e24fa35aa76db4580ce4dd" integrity sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw== -kleur@^4.1.4: - version "4.1.5" - resolved "https://registry.yarnpkg.com/kleur/-/kleur-4.1.5.tgz#95106101795f7050c6c650f350c683febddb1780" - integrity sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ== - known-css-properties@^0.37.0: version "0.37.0" resolved "https://registry.yarnpkg.com/known-css-properties/-/known-css-properties-0.37.0.tgz#10ebe49b9dbb6638860ff8a002fb65a053f4aec5" @@ -11619,20 +11499,20 @@ levn@^0.4.1: prelude-ls "^1.2.1" type-check "~0.4.0" -lib0@^0.2.109, lib0@^0.2.47, lib0@^0.2.85, lib0@^0.2.87: - version "0.2.114" - resolved "https://registry.yarnpkg.com/lib0/-/lib0-0.2.114.tgz#0b0e55c3ffa8768fe3d9efca971059f465db4baf" - integrity sha512-gcxmNFzA4hv8UYi8j43uPlQ7CGcyMJ2KQb5kZASw6SnAKAf10hK12i2fjrS3Cl/ugZa5Ui6WwIu1/6MIXiHttQ== - dependencies: - isomorphic.js "^0.2.4" - -lib0@^0.2.99: +lib0@^0.2.102, lib0@^0.2.99: version "0.2.117" resolved "https://registry.yarnpkg.com/lib0/-/lib0-0.2.117.tgz#6c3f926475d28904af05b590703cbbbc29475716" integrity sha512-DeXj9X5xDCjgKLU/7RR+/HQEVzuuEUiwldwOGsHK/sfAfELGWEyTcf0x+uOvCvK3O2zPmZePXWL85vtia6GyZw== dependencies: isomorphic.js "^0.2.4" +lib0@^0.2.109, lib0@^0.2.85: + version "0.2.114" + resolved "https://registry.yarnpkg.com/lib0/-/lib0-0.2.114.tgz#0b0e55c3ffa8768fe3d9efca971059f465db4baf" + integrity sha512-gcxmNFzA4hv8UYi8j43uPlQ7CGcyMJ2KQb5kZASw6SnAKAf10hK12i2fjrS3Cl/ugZa5Ui6WwIu1/6MIXiHttQ== + dependencies: + isomorphic.js "^0.2.4" + lie@~3.3.0: version "3.3.0" resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a" @@ -13045,11 +12925,6 @@ proxy-from-env@^1.1.0: resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== -proxy-from-env@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-2.1.0.tgz#a7487568adad577cfaaa7e88c49cab3ab3081aba" - integrity sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA== - pstree.remy@^1.1.8: version "1.1.8" resolved "https://registry.yarnpkg.com/pstree.remy/-/pstree.remy-1.1.8.tgz#c242224f4a67c21f686839bbdb4ac282b8373d3a" @@ -16095,17 +15970,7 @@ write-file-atomic@^5.0.1: imurmurhash "^0.1.4" signal-exit "^4.0.1" -ws@8.21.1: - version "8.21.1" - resolved "https://registry.yarnpkg.com/ws/-/ws-8.21.1.tgz#045650cd4b1207809e7547146223c3814a9af586" - integrity sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw== - -ws@^7.4.6: - version "7.5.11" - resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.11.tgz#9460daf1812bb81a423c5b9eac746941a86310fa" - integrity sha512-zS54Oen9bITtp7kp2XM3AydrCIq1D+HwJOuH+c+e4LfpL/lotP5osijd+UoMnxwAam1GN8R4KtLAyIrIcBNpiA== - -ws@^8.17.1, ws@^8.18.0, ws@^8.5.0: +ws@^8.18.0: version "8.21.0" resolved "https://registry.yarnpkg.com/ws/-/ws-8.21.0.tgz#012e413fc07429945121b0c153158c4343086951" integrity sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g== @@ -16144,19 +16009,20 @@ y-prosemirror@^1.3.7: dependencies: lib0 "^0.2.109" -y-protocols@1.0.7: +y-protocols@1.0.7, y-protocols@^1.0.5, y-protocols@^1.0.6: version "1.0.7" resolved "https://registry.yarnpkg.com/y-protocols/-/y-protocols-1.0.7.tgz#6631c492e75b78b3a61353a60067e6f8a4c38d5f" integrity sha512-YSVsLoXxO67J6eE/nV4AtFtT3QEotZf5sK5BHxFBXso7VDUT3Tx07IfA6hsu5Q5OmBdMkQVmFZ9QOA7fikWvnw== dependencies: lib0 "^0.2.85" -y-protocols@^1.0.6: - version "1.0.6" - resolved "https://registry.yarnpkg.com/y-protocols/-/y-protocols-1.0.6.tgz#66dad8a95752623443e8e28c0e923682d2c0d495" - integrity sha512-vHRF2L6iT3rwj1jub/K5tYcTT/mEYDUppgNPXwp8fmLpui9f7Yeq3OEtTLVF012j39QnV+KEQpNqoN7CWU7Y9Q== +y-websocket@3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/y-websocket/-/y-websocket-3.0.0.tgz#e86bdb29cc0a53cb8d6e33ec8d24614a723832af" + integrity sha512-mUHy7AzkOZ834T/7piqtlA8Yk6AchqKqcrCXjKW8J1w2lPtRDjz8W5/CvXz9higKAHgKRKqpI3T33YkRFLkPtg== dependencies: - lib0 "^0.2.85" + lib0 "^0.2.102" + y-protocols "^1.0.5" y18n@^5.0.5: version "5.0.8" diff --git a/src/yhub-server/Dockerfile b/src/yhub-server/Dockerfile new file mode 100644 index 0000000000..31394e2d9c --- /dev/null +++ b/src/yhub-server/Dockerfile @@ -0,0 +1,13 @@ +# trixie for glibc >= 2.38 — uws prebuilt binaries reject bookworm's 2.36 +FROM node:22-trixie AS yhub + +WORKDIR /app + +COPY package.json package-lock.json ./ +RUN npm ci --omit=dev + +COPY server.js ./ + +EXPOSE 3002 + +CMD ["node", "server.js"] diff --git a/src/yhub-server/package-lock.json b/src/yhub-server/package-lock.json new file mode 100644 index 0000000000..050099a8d6 --- /dev/null +++ b/src/yhub-server/package-lock.json @@ -0,0 +1,738 @@ +{ + "name": "yhub-server", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "yhub-server", + "dependencies": { + "@y/hub": "0.3.1" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@nodable/entities": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", + "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, + "node_modules/@pinojs/redact": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", + "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==", + "license": "MIT" + }, + "node_modules/@redis/bloom": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-5.12.1.tgz", + "integrity": "sha512-PUUfv+ms7jgPSBVoo/DN4AkPHj4D5TZSd6SbJX7egzBplkYUcKmHRE8RKia7UtZ8bSQbLguLvxVO+asKtQfZWA==", + "license": "MIT", + "engines": { + "node": ">= 18.19.0" + }, + "peerDependencies": { + "@redis/client": "^5.12.1" + } + }, + "node_modules/@redis/client": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-5.12.1.tgz", + "integrity": "sha512-7aPGWeqA3uFm43o19umzdl16CEjK/JQGtSXVPevplTaOU3VJA/rseBC1QvYUz9lLDIMBimc4SW/zrW4S89BaCA==", + "license": "MIT", + "dependencies": { + "cluster-key-slot": "1.1.2" + }, + "engines": { + "node": ">= 18.19.0" + }, + "peerDependencies": { + "@node-rs/xxhash": "^1.1.0", + "@opentelemetry/api": ">=1 <2" + }, + "peerDependenciesMeta": { + "@node-rs/xxhash": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + } + } + }, + "node_modules/@redis/json": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-5.12.1.tgz", + "integrity": "sha512-eOze75esLve4vfqDel7aMX08CNaiLLQS2fV8mpRN9NxPe1rVR4vQyYiW/OgtGUysF6QOr9ANhfxABKNOJfXdKg==", + "license": "MIT", + "engines": { + "node": ">= 18.19.0" + }, + "peerDependencies": { + "@redis/client": "^5.12.1" + } + }, + "node_modules/@redis/search": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-5.12.1.tgz", + "integrity": "sha512-ItlxbxC9cKI6IU1TLWoczwJCRb6TdmkEpWv05UrPawqaAnWGRu3rcIqsc5vN483T2fSociuyV1UkWIL5I4//2w==", + "license": "MIT", + "engines": { + "node": ">= 18.19.0" + }, + "peerDependencies": { + "@redis/client": "^5.12.1" + } + }, + "node_modules/@redis/time-series": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-5.12.1.tgz", + "integrity": "sha512-c6JL6E3EcZJuNqKFz+KM+l9l5mpcQiKvTwgA3blt5glWJ8hjDk0yeHN3beE/MpqYIQ8UEX44ItQzgkE/gCBELQ==", + "license": "MIT", + "engines": { + "node": ">= 18.19.0" + }, + "peerDependencies": { + "@redis/client": "^5.12.1" + } + }, + "node_modules/@y-crdt/yn": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/@y-crdt/yn/-/yn-0.1.4.tgz", + "integrity": "sha512-BrRpTE4tvONSx+hYXbpZERu7Cx3/xyHFNXKNnBd9/maTJNsCGaVwfkfH3PN9fx/IrIXxa3GTjX0t8zJam273BQ==", + "license": "ISC" + }, + "node_modules/@y/hub": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@y/hub/-/hub-0.3.1.tgz", + "integrity": "sha512-gauvqZ2XwOi7c/Fu13xbOCV3uR+OCHH5QUMyIPXJfDvdTwILEo0hatQWaLK1NhfOZCcaxTPbpgeu2GdLGs7xhg==", + "license": "AGPL-3.0 OR PROPRIETARY", + "dependencies": { + "@y-crdt/yn": "^0.1.4", + "@y/protocols": "^1.0.6-rc.1", + "@y/y": "^14.0.0-rc.24", + "lib0": "^1.0.0-rc.22", + "minio": "^8.0.6", + "pino": "^10.3.1", + "postgres": "^3.4.3", + "redis": "^5.10.0", + "uws": "github:uNetworking/uWebSockets.js#v20.57.0" + }, + "bin": { + "yhub": "bin/yhub.js" + }, + "engines": { + "node": ">=22.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, + "node_modules/@y/protocols": { + "version": "1.0.6-rc.1", + "resolved": "https://registry.npmjs.org/@y/protocols/-/protocols-1.0.6-rc.1.tgz", + "integrity": "sha512-e/qs7hXcLk/SeNitxMXv2ymozyWFTULwbJEi7cAf/K/iXw9nGwGXHrR5TNluQ/bMwOX1cwuUT0hjEojkfH0gsA==", + "license": "MIT", + "dependencies": { + "lib0": "^1.0.0-rc.1" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + }, + "peerDependencies": { + "@y/y": "*" + } + }, + "node_modules/@y/y": { + "version": "14.0.0-rc.24", + "resolved": "https://registry.npmjs.org/@y/y/-/y-14.0.0-rc.24.tgz", + "integrity": "sha512-E22nv/q6CWNodzU/yowxEp95TUVBTP5T63kAiJy2FpsuB5zxqfGhdW2222S4tFF9mo9UtBqg8ep7tPMJ9bZVOg==", + "license": "MIT", + "dependencies": { + "lib0": "^1.0.0-rc.21" + }, + "engines": { + "node": ">=22.0.0", + "npm": ">=8.0.0" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, + "node_modules/anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "license": "MIT" + }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/block-stream2": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/block-stream2/-/block-stream2-2.1.0.tgz", + "integrity": "sha512-suhjmLI57Ewpmq00qaygS8UgEq2ly2PCItenIyhMqVjo4t4pGzqMvfgJuX8iWTeSDdfSSqS6j38fL4ToNL7Pfg==", + "license": "MIT", + "dependencies": { + "readable-stream": "^3.4.0" + } + }, + "node_modules/browser-or-node": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/browser-or-node/-/browser-or-node-2.1.1.tgz", + "integrity": "sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg==", + "license": "MIT" + }, + "node_modules/buffer-crc32": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/decode-uri-component": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", + "integrity": "sha512-FqUYQ+8o158GyGTrMFJms9qh3CqTKvAqgqsTnkLI8sKu0028orqBhxNMFkFen0zGyg6epACD32pjVk58ngIErQ==", + "license": "MIT", + "engines": { + "node": ">=0.10" + } + }, + "node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "license": "MIT" + }, + "node_modules/fast-xml-builder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.0.tgz", + "integrity": "sha512-F74cZEdCvuw9P41GAC3rod4X04jjWGM1JPEv/GWSqFTWLsdyMSBMBMlm9Hk3GLBgLBbdBNY8yee0pQh2RBVESQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.6.2", + "xml-naming": "^0.3.0" + } + }, + "node_modules/fast-xml-parser": { + "version": "5.10.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.10.1.tgz", + "integrity": "sha512-IEMIf7298kXuZSRFoGfMYrl7is8LpavODgbNz1cwIudv7KwVFnuU+UsMporfq6PD6aXSlawZlARiA3UywCTfMw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^3.0.0", + "fast-xml-builder": "^1.2.0", + "is-unsafe": "^2.0.0", + "path-expression-matcher": "^1.6.2", + "strnum": "^2.4.1", + "xml-naming": "^0.3.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/filter-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/filter-obj/-/filter-obj-1.1.0.tgz", + "integrity": "sha512-8rXg1ZnX7xzy2NGDVkBVaAy+lSlPNwad13BtgSlLuxfIslyt5Vg64U7tFcCt4WS1R0hvtnQybT/IyCkGZ3DpXQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.4.0.tgz", + "integrity": "sha512-9VGk3HGanVE6JoZXHiCpnGy5X0jYDnN4EA4lntFPj+1vIWlFhIylq2CrrCOJH9EAhc5CYhq18F2Av2tgoAPsYQ==", + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/is-unsafe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.0.tgz", + "integrity": "sha512-2LdV822R+wmI86unXA93WCFpL6g+av8ynWk0nrHyJqGop5VoocYsSLFgN8jrfalT6iGeLNM4KXuVSsULP53kEA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, + "node_modules/lib0": { + "version": "1.0.0-rc.22", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-1.0.0-rc.22.tgz", + "integrity": "sha512-KNefJloRQIsWncTF2tIcRqQXSQ7bDRYHwVSUhf1lY2P65Rej4WWFnen6L8L+odJQIo1ZNJGVVjK2WzqB9a+B/g==", + "license": "MIT", + "bin": { + "0ecdsa-generate-keypair": "src/bin/0ecdsa-generate-keypair.js", + "0gentesthtml": "src/bin/gentesthtml.js", + "0serve": "src/bin/0serve.js" + }, + "engines": { + "node": ">=22" + }, + "funding": { + "type": "GitHub Sponsors ❤", + "url": "https://github.com/sponsors/dmonad" + } + }, + "node_modules/lodash": { + "version": "4.18.1", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", + "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==", + "license": "MIT" + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/minio": { + "version": "8.0.7", + "resolved": "https://registry.npmjs.org/minio/-/minio-8.0.7.tgz", + "integrity": "sha512-E737MgufW8CeQAsTAtnEMrxZ9scMSf29kkhZoXzDTKj/Jszzo2SfeZUH9wbDQH2Rsq6TCtl/yQL0+XdVKZansQ==", + "license": "Apache-2.0", + "dependencies": { + "async": "^3.2.4", + "block-stream2": "^2.1.0", + "browser-or-node": "^2.1.1", + "buffer-crc32": "^1.0.0", + "eventemitter3": "^5.0.1", + "fast-xml-parser": "^5.3.4", + "ipaddr.js": "^2.0.1", + "lodash": "^4.17.21", + "mime-types": "^2.1.35", + "query-string": "^7.1.3", + "stream-json": "^1.8.0", + "through2": "^4.0.2", + "xml2js": "^0.5.0 || ^0.6.2" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, + "node_modules/on-exit-leak-free": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", + "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==", + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/path-expression-matcher": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", + "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/pino": { + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz", + "integrity": "sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==", + "license": "MIT", + "dependencies": { + "@pinojs/redact": "^0.4.0", + "atomic-sleep": "^1.0.0", + "on-exit-leak-free": "^2.1.0", + "pino-abstract-transport": "^3.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^5.0.0", + "quick-format-unescaped": "^4.0.3", + "real-require": "^0.2.0", + "safe-stable-stringify": "^2.3.1", + "sonic-boom": "^4.0.1", + "thread-stream": "^4.0.0" + }, + "bin": { + "pino": "bin.js" + } + }, + "node_modules/pino-abstract-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz", + "integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==", + "license": "MIT", + "dependencies": { + "split2": "^4.0.0" + } + }, + "node_modules/pino-std-serializers": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.1.0.tgz", + "integrity": "sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==", + "license": "MIT" + }, + "node_modules/postgres": { + "version": "3.4.9", + "resolved": "https://registry.npmjs.org/postgres/-/postgres-3.4.9.tgz", + "integrity": "sha512-GD3qdB0x1z9xgFI6cdRD6xu2Sp2WCOEoe3mtnyB5Ee0XrrL5Pe+e4CCnJrRMnL1zYtRDZmQQVbvOttLnKDLnaw==", + "license": "Unlicense", + "engines": { + "node": ">=12" + }, + "funding": { + "type": "individual", + "url": "https://github.com/sponsors/porsager" + } + }, + "node_modules/process-warning": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-5.1.0.tgz", + "integrity": "sha512-jQSaVHsPgtyw60e1rQ/A+/ArPEj/S8pS/vFnyGa/gYFXrKk/6RuDkoqVDQ5NI5MmS01698ltlAk0NoDBNLujRw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, + "node_modules/query-string": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/query-string/-/query-string-7.1.3.tgz", + "integrity": "sha512-hh2WYhq4fi8+b+/2Kg9CEge4fDPvHS534aOOvOZeQ3+Vf2mCFsaFBYj0i+iXcAq6I9Vzp5fjMFBlONvayDC1qg==", + "license": "MIT", + "dependencies": { + "decode-uri-component": "^0.2.2", + "filter-obj": "^1.1.0", + "split-on-first": "^1.0.0", + "strict-uri-encode": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/quick-format-unescaped": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==", + "license": "MIT" + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/real-require": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", + "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", + "license": "MIT", + "engines": { + "node": ">= 12.13.0" + } + }, + "node_modules/redis": { + "version": "5.12.1", + "resolved": "https://registry.npmjs.org/redis/-/redis-5.12.1.tgz", + "integrity": "sha512-LDsoVvb/CpoV9EN3FXvgvSHNJWuCIzl9MiO3ppOevuGLpSGJhwfQjpEwfFJcQvNSddHADDdZaWx0HnmMxRXG7g==", + "license": "MIT", + "dependencies": { + "@redis/bloom": "5.12.1", + "@redis/client": "5.12.1", + "@redis/json": "5.12.1", + "@redis/search": "5.12.1", + "@redis/time-series": "5.12.1" + }, + "engines": { + "node": ">= 18.19.0" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/sax": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.1.tgz", + "integrity": "sha512-42tBVwLWnaQvW5zc4HbZrTuWccECCZfBi92FDuwtqxasH+JbPB3/FOKb1m222K42R4WxuxzzMsTswfzgtSu64Q==", + "license": "BlueOak-1.0.0", + "engines": { + "node": ">=11.0.0" + } + }, + "node_modules/sonic-boom": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz", + "integrity": "sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==", + "license": "MIT", + "dependencies": { + "atomic-sleep": "^1.0.0" + } + }, + "node_modules/split-on-first": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/split-on-first/-/split-on-first-1.1.0.tgz", + "integrity": "sha512-43ZssAJaMusuKWL8sKUBQXHWOpq8d6CfN/u1p4gUzfJkM05C8rxTmYrkIPTXapZpORA6LkkzcUulJ8FqA7Uudw==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, + "node_modules/stream-chain": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.5.tgz", + "integrity": "sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==", + "license": "BSD-3-Clause" + }, + "node_modules/stream-json": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.9.1.tgz", + "integrity": "sha512-uWkjJ+2Nt/LO9Z/JyKZbMusL8Dkh97uUBTv3AJQ74y07lVahLY4eEFsPsE97pxYBwr8nnjMAIch5eqI0gPShyw==", + "license": "BSD-3-Clause", + "dependencies": { + "stream-chain": "^2.2.5" + } + }, + "node_modules/strict-uri-encode": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz", + "integrity": "sha512-QwiXZgpRcKkhTj2Scnn++4PKtWsH0kpzZ62L2R6c/LUVYv7hVnZqcg2+sMuT6R7Jusu1vviK/MFsu6kNJfWlEQ==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/strnum": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.1.tgz", + "integrity": "sha512-M9eUSMT2dCB2cTNPG7UYj6KuK7RJR2SN2+yCV/fTW3xzTCS6EaGZ5pSMgDIjB7r8zSfTGk+dvvn9rTjpVS9Mwg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "anynum": "^1.0.1" + } + }, + "node_modules/thread-stream": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.2.0.tgz", + "integrity": "sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==", + "license": "MIT", + "dependencies": { + "real-require": "^1.0.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/thread-stream/node_modules/real-require": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/real-require/-/real-require-1.0.0.tgz", + "integrity": "sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==", + "license": "MIT" + }, + "node_modules/through2": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz", + "integrity": "sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==", + "license": "MIT", + "dependencies": { + "readable-stream": "3" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/uws": { + "name": "uWebSockets.js", + "version": "20.57.0", + "resolved": "git+ssh://git@github.com/uNetworking/uWebSockets.js.git#fcfc622a4286909593b7f390056d89e0ca3b56b9", + "license": "Apache-2.0" + }, + "node_modules/xml-naming": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", + "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/xml2js": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", + "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==", + "license": "MIT", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "license": "MIT", + "engines": { + "node": ">=4.0" + } + } + } +} diff --git a/src/yhub-server/package.json b/src/yhub-server/package.json new file mode 100644 index 0000000000..0d5e5d0d90 --- /dev/null +++ b/src/yhub-server/package.json @@ -0,0 +1,14 @@ +{ + "name": "yhub-server", + "private": true, + "type": "module", + "scripts": { + "start": "node server.js" + }, + "dependencies": { + "@y/hub": "0.3.1" + }, + "engines": { + "node": ">=22" + } +} diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js new file mode 100644 index 0000000000..8577706a55 --- /dev/null +++ b/src/yhub-server/server.js @@ -0,0 +1,98 @@ +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; + +import { createAuthPlugin, createYHub } from '@y/hub'; + +// mirror y-provider's env.ts secret-file support +const secret = (name, dflt) => + process.env[`${name}_FILE`] + ? readFileSync(process.env[`${name}_FILE`], 'utf8').trim() + : process.env[name] || dflt; + +const PORT = Number(process.env.PORT || 3002); +const REDIS = process.env.REDIS; +const POSTGRES = process.env.POSTGRES; +const REDIS_PREFIX = process.env.REDIS_PREFIX || 'yhub'; +const COLLABORATION_BACKEND_BASE_URL = + process.env.COLLABORATION_BACKEND_BASE_URL || 'http://app-dev:8000'; +const allowedOrigins = ( + process.env.COLLABORATION_SERVER_ORIGIN || 'http://localhost:3000' +).split(','); +const Y_PROVIDER_API_KEY = secret('Y_PROVIDER_API_KEY', 'yprovider-api-key'); +const ORG = process.env.YHUB_ORG || 'docs'; +const UUID4 = + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +const backendFetch = async (path, { cookie, origin }) => { + const res = await fetch(`${COLLABORATION_BACKEND_BASE_URL}${path}`, { + headers: { + cookie, + origin, + 'X-Y-Provider-Key': Y_PROVIDER_API_KEY, + }, + }); + if (!res.ok) { + throw new Error(`Failed to fetch ${path}: ${res.status}`); + } + return res.json(); +}; + +const auth = createAuthPlugin({ + // uws req is only valid synchronously — read headers AND query before first await. + async readAuthInfo(req) { + const cookie = req.getHeader('cookie'); + const origin = req.getHeader('origin'); + const gcOff = req.getQuery('gc') === 'false'; + if (gcOff) return null; // full-history connections: not for Docs users + if (!origin || !allowedOrigins.includes(origin)) return null; // was 4001 'Origin not allowed' + if (!cookie) return null; // was 4001 'No cookies' + try { + const user = await backendFetch('/api/v1.0/users/me/', { + cookie, + origin, + }); + return { userid: String(user.id), cookie, origin }; // MUST be string (yhub server.js:667) + } catch { + // anonymous (public docs): stable per-session id — random ids would mint a new + // permanent attribution identity per reconnect + const anon = createHash('sha256') + .update(cookie) + .digest('base64url') + .slice(0, 16); + return { userid: `anon:${anon}`, cookie, origin }; + } + }, + async getAccessType(authInfo, { org, docid, branch }) { + if (org !== ORG || branch !== 'main' || !UUID4.test(docid)) { + return null; + } + try { + const doc = await backendFetch( + `/api/v1.0/documents/${docid}/`, + authInfo, + ); + if (!doc.abilities?.retrieve) { + return null; + } + return doc.abilities.update ? 'rw' : 'r'; + } catch { + return null; + } + }, +}); + +await createYHub({ + redis: { + url: REDIS, + prefix: REDIS_PREFIX, + taskDebounce: 10000, + minMessageLifetime: 60000, + }, + postgres: POSTGRES, + persistence: [], // blobs live in yhub's postgres + server: { port: PORT, auth }, + worker: { taskConcurrency: 5 }, + // TODO(yhub): worker.events.docUpdate could push snapshots to Django and replace the + // client useSaveDoc PATCH flow — blocked upstream: the payload is a DocTable without + // room/org/docid (yhub src/index.js:90); needs an upstream change first. +}); From 4e893efb57b4f302bf05b84c9bff83df89ec844a Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Tue, 4 Aug 2026 16:54:29 +0200 Subject: [PATCH 02/59] =?UTF-8?q?=F0=9F=93=84(collaboration)=20add=20licen?= =?UTF-8?q?se=20notice=20for=20yhub-server=20directory?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Kevin Jahns --- src/yhub-server/LICENSE | 37 +++++++++++++++++++++++++++++++++++++ src/yhub-server/README.md | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 src/yhub-server/LICENSE create mode 100644 src/yhub-server/README.md diff --git a/src/yhub-server/LICENSE b/src/yhub-server/LICENSE new file mode 100644 index 0000000000..135d203508 --- /dev/null +++ b/src/yhub-server/LICENSE @@ -0,0 +1,37 @@ +License notice for the src/yhub-server directory +================================================ + +The source code in this directory is, like the rest of this repository, +released under the MIT License (see the LICENSE file at the repository root, +Copyright (c) 2023 Direction Interministérielle du Numérique - Gouvernement +Français). + +Dependency on AGPL-licensed code +-------------------------------- + +However, the code in this directory (in particular `server.js`) depends on +and runs in the same process as the `@y/hub` package ("yhub"), which is +licensed under the GNU Affero General Public License v3.0 (AGPL-3.0) — or, +alternatively, under a proprietary license available from its author. + +Unless you have obtained such a proprietary license for yhub, the combined +work formed by yhub together with the code in this directory is governed by +the terms of the AGPL-3.0. This means in particular: + +- If you modify the code in this directory and run the resulting server for + users over a network, or distribute it, you must license your + modifications under the AGPL-3.0 or an AGPL-compatible license and make + the corresponding source available (AGPL-3.0, section 13). +- The MIT license of the files in this directory is compatible with this + obligation: MIT-licensed code may be incorporated into an AGPL-licensed + combined work. + +Scope +----- + +This notice applies only to this directory. The rest of the La Suite Docs +software (the Django backend, the frontend, and all other components in this +repository) does not link against yhub and communicates with it exclusively +through network requests (REST/HTTP and WebSocket). It therefore does not +form a combined work with yhub and remains governed solely by the MIT +License at the repository root. diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md new file mode 100644 index 0000000000..229c5b9826 --- /dev/null +++ b/src/yhub-server/README.md @@ -0,0 +1,36 @@ +# yhub-server + +This directory contains the La Suite Docs-specific configuration for +[yhub](https://www.npmjs.com/package/@y/hub) (`@y/hub`), the collaboration +server that synchronizes Yjs documents between editors in real time. + +It is not a fork of yhub — it is a thin wrapper (`server.js`) that: + +- starts a yhub instance (websocket sync on port 3002, backed by Redis/Valkey + and PostgreSQL), +- plugs in an auth plugin that resolves users and per-document access rights + by calling the Docs Django backend (`/api/v1.0/users/me/` and + `/api/v1.0/documents/{id}/`), +- mirrors the environment conventions used elsewhere in this repository + (`*_FILE` secret indirection, `COLLABORATION_SERVER_ORIGIN` allowlist, …). + +The `Dockerfile` builds the container image used by the `yhub` service in +`compose.yml`. + +## ⚠️ License warning (AGPL) + +This directory depends on `@y/hub`, which is licensed under the +**GNU AGPL-3.0** (or a separate proprietary license from its author). Unlike +the rest of this repository (MIT), the code in this directory is loaded into +the same process as AGPL-licensed code. As a consequence: + +- **Any modification to the code in this directory (in particular + `server.js`) must be released under an AGPL-compatible license** if you run + or distribute the resulting server, including making it available to users + over a network (AGPL section 13). +- See the [LICENSE](./LICENSE) file in this directory for details. + +**The rest of La Suite Docs is not affected.** The Django backend and the +frontend never link against yhub; they communicate with it exclusively +through network requests (REST/HTTP and WebSocket). They remain under the +MIT license of the repository root. From 7ea52473dcb610a3b66ecddfccfe4df86cb01974 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Tue, 4 Aug 2026 15:27:15 +0200 Subject: [PATCH 03/59] =?UTF-8?q?=E2=9C=A8(backend)=20add=20a=20service=20?= =?UTF-8?q?generating=20cached=20RS256=20JWT=20tokens?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We want to generate jwt token using the RS256 algotrithm. This token will be used for internal call with the yhub service. --- CHANGELOG.md | 1 + documentation/env.md | 2 + src/backend/core/services/jwt_services.py | 117 ++++++++++ .../core/tests/test_services_jwt_services.py | 207 ++++++++++++++++++ src/backend/impress/settings.py | 17 ++ src/backend/pyproject.toml | 2 +- src/backend/uv.lock | 9 +- 7 files changed, 352 insertions(+), 3 deletions(-) create mode 100644 src/backend/core/services/jwt_services.py create mode 100644 src/backend/core/tests/test_services_jwt_services.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e188d43e4b..de422ea340 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to ### Added +- ✨(backend) add a service generating cached RS256 JWT tokens - ♿️(frontend) restore skip to content link after header redesign #2510 - 🌐(i18n) rename cn_CN to zh_CN, add eo_PL and zh_TW locales #2486 - ✨(backend) conditional email notification in server to server api #2554 diff --git a/documentation/env.md b/documentation/env.md index 6da1e30e86..65fe38b476 100644 --- a/documentation/env.md +++ b/documentation/env.md @@ -81,6 +81,8 @@ These are the environment variables you can set for the `impress-backend` contai | FRONTEND_JS_URL | To add a external js file to the app | | | FRONTEND_HOMEPAGE_FEATURE_ENABLED | Frontend feature flag to display the homepage | false | | FRONTEND_THEME | Frontend theme to use | | +| JWT_PRIVATE_KEY | PEM encoded RSA private key used to sign the JWT tokens (RS256). Can be read from a file with JWT_PRIVATE_KEY_FILE | | +| JWT_TOKEN_LIFETIME | Lifetime in seconds of the generated JWT tokens. Also used as the cache timeout of these tokens | 3600 | | LANGUAGE_CODE | Default language | en-us | | LANGFUSE_SECRET_KEY | The Langfuse secret key used by the sdk | None | | LANGFUSE_PUBLIC_KEY | The Langfuse public key used by the sdk | None | diff --git a/src/backend/core/services/jwt_services.py b/src/backend/core/services/jwt_services.py new file mode 100644 index 0000000000..a57fa6acdf --- /dev/null +++ b/src/backend/core/services/jwt_services.py @@ -0,0 +1,117 @@ +"""JWT services.""" + +import hashlib +import json +import logging +from datetime import timedelta + +from django.conf import settings +from django.core.cache import cache +from django.utils import timezone + +import jwt + +logger = logging.getLogger(__name__) + +ALGORITHM = "RS256" +CACHE_KEY_PREFIX = "jwt_token" + + +class JWTError(Exception): + """Base exception for JWT related errors.""" + + +class ConfigurationError(JWTError): + """Raised when the JWT service is not properly configured.""" + + +class TokenGenerationError(JWTError): + """Raised when a token cannot be signed.""" + + +class JWTService: + """ + Service class issuing RS256 signed JSON Web Tokens. + + The claims are injected by the caller at generation time, the service only + owns the signature and the token lifetime. Generated tokens are cached for + their whole lifetime so that repeated calls with the same claims reuse the + same token instead of signing a new one. + """ + + algorithm = ALGORITHM + + @property + def private_key(self): + """Return the RSA private key used to sign the tokens.""" + private_key = settings.JWT_PRIVATE_KEY + if not private_key: + raise ConfigurationError( + "The JWT_PRIVATE_KEY setting is required to sign tokens." + ) + return private_key + + @property + def lifetime(self): + """Return the token lifetime, in seconds.""" + return settings.JWT_TOKEN_LIFETIME + + def get_cache_key(self, claims): + """ + Build the cache key identifying a token for the given claims. + + The signing key and the lifetime are part of the fingerprint so that + rotating the key or changing the lifetime never serves a stale token. + """ + fingerprint = json.dumps( + { + "claims": claims, + "lifetime": self.lifetime, + "key": self.private_key, + }, + sort_keys=True, + default=str, + ) + digest = hashlib.sha256(fingerprint.encode("utf-8")).hexdigest() + return f"{CACHE_KEY_PREFIX}:{digest}" + + def generate_token(self, claims): + """ + Sign a new token embedding the given claims. + + The "iat" and "exp" claims are always set by the service, from the + configured lifetime, and take precedence over the caller's claims. + """ + issued_at = timezone.now() + payload = { + **claims, + "iat": issued_at, + "exp": issued_at + timedelta(seconds=self.lifetime), + } + + try: + return jwt.encode(payload, self.private_key, algorithm=self.algorithm) + except (jwt.PyJWTError, TypeError, ValueError) as err: + logger.exception( + "Unable to sign a JWT token with algorithm %s", self.algorithm + ) + raise TokenGenerationError("Unable to sign the JWT token") from err + + def get_token(self, claims): + """ + Return a token embedding the given claims, generating it if needed. + + The token is cached for its own lifetime, so a cached token can be + returned close to its expiry. Callers needing a guaranteed remaining + validity should account for it in the configured lifetime. + """ + cache_key = self.get_cache_key(claims) + + token = cache.get(cache_key) + if token is not None: + return token + + token = self.generate_token(claims) + cache.set(cache_key, token, self.lifetime) + + return token diff --git a/src/backend/core/tests/test_services_jwt_services.py b/src/backend/core/tests/test_services_jwt_services.py new file mode 100644 index 0000000000..423fc3e99d --- /dev/null +++ b/src/backend/core/tests/test_services_jwt_services.py @@ -0,0 +1,207 @@ +""" +This module contains tests for the JWTService class in the +core.services.jwt_services module. +""" + +from datetime import datetime, timezone +from unittest import mock + +from django.core.cache import cache + +import jwt +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from freezegun import freeze_time + +from core.services.jwt_services import ( + ConfigurationError, + JWTService, + TokenGenerationError, +) + + +def generate_key_pair(): + """Generate a PEM encoded RSA key pair to sign and verify test tokens.""" + private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + private_pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ).decode("utf-8") + public_pem = ( + private_key.public_key() + .public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ) + .decode("utf-8") + ) + return private_pem, public_pem + + +# Generating RSA keys is expensive, do it once for the whole module +PRIVATE_KEY, PUBLIC_KEY = generate_key_pair() +OTHER_PRIVATE_KEY, OTHER_PUBLIC_KEY = generate_key_pair() + + +@pytest.fixture(name="jwt_settings") +def jwt_settings_fixture(settings): + """Setup valid settings for the JWT service.""" + settings.JWT_PRIVATE_KEY = PRIVATE_KEY + settings.JWT_TOKEN_LIFETIME = 3600 + return settings + + +@pytest.mark.usefixtures("jwt_settings") +def test_get_token_signs_the_injected_claims_with_rs256(): + """The generated token is signed with RS256 and carries the given claims.""" + token = JWTService().get_token({"sub": "user-id", "abilities": ["read"]}) + + assert jwt.get_unverified_header(token)["alg"] == "RS256" + + payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + assert payload["sub"] == "user-id" + assert payload["abilities"] == ["read"] + + +@pytest.mark.usefixtures("jwt_settings") +def test_get_token_cannot_be_verified_with_another_key(): + """The token is signed with the private key defined in the settings.""" + token = JWTService().get_token({"sub": "user-id"}) + + with pytest.raises(jwt.InvalidSignatureError): + jwt.decode(token, OTHER_PUBLIC_KEY, algorithms=["RS256"]) + + +def test_generate_token_expires_after_the_configured_lifetime(jwt_settings): + """The "iat" and "exp" claims are computed from the configured lifetime.""" + jwt_settings.JWT_TOKEN_LIFETIME = 300 + + now = datetime(2026, 8, 4, 10, 0, 0, tzinfo=timezone.utc) + with freeze_time(now): + token = JWTService().generate_token({"sub": "user-id"}) + payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + + assert payload["iat"] == now.timestamp() + assert payload["exp"] == now.timestamp() + 300 + + +def test_generate_token_ignores_the_expiry_claims_given_by_the_caller(jwt_settings): + """The service owns the token lifetime, the caller cannot extend it.""" + jwt_settings.JWT_TOKEN_LIFETIME = 60 + + now = datetime(2026, 8, 4, 10, 0, 0, tzinfo=timezone.utc) + with freeze_time(now): + token = JWTService().generate_token( + {"sub": "user-id", "iat": 0, "exp": 99999999999} + ) + payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + + assert payload["iat"] == now.timestamp() + assert payload["exp"] == now.timestamp() + 60 + + +@pytest.mark.usefixtures("jwt_settings") +def test_get_token_reuses_the_cached_token(): + """A token already in cache is returned without signing a new one.""" + service = JWTService() + claims = {"sub": "user-id"} + + token = service.get_token(claims) + assert cache.get(service.get_cache_key(claims)) == token + + with mock.patch("core.services.jwt_services.jwt.encode") as mock_encode: + assert service.get_token(claims) == token + + mock_encode.assert_not_called() + + +def test_get_token_caches_the_token_for_its_lifetime(jwt_settings): + """The cache entry expires along with the token it holds.""" + jwt_settings.JWT_TOKEN_LIFETIME = 300 + + service = JWTService() + claims = {"sub": "user-id"} + + with freeze_time("2026-08-04 10:00:00") as frozen_time: + token = service.get_token(claims) + + frozen_time.move_to("2026-08-04 10:04:59") + assert cache.get(service.get_cache_key(claims)) == token + + frozen_time.move_to("2026-08-04 10:05:01") + assert cache.get(service.get_cache_key(claims)) is None + + +@pytest.mark.usefixtures("jwt_settings") +def test_get_token_caches_each_set_of_claims_separately(): + """Two different sets of claims get two different tokens.""" + service = JWTService() + + first_token = service.get_token({"sub": "user-id"}) + second_token = service.get_token({"sub": "other-user-id"}) + + assert first_token != second_token + assert jwt.decode(first_token, PUBLIC_KEY, algorithms=["RS256"])["sub"] == "user-id" + assert ( + jwt.decode(second_token, PUBLIC_KEY, algorithms=["RS256"])["sub"] + == "other-user-id" + ) + + +@pytest.mark.usefixtures("jwt_settings") +def test_get_token_ignores_the_claims_ordering(): + """Claims given in a different order hit the same cache entry.""" + service = JWTService() + + assert service.get_cache_key({"a": 1, "b": 2}) == service.get_cache_key( + {"b": 2, "a": 1} + ) + + +def test_get_token_generates_a_new_token_after_a_key_rotation(jwt_settings): + """A token signed with a rotated out key is never served from the cache.""" + service = JWTService() + claims = {"sub": "user-id"} + + service.get_token(claims) + + jwt_settings.JWT_PRIVATE_KEY = OTHER_PRIVATE_KEY + token = service.get_token(claims) + + assert jwt.decode(token, OTHER_PUBLIC_KEY, algorithms=["RS256"])["sub"] == "user-id" + + +def test_get_token_generates_a_new_token_when_the_lifetime_changes(jwt_settings): + """A token cached with the former lifetime is never served.""" + jwt_settings.JWT_TOKEN_LIFETIME = 300 + + service = JWTService() + claims = {"sub": "user-id"} + + with freeze_time("2026-08-04 10:00:00"): + service.get_token(claims) + + jwt_settings.JWT_TOKEN_LIFETIME = 600 + token = service.get_token(claims) + payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + + assert payload["exp"] - payload["iat"] == 600 + + +@pytest.mark.parametrize("private_key", [None, ""]) +def test_get_token_without_private_key(jwt_settings, private_key): + """The service refuses to issue a token when no private key is configured.""" + jwt_settings.JWT_PRIVATE_KEY = private_key + + with pytest.raises(ConfigurationError, match="JWT_PRIVATE_KEY"): + JWTService().get_token({"sub": "user-id"}) + + +def test_generate_token_with_an_invalid_private_key(jwt_settings): + """An unusable private key is reported as a token generation error.""" + jwt_settings.JWT_PRIVATE_KEY = "not-a-pem-key" + + with pytest.raises(TokenGenerationError, match="Unable to sign the JWT token"): + JWTService().generate_token({"sub": "user-id"}) diff --git a/src/backend/impress/settings.py b/src/backend/impress/settings.py index a6a2f79ff1..173bf5c2da 100755 --- a/src/backend/impress/settings.py +++ b/src/backend/impress/settings.py @@ -540,6 +540,23 @@ class Base(Configuration): environ_prefix=None, ) + # JWT + # RSA private key (PEM) used to sign the tokens issued by + # core.services.jwt_services.JWTService. Prefer the JWT_PRIVATE_KEY_FILE + # environment variable, a PEM does not fit well in an environment variable. + JWT_PRIVATE_KEY = SecretFileValue( + None, + environ_name="JWT_PRIVATE_KEY", + environ_prefix=None, + ) + # Lifetime, in seconds, of the tokens issued by the JWT service. It is both + # the "exp" claim horizon and the cache timeout of the generated tokens. + JWT_TOKEN_LIFETIME = values.IntegerValue( + default=3600, + environ_name="JWT_TOKEN_LIFETIME", + environ_prefix=None, + ) + # Frontend FRONTEND_THEME = values.Value( None, environ_name="FRONTEND_THEME", environ_prefix=None diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index cfbf3d3675..82cb5ca461 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -61,7 +61,7 @@ dependencies = [ "pycrdt==0.14.1", "pydantic==2.13.4", "pydantic-ai-slim[openai,mistral,logfire,web]==1.107.1", - "PyJWT==2.13.0", + "PyJWT[crypto]==2.13.0", "python-magic==0.4.27", "redis<6.0.0", "requests==2.34.2", diff --git a/src/backend/uv.lock b/src/backend/uv.lock index 5db67ffad0..e6fd18c296 100644 --- a/src/backend/uv.lock +++ b/src/backend/uv.lock @@ -965,7 +965,7 @@ dependencies = [ { name = "pycrdt" }, { name = "pydantic" }, { name = "pydantic-ai-slim", extra = ["logfire", "mistral", "openai", "web"] }, - { name = "pyjwt" }, + { name = "pyjwt" , extra = ["crypto"] }, { name = "python-magic" }, { name = "redis" }, { name = "requests" }, @@ -1042,7 +1042,7 @@ requires-dist = [ { name = "pydantic", specifier = "==2.13.4" }, { name = "pydantic-ai-slim", extras = ["openai", "mistral", "logfire", "web"], specifier = "==1.107.1" }, { name = "pyfakefs", marker = "extra == 'dev'", specifier = "==6.2.0" }, - { name = "pyjwt", specifier = "==2.13.0" }, + { name = "pyjwt", extras = ["crypto"], specifier = "==2.13.0" }, { name = "pylint", marker = "extra == 'dev'", specifier = "==4.0.6" }, { name = "pylint-django", marker = "extra == 'dev'", specifier = "==2.8.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = "==9.1.1" }, @@ -1978,6 +1978,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, ] +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + [[package]] name = "pylint" version = "4.0.6" From cb58076edda7b722a8a62e75d9d70d1b5e13e7ea Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Tue, 4 Aug 2026 16:05:18 +0200 Subject: [PATCH 04/59] =?UTF-8?q?=E2=9C=A8(backend)=20publish=20the=20JWT?= =?UTF-8?q?=20public=20key=20on=20a=20JWKS=20endpoint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The yhub service will need our public key in order to validate the jwt token we will used. We choose to expose a jwks endpoint as it is a standard wat to do this. --- CHANGELOG.md | 3 + UPGRADE.md | 7 + documentation/resource_server.md | 5 + src/backend/core/api/viewsets.py | 24 +++ src/backend/core/services/jwt_services.py | 60 ++++++- src/backend/core/tests/test_api_jwks.py | 166 ++++++++++++++++++ .../core/tests/test_services_jwt_services.py | 41 ++++- src/backend/core/urls.py | 13 +- src/backend/pyproject.toml | 1 + src/backend/uv.lock | 2 + 10 files changed, 317 insertions(+), 5 deletions(-) create mode 100644 src/backend/core/tests/test_api_jwks.py diff --git a/CHANGELOG.md b/CHANGELOG.md index de422ea340..28dfa185a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,15 @@ and this project adheres to ### Added - ✨(backend) add a service generating cached RS256 JWT tokens +- ✨(backend) publish the JWT public key on a JWKS endpoint - ♿️(frontend) restore skip to content link after header redesign #2510 - 🌐(i18n) rename cn_CN to zh_CN, add eo_PL and zh_TW locales #2486 - ✨(backend) conditional email notification in server to server api #2554 ### Changed +- 💥(backend) move the resource server JWKS from `/api/{version}/jwks` to + `/external_api/{version}/jwks` - ♿️(frontend) use semantic `
` structure in document info card #2379 - 💄(frontend) use the same highlight color for cells and moves #2575 diff --git a/UPGRADE.md b/UPGRADE.md index a142fb47f4..feafc80ff9 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -16,6 +16,13 @@ the following command inside your docker container: ## [Unreleased] +- The JWKS of the resource server moved from `/api/{version}/jwks` to + `/external_api/{version}/jwks`, alongside the rest of the resource server + endpoints. `/api/{version}/jwks` now publishes the public key validating the + tokens Docs issues to call external services. If you enabled the resource + server (`OIDC_RESOURCE_SERVER_ENABLED`), update the JWKS URI declared to your + OIDC provider accordingly. + ### [5.0.0] - 2026-04-30 We made several changes around document content management leading to several breaking changes in the API. diff --git a/documentation/resource_server.md b/documentation/resource_server.md index d2d3531159..115fe4ac7f 100644 --- a/documentation/resource_server.md +++ b/documentation/resource_server.md @@ -20,6 +20,11 @@ OIDC_RS_ALLOWED_AUDIENCES= It implements the resource server using `django-lasuite`, see the [documentation](https://github.com/suitenumerique/django-lasuite/blob/main/documentation/how-to-use-oidc-resource-server-backend.md) +When `OIDC_RS_PRIVATE_KEY_STR` is set, the resource server publishes its public +key on `/external_api/{version}/jwks`. This is the URI to declare to your OIDC +provider. Do not confuse it with `/api/{version}/jwks`, which publishes the key +validating the tokens Docs itself issues to call external services. + ## Customise allowed routes Configure the `EXTERNAL_API` setting to control which routes and actions are available in the external API. Set it via the `EXTERNAL_API` environment variable (as JSON) or in Django settings. diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index 5d9991bcbf..292aa9084d 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -64,6 +64,10 @@ from core.services.converter_services import ( ValidationError as YProviderValidationError, ) +from core.services.jwt_services import ( + ConfigurationError as JWTConfigurationError, +) +from core.services.jwt_services import JWTService from core.services.search_indexers import ( get_document_indexer, get_visited_document_ids_of, @@ -3146,6 +3150,26 @@ def _load_theme_customization(self): return theme_customization +class JWKSView(drf.views.APIView): + """API ViewSet exposing the public key validating the tokens we issue.""" + + authentication_classes = [] + permission_classes = [AllowAny] + + def get(self, request): + """ + GET /api/v1.0/jwks + Return the JSON Web Key Set of the tokens issued by this service. + """ + try: + jwks = JWTService().get_jwks() + except JWTConfigurationError: + logger.exception("Unable to publish the JWKS") + raise drf.exceptions.NotFound("No JWKS available.") from None + + return drf.response.Response(jwks) + + class CommentViewSetMixin: """Comment ViewSet Mixin.""" diff --git a/src/backend/core/services/jwt_services.py b/src/backend/core/services/jwt_services.py index a57fa6acdf..5a4ac4eb74 100644 --- a/src/backend/core/services/jwt_services.py +++ b/src/backend/core/services/jwt_services.py @@ -1,5 +1,6 @@ """JWT services.""" +import functools import hashlib import json import logging @@ -10,6 +11,7 @@ from django.utils import timezone import jwt +from joserfc.jwk import KeySet, RSAKey logger = logging.getLogger(__name__) @@ -29,6 +31,33 @@ class TokenGenerationError(JWTError): """Raised when a token cannot be signed.""" +@functools.cache +def import_private_key(private_key): + """ + Import a PEM encoded RSA private key as a JWK. + + The "kid" is the RFC 7638 thumbprint of the key, so it is stable across + restarts and changes on its own when the key is rotated. It is computed + from the public components only, which lets a consumer of the JWKS match + it against the "kid" advertised in the header of our tokens. + + Parsing a RSA key is expensive, hence the cache. It is keyed on the PEM + itself so that rotating the key in the settings imports the new one. + """ + try: + key = RSAKey.import_key(private_key) + return RSAKey.import_key( + private_key, + parameters={ + "alg": ALGORITHM, + "use": "sig", + "kid": key.thumbprint(), + }, + ) + except (TypeError, ValueError) as err: + raise ConfigurationError("The JWT private key cannot be imported.") from err + + class JWTService: """ Service class issuing RS256 signed JSON Web Tokens. @@ -56,6 +85,26 @@ def lifetime(self): """Return the token lifetime, in seconds.""" return settings.JWT_TOKEN_LIFETIME + @property + def key(self): + """Return the signing key, as a JWK.""" + return import_private_key(self.private_key) + + @property + def kid(self): + """Return the identifier of the signing key, as advertised in the JWKS.""" + return self.key.kid + + def get_jwks(self): + """ + Return the JSON Web Key Set publishing the public part of our key. + + External services validating our tokens fetch it to get the public key + matching the "kid" of the token they received. It never exposes the + private components of the key. + """ + return KeySet([self.key]).as_dict(private=False) + def get_cache_key(self, claims): """ Build the cache key identifying a token for the given claims. @@ -80,7 +129,9 @@ def generate_token(self, claims): Sign a new token embedding the given claims. The "iat" and "exp" claims are always set by the service, from the - configured lifetime, and take precedence over the caller's claims. + configured lifetime, and take precedence over the caller's claims. The + header carries the "kid" of the signing key, so that a service + validating the token can pick the matching key in our JWKS. """ issued_at = timezone.now() payload = { @@ -90,7 +141,12 @@ def generate_token(self, claims): } try: - return jwt.encode(payload, self.private_key, algorithm=self.algorithm) + return jwt.encode( + payload, + self.private_key, + algorithm=self.algorithm, + headers={"kid": self.kid}, + ) except (jwt.PyJWTError, TypeError, ValueError) as err: logger.exception( "Unable to sign a JWT token with algorithm %s", self.algorithm diff --git a/src/backend/core/tests/test_api_jwks.py b/src/backend/core/tests/test_api_jwks.py new file mode 100644 index 0000000000..3c76bd1fb5 --- /dev/null +++ b/src/backend/core/tests/test_api_jwks.py @@ -0,0 +1,166 @@ +""" +Tests for the JWKS endpoint publishing the public key of the tokens we issue. +""" + +from django.urls import resolve + +import jwt +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from rest_framework.test import APIClient + +from core.services.jwt_services import JWTService +from core.tests.utils.urls import reload_urls + +pytestmark = pytest.mark.django_db + +# Private members of a RSA JWK, none of them may ever leak in the JWKS +PRIVATE_JWK_MEMBERS = {"d", "p", "q", "dp", "dq", "qi", "oth"} + + +def generate_private_key(): + """Generate a PEM encoded RSA private key.""" + return ( + rsa.generate_private_key(public_exponent=65537, key_size=2048) + .private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + .decode("utf-8") + ) + + +# Generating RSA keys is expensive, do it once for the whole module +PRIVATE_KEY = generate_private_key() +OTHER_PRIVATE_KEY = generate_private_key() + + +@pytest.fixture(name="jwt_settings") +def jwt_settings_fixture(settings): + """Setup valid settings for the JWT service.""" + settings.JWT_PRIVATE_KEY = PRIVATE_KEY + settings.JWT_TOKEN_LIFETIME = 3600 + return settings + + +@pytest.mark.usefixtures("jwt_settings") +def test_api_jwks_is_public(): + """External services must reach the JWKS without authenticating.""" + response = APIClient().get("/api/v1.0/jwks") + + assert response.status_code == 200 + assert len(response.json()["keys"]) == 1 + + +@pytest.mark.usefixtures("jwt_settings") +def test_api_jwks_publishes_a_signature_key(): + """The published key advertises what it is meant to be used for.""" + key = APIClient().get("/api/v1.0/jwks").json()["keys"][0] + + assert key["kty"] == "RSA" + assert key["alg"] == "RS256" + assert key["use"] == "sig" + assert key["kid"] + + +@pytest.mark.usefixtures("jwt_settings") +def test_api_jwks_never_exposes_the_private_key(): + """🔒 The JWKS exposes the public components of the key, and nothing else.""" + key = APIClient().get("/api/v1.0/jwks").json()["keys"][0] + + assert PRIVATE_JWK_MEMBERS & set(key) == set() + assert set(key) == {"kty", "alg", "use", "kid", "n", "e"} + + +@pytest.mark.usefixtures("jwt_settings") +def test_api_jwks_key_validates_the_tokens_we_issue(): + """ + The whole point of the endpoint: a service fetching the JWKS can validate + a token we issued, the way an external service does. + """ + token = JWTService().get_token({"sub": "user-id", "scope": "read"}) + + jwks = APIClient().get("/api/v1.0/jwks").json() + + # This is what an external service does with the JWKS we serve + key = jwt.PyJWKSet.from_dict(jwks).keys[0] + payload = jwt.decode(token, key, algorithms=["RS256"]) + + assert payload["sub"] == "user-id" + assert payload["scope"] == "read" + + +@pytest.mark.usefixtures("jwt_settings") +def test_api_jwks_key_id_matches_the_token_header(): + """A consumer selects the right key by matching the "kid" of the token.""" + token = JWTService().get_token({"sub": "user-id"}) + + jwks = APIClient().get("/api/v1.0/jwks").json() + + kid = jwt.get_unverified_header(token)["kid"] + assert [key["kid"] for key in jwks["keys"]] == [kid] + + +def test_api_jwks_follows_the_key_rotation(jwt_settings): + """After a rotation, the JWKS validates the tokens signed with the new key.""" + first_jwks = APIClient().get("/api/v1.0/jwks").json() + + jwt_settings.JWT_PRIVATE_KEY = OTHER_PRIVATE_KEY + token = JWTService().get_token({"sub": "user-id"}) + second_jwks = APIClient().get("/api/v1.0/jwks").json() + + assert first_jwks != second_jwks + + key = jwt.PyJWKSet.from_dict(second_jwks).keys[0] + assert jwt.decode(token, key, algorithms=["RS256"])["sub"] == "user-id" + + # The retired key can no longer validate the new tokens + with pytest.raises(jwt.InvalidSignatureError): + jwt.decode( + token, jwt.PyJWKSet.from_dict(first_jwks).keys[0], algorithms=["RS256"] + ) + + +@pytest.mark.parametrize("private_key", [None, ""]) +def test_api_jwks_without_private_key(jwt_settings, private_key): + """Without a configured key there is nothing to publish.""" + jwt_settings.JWT_PRIVATE_KEY = private_key + + assert APIClient().get("/api/v1.0/jwks").status_code == 404 + + +def test_api_jwks_with_an_invalid_private_key(jwt_settings): + """An unusable key is reported as a missing JWKS, not as a server error.""" + jwt_settings.JWT_PRIVATE_KEY = "not-a-pem-key" + + assert APIClient().get("/api/v1.0/jwks").status_code == 404 + + +@pytest.mark.usefixtures("jwt_settings", "resource_server_backend_conf") +def test_api_jwks_does_not_shadow_the_resource_server_jwks(settings): + """ + The resource server publishes its own JWKS, holding its encryption key. + Both must stay reachable, on their own path. + """ + settings.OIDC_RS_PRIVATE_KEY_STR = PRIVATE_KEY + reload_urls() + + assert resolve("/api/v1.0/jwks").url_name == "jwks" + assert resolve("/external_api/v1.0/jwks").url_name == "resource_server_jwks" + + ours = APIClient().get("/api/v1.0/jwks").json()["keys"][0] + theirs = APIClient().get("/external_api/v1.0/jwks").json()["keys"][0] + + assert ours["use"] == "sig" + assert theirs["use"] == "enc" + + +@pytest.mark.parametrize("method", ["post", "put", "patch", "delete"]) +@pytest.mark.usefixtures("jwt_settings") +def test_api_jwks_is_read_only(method): + """The JWKS is only exposed for reading.""" + response = getattr(APIClient(), method)("/api/v1.0/jwks") + + assert response.status_code == 405 diff --git a/src/backend/core/tests/test_services_jwt_services.py b/src/backend/core/tests/test_services_jwt_services.py index 423fc3e99d..6e77a2dd5e 100644 --- a/src/backend/core/tests/test_services_jwt_services.py +++ b/src/backend/core/tests/test_services_jwt_services.py @@ -150,6 +150,36 @@ def test_get_token_caches_each_set_of_claims_separately(): ) +@pytest.mark.usefixtures("jwt_settings") +def test_get_jwks_exposes_only_the_public_key(): + """🔒 The JWKS must never carry the private components of the key.""" + keys = JWTService().get_jwks()["keys"] + + assert len(keys) == 1 + assert set(keys[0]) == {"kty", "alg", "use", "kid", "n", "e"} + + +@pytest.mark.usefixtures("jwt_settings") +def test_kid_is_stable_and_matches_the_signed_tokens(): + """The "kid" identifies the key across the JWKS and the tokens.""" + service = JWTService() + + assert service.kid == JWTService().kid + assert ( + jwt.get_unverified_header(service.get_token({"sub": "user-id"}))["kid"] + == service.kid + ) + + +def test_kid_changes_when_the_key_is_rotated(jwt_settings): + """A rotated key is a different key, hence a different "kid".""" + kid = JWTService().kid + + jwt_settings.JWT_PRIVATE_KEY = OTHER_PRIVATE_KEY + + assert JWTService().kid != kid + + @pytest.mark.usefixtures("jwt_settings") def test_get_token_ignores_the_claims_ordering(): """Claims given in a different order hit the same cache entry.""" @@ -200,8 +230,15 @@ def test_get_token_without_private_key(jwt_settings, private_key): def test_generate_token_with_an_invalid_private_key(jwt_settings): - """An unusable private key is reported as a token generation error.""" + """An unusable private key is a configuration problem.""" jwt_settings.JWT_PRIVATE_KEY = "not-a-pem-key" - with pytest.raises(TokenGenerationError, match="Unable to sign the JWT token"): + with pytest.raises(ConfigurationError, match="cannot be imported"): JWTService().generate_token({"sub": "user-id"}) + + +@pytest.mark.usefixtures("jwt_settings") +def test_generate_token_with_claims_that_cannot_be_serialized(): + """Claims that cannot be encoded are reported as a generation error.""" + with pytest.raises(TokenGenerationError, match="Unable to sign the JWT token"): + JWTService().generate_token({"sub": {"unserializable"}}) diff --git a/src/backend/core/urls.py b/src/backend/core/urls.py index e89618650b..cdcf8f94ca 100644 --- a/src/backend/core/urls.py +++ b/src/backend/core/urls.py @@ -82,6 +82,14 @@ ), ), path(f"api/{settings.API_VERSION}/config/", viewsets.ConfigView.as_view()), + # Public keys validating the tokens we issue to call external services. + # Nested under "api/" because this is the only prefix routed to the backend + # by the ingress, a root "/.well-known/" would be served by the frontend. + path( + f"api/{settings.API_VERSION}/jwks", + viewsets.JWKSView.as_view(), + name="jwks", + ), ] if settings.OIDC_RESOURCE_SERVER_ENABLED: @@ -120,9 +128,12 @@ ) if settings.OIDC_RS_PRIVATE_KEY_STR: + # Served under "external_api/" alongside the rest of the resource + # server, so that it does not collide with the JWKS of the tokens we + # issue, which lives at "api//jwks". urlpatterns.append( path( - f"api/{settings.API_VERSION}/", + f"external_api/{settings.API_VERSION}/", include([*oidc_resource_server_urls]), ) ) diff --git a/src/backend/pyproject.toml b/src/backend/pyproject.toml index 82cb5ca461..a26d8eff9f 100644 --- a/src/backend/pyproject.toml +++ b/src/backend/pyproject.toml @@ -49,6 +49,7 @@ dependencies = [ "emoji==2.15.0", "factory_boy==3.3.3", "gunicorn==26.0.0", + "joserfc==1.6.5", "jsonschema==4.26.0", "langfuse==3.11.2", "lxml==6.1.1", diff --git a/src/backend/uv.lock b/src/backend/uv.lock index e6fd18c296..9b66bdf98b 100644 --- a/src/backend/uv.lock +++ b/src/backend/uv.lock @@ -953,6 +953,7 @@ dependencies = [ { name = "emoji" }, { name = "factory-boy" }, { name = "gunicorn" }, + { name = "joserfc" }, { name = "jsonschema" }, { name = "langfuse" }, { name = "lxml" }, @@ -1029,6 +1030,7 @@ requires-dist = [ { name = "gunicorn", specifier = "==26.0.0" }, { name = "ipdb", marker = "extra == 'dev'", specifier = "==0.13.13" }, { name = "ipython", marker = "extra == 'dev'", specifier = "==9.15.0" }, + { name = "joserfc", specifier = "==1.6.5" }, { name = "jsonschema", specifier = "==4.26.0" }, { name = "langfuse", specifier = "==3.11.2" }, { name = "lxml", specifier = "==6.1.1" }, From 58e577b0cce40faacd61ce35d56c2b1356be2e71 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Tue, 4 Aug 2026 16:17:12 +0200 Subject: [PATCH 05/59] =?UTF-8?q?=E2=9C=A8(backend)=20add=20a=20method=20t?= =?UTF-8?q?o=20create=20a=20dedicated=20admin=20token?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For now the only token we will need is ont with the admin claim set to True. To not repeat the creation of this token again and again, we created a dedicated method to issue this token in the JWTService class. --- src/backend/core/services/jwt_services.py | 10 ++ .../core/tests/test_services_jwt_services.py | 102 ++++++++++++++++++ 2 files changed, 112 insertions(+) diff --git a/src/backend/core/services/jwt_services.py b/src/backend/core/services/jwt_services.py index 5a4ac4eb74..92fa152881 100644 --- a/src/backend/core/services/jwt_services.py +++ b/src/backend/core/services/jwt_services.py @@ -171,3 +171,13 @@ def get_token(self, claims): cache.set(cache_key, token, self.lifetime) return token + + def get_admin_token(self, claims=None): + """ + Return a token with the `admin: true` claim. + + Extra claims can be injected alongside it. They cannot turn the "admin" + claim off: a token issued by this method always grants admin. + """ + + return self.get_token({**(claims or {}), "admin": True}) diff --git a/src/backend/core/tests/test_services_jwt_services.py b/src/backend/core/tests/test_services_jwt_services.py index 6e77a2dd5e..be8b584183 100644 --- a/src/backend/core/tests/test_services_jwt_services.py +++ b/src/backend/core/tests/test_services_jwt_services.py @@ -150,6 +150,108 @@ def test_get_token_caches_each_set_of_claims_separately(): ) +@pytest.mark.usefixtures("jwt_settings") +def test_get_admin_token_carries_the_admin_claim(): + """The admin token is a regular token carrying the "admin" claim.""" + token = JWTService().get_admin_token() + + payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + assert payload["admin"] is True + + +@pytest.mark.usefixtures("jwt_settings") +def test_get_admin_token_embeds_the_extra_claims(): + """Extra claims are carried alongside the "admin" one.""" + token = JWTService().get_admin_token({"sub": "user-id", "scope": "read"}) + + payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + assert payload["admin"] is True + assert payload["sub"] == "user-id" + assert payload["scope"] == "read" + + +@pytest.mark.usefixtures("jwt_settings") +def test_get_admin_token_extra_claims_cannot_turn_admin_off(): + """🔒 A token issued by get_admin_token always grants admin.""" + token = JWTService().get_admin_token({"admin": False}) + + payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + assert payload["admin"] is True + + +@pytest.mark.usefixtures("jwt_settings") +def test_get_admin_token_caches_each_set_of_extra_claims_separately(): + """Two callers passing different extra claims get their own token.""" + service = JWTService() + + first_token = service.get_admin_token({"sub": "user-id"}) + second_token = service.get_admin_token({"sub": "other-user-id"}) + + assert first_token != second_token + assert jwt.decode(first_token, PUBLIC_KEY, algorithms=["RS256"])["sub"] == "user-id" + assert ( + jwt.decode(second_token, PUBLIC_KEY, algorithms=["RS256"])["sub"] + == "other-user-id" + ) + + +@pytest.mark.usefixtures("jwt_settings") +def test_get_admin_token_does_not_mutate_the_given_claims(): + """The caller's dictionary is left untouched.""" + claims = {"sub": "user-id"} + + JWTService().get_admin_token(claims) + + assert claims == {"sub": "user-id"} + + +@pytest.mark.usefixtures("jwt_settings") +def test_get_admin_token_reuses_the_cached_token(): + """The admin token is cached, like any other token.""" + service = JWTService() + + token = service.get_admin_token() + + with mock.patch("core.services.jwt_services.jwt.encode") as mock_encode: + assert service.get_admin_token() == token + + mock_encode.assert_not_called() + + +@pytest.mark.usefixtures("jwt_settings") +def test_get_admin_token_is_not_served_to_a_non_admin_caller(): + """ + 🔒 The admin token has its own cache entry. Asking for any other set of + claims must never hand out a token granting admin. + """ + service = JWTService() + + admin_token = service.get_admin_token() + tokens = [ + service.get_token({"admin": False}), + service.get_token({"sub": "user-id"}), + service.get_token({}), + ] + + assert admin_token not in tokens + for token in tokens: + assert ( + jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]).get("admin") is not True + ) + + +def test_get_admin_token_expires_like_any_other_token(jwt_settings): + """The admin token does not outlive the configured lifetime.""" + jwt_settings.JWT_TOKEN_LIFETIME = 120 + + now = datetime(2026, 8, 4, 10, 0, 0, tzinfo=timezone.utc) + with freeze_time(now): + token = JWTService().get_admin_token() + payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + + assert payload["exp"] == now.timestamp() + 120 + + @pytest.mark.usefixtures("jwt_settings") def test_get_jwks_exposes_only_the_public_key(): """🔒 The JWKS must never carry the private components of the key.""" From 1cb1c23ffe00564db364f07ae495ef35bb6af627 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Tue, 4 Aug 2026 16:33:41 +0200 Subject: [PATCH 06/59] =?UTF-8?q?=F0=9F=94=A7(dev)=20generate=20the=20JWT?= =?UTF-8?q?=20signing=20key=20when=20bootstrapping=20the=20dev=20stack?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thw private key needed to generate a jwt token will be mandatory. In order to ease the development we want to automate its generation --- CHANGELOG.md | 1 + Makefile | 9 ++++++++- bin/_config.sh | 4 ++++ bin/generate-jwt-private-key.sh | 23 +++++++++++++++++++++++ compose.yml | 2 ++ env.d/development/common | 5 +++++ 6 files changed, 43 insertions(+), 1 deletion(-) create mode 100755 bin/generate-jwt-private-key.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 28dfa185a0..7b3c6e9394 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to - ✨(backend) add a service generating cached RS256 JWT tokens - ✨(backend) publish the JWT public key on a JWKS endpoint +- 🔧(dev) generate the JWT signing key when bootstrapping the dev stack - ♿️(frontend) restore skip to content link after header redesign #2510 - 🌐(i18n) rename cn_CN to zh_CN, add eo_PL and zh_TW locales #2486 - ✨(backend) conditional email notification in server to server api #2554 diff --git a/Makefile b/Makefile index fcac8eff2b..6f0a033b2a 100644 --- a/Makefile +++ b/Makefile @@ -69,6 +69,11 @@ data/media: data/static: @mkdir -p data/static +# RSA key signing the JWT tokens the backend issues. Generated locally, never +# committed: "data/" is gitignored. Regenerate it by deleting the file. +data/jwt/private.pem: + @bin/generate-jwt-private-key.sh + # -- Project create-env-local-files: ## create env.local files in env.d/development @@ -81,7 +86,8 @@ create-env-local-files: .PHONY: create-env-local-files generate-secret-keys: -generate-secret-keys: ## generate secret keys to be stored in common.local +generate-secret-keys: ## generate the secret keys needed by the dev stack +generate-secret-keys: data/jwt/private.pem @bin/generate-oidc-store-refresh-token-key.sh .PHONY: generate-secret-keys @@ -237,6 +243,7 @@ logs: ## display app-dev logs (follow mode) run-backend: ## Start only the backend application and all needed services @$(MAKE) create-docker-network + @$(MAKE) data/jwt/private.pem @$(COMPOSE) up --force-recreate -d docspec @$(COMPOSE) up --force-recreate -d celery-dev @$(COMPOSE) up --force-recreate -d y-provider-development-converter diff --git a/bin/_config.sh b/bin/_config.sh index b317352f5b..376de65ece 100644 --- a/bin/_config.sh +++ b/bin/_config.sh @@ -38,6 +38,10 @@ function _set_user() { # options: docker compose command options # ARGS : docker compose command arguments function _docker_compose() { + # The backend settings point at this key and the containers mount it, so it + # has to exist before any of them starts. + "${REPO_DIR}/bin/generate-jwt-private-key.sh" + # Set DOCKER_USER for Windows compatibility with MinIO if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" || -n "${WSL_DISTRO_NAME:-}" ]]; then export DOCKER_USER="0:0" diff --git a/bin/generate-jwt-private-key.sh b/bin/generate-jwt-private-key.sh new file mode 100755 index 0000000000..3d2963bd0c --- /dev/null +++ b/bin/generate-jwt-private-key.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# Generate the RSA private key signing the JWT tokens issued by the backend. +# +# Development only. The key is generated locally and never committed: it lands +# in "data/", which is gitignored. The dev stack mounts it in the backend +# containers, where JWT_PRIVATE_KEY_FILE points at it. +# +# Idempotent: an existing key is kept. Delete the file to roll the key. + +set -eo pipefail + +REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +KEY_PATH="${REPO_DIR}/data/jwt/private.pem" + +if [ -f "${KEY_PATH}" ]; then + exit 0 +fi + +mkdir -p "$(dirname "${KEY_PATH}")" +openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${KEY_PATH}" 2>/dev/null +chmod 600 "${KEY_PATH}" +echo "✓ JWT private key generated in ${KEY_PATH}" diff --git a/compose.yml b/compose.yml index b8ac0bf474..7f7b95f8b6 100644 --- a/compose.yml +++ b/compose.yml @@ -82,6 +82,7 @@ services: volumes: - ./src/backend:/app - ./data/static:/data/static + - ./data/jwt:/data/jwt:ro - /app/.venv depends_on: postgresql: @@ -111,6 +112,7 @@ services: volumes: - ./src/backend:/app - ./data/static:/data/static + - ./data/jwt:/data/jwt:ro - /app/.venv depends_on: - app-dev diff --git a/env.d/development/common b/env.d/development/common index ef3a8010bf..c1a438c74f 100644 --- a/env.d/development/common +++ b/env.d/development/common @@ -22,6 +22,11 @@ DJANGO_EMAIL_LOGO_IMG="http://localhost:3000/assets/logo-suite-numerique.png" DJANGO_EMAIL_PORT=1025 DJANGO_EMAIL_URL_APP="http://localhost:3000" +# JWT +# The key itself is generated locally by "make generate-secret-keys", it is +# never committed. A PEM does not fit in an env var, hence the _FILE variant. +JWT_PRIVATE_KEY_FILE=/data/jwt/private.pem + # Backend url IMPRESS_BASE_URL="http://localhost:8072" From 7cbee2f26a960c9e77db16ad42dd80f52f056393 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Tue, 4 Aug 2026 17:13:20 +0200 Subject: [PATCH 07/59] =?UTF-8?q?=F0=9F=94=A5(ci)=20remove=20checking=20pr?= =?UTF-8?q?int=20statement=20in=20lint-git?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since we use ruff, it is not needed anymore to check the presence of print statement, the rule T201 is already doing it in a more performant way. --- .github/workflows/impress.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/impress.yml b/.github/workflows/impress.yml index 9aae0cd288..b0c6fbc319 100644 --- a/.github/workflows/impress.yml +++ b/.github/workflows/impress.yml @@ -27,10 +27,6 @@ jobs: fetch-depth: 0 - name: show run: git log - - name: Enforce absence of print statements in code - if: always() - run: | - ! git diff origin/${{ github.event.pull_request.base.ref }}..HEAD -- src/backend ':(exclude)**/impress.yml' | grep "print(" - name: Check absence of fixup commits if: always() run: | From 27c1f8d1b4f59335da638e280ff4a9e8920f7298 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Tue, 4 Aug 2026 18:47:07 +0200 Subject: [PATCH 08/59] =?UTF-8?q?=F0=9F=94=A5(backend)=20remove=20`Collabo?= =?UTF-8?q?rationService`=20and=20`can-edit`=20endpoint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CollaborationService was doing nothing since we started the migration to yhub, all the code using it is now removed. Also the `can-edit` endpoint and all the safeguard mechanism relying on the presence of other users connected to the websocket will not be used anymore, it will be possible to replace all of this with yhub, so all this code is also removed. --- CHANGELOG.md | 2 + documentation/collaboration.md | 27 +- documentation/env.md | 4 - .../examples/helm/impress.values.yaml | 1 - env.d/development/common | 1 - env.d/production.dist/yprovider | 1 - src/backend/core/api/serializers.py | 3 - src/backend/core/api/viewsets.py | 109 +--- src/backend/core/models.py | 1 - .../core/services/collaboration_services.py | 40 -- src/backend/core/tasks/access.py | 14 - .../documents/test_api_document_accesses.py | 149 ++--- .../documents/test_api_documents_can_edit.py | 203 ------- .../test_api_documents_content_update.py | 219 +------- .../test_api_documents_link_configuration.py | 85 +-- .../documents/test_api_documents_retrieve.py | 5 - .../documents/test_api_documents_trashbin.py | 2 - .../documents/test_api_documents_update.py | 522 +----------------- ...ternal_api_documents_link_configuration.py | 8 +- src/backend/core/tests/test_api_config.py | 2 - .../core/tests/test_models_documents.py | 11 - .../test_services_collaboration_services.py | 30 - src/backend/core/tests/test_tasks_access.py | 31 -- src/backend/impress/settings.py | 25 - src/helm/env.d/dev/values.impress.yaml.gotmpl | 3 - .../env.d/feature/values.impress.yaml.gotmpl | 3 - 26 files changed, 95 insertions(+), 1406 deletions(-) delete mode 100644 src/backend/core/services/collaboration_services.py delete mode 100644 src/backend/core/tasks/access.py delete mode 100644 src/backend/core/tests/documents/test_api_documents_can_edit.py delete mode 100644 src/backend/core/tests/test_services_collaboration_services.py delete mode 100644 src/backend/core/tests/test_tasks_access.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b3c6e9394..185474fedd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,8 @@ and this project adheres to the dev stack gains dedicated valkey and postgres services for yhub, and the kick (reset-connections) and get-connections APIs have no yhub equivalent yet — they are deferred with TODO(yhub) stubs +- 🔥(backend) remove the unused `CollaborationService` +- 💥(backend) remove the `documents/{id}/can-edit/` endpoint - 💥(y-provider) the published `lasuite/impress-y-provider` image becomes converter-only and no longer serves `/collaboration/ws/`; deployments using the existing helm values lose collaboration until the helm chart routes diff --git a/documentation/collaboration.md b/documentation/collaboration.md index 2375ee1ba3..873a87a808 100644 --- a/documentation/collaboration.md +++ b/documentation/collaboration.md @@ -1,36 +1,13 @@ # Collaboration -By default with Docs, collaboration is enabled. To allow the collaboration between users, a connection to a websocket server is made (the y-provider service), you only have to configure the Django backend URL in your y-provider service: +By default with Docs, collaboration is enabled. To allow the collaboration between users, a connection to a websocket server is made (the y-provider service), you only have to configure the Django backend URL and the allowed origin in your y-provider service: ```yaml COLLABORATION_BACKEND_BASE_URL: https://{yourdocsdomain.tld} +COLLABORATION_SERVER_ORIGIN: https://{yourdocsdomain.tld} ``` -An advanced configuration can be used in some cases when your users are not allowed to use websocket on their network. - ## What happens when connection to the websocket is not allowed? When multiple users access a Docs and the connection to the websocket is not allowed, then they will be in a situation where they can lose data. They will lose data because they will erase each other modifications. You can also have a scenario with a mix of users connected to the websocket and some other not. - -## Safeguard configuration - -We have imagined a safeguard scenario, not enabled by default. -The idea is to give the priority to users connected to the websocket. While there is at least one user connected to the websocket, all other users not connected to the websocket can access the Docs in **read-only** mode. - -To enable this safeguard, the Django application will have to fetch the `y-provider` service to retrieve some information in it. - -In the Django configuration, you have to set these environment variables: - -```yaml -COLLABORATION_WS_NOT_CONNECTED_READ_ONLY: True -COLLABORATION_API_URL: https://{yourdocsdomain.tld}/collaboration/api/ -COLLABORATION_SERVER_SECRET: A-shared-secret-with-y-provider-service -``` - -In the y-provider service, you have to set these environment variables: - -```yaml -COLLABORATION_SERVER_SECRET: A-shared-secret-with-y-provider-service -COLLABORATION_SERVER_ORIGIN: https://{yourdocsdomain.tld} -``` diff --git a/documentation/env.md b/documentation/env.md index 65fe38b476..d5ef078c99 100644 --- a/documentation/env.md +++ b/documentation/env.md @@ -32,10 +32,7 @@ These are the environment variables you can set for the `impress-backend` contai | AWS_STORAGE_BUCKET_NAME | Bucket name for s3 endpoint | impress-media-storage | | CACHES_DEFAULT_TIMEOUT | Cache default timeout | 30 | | CACHES_DEFAULT_KEY_PREFIX | The prefix used to every cache keys. | docs | -| COLLABORATION_API_URL | Collaboration api host | | -| COLLABORATION_SERVER_SECRET | Collaboration api secret | | | COLLABORATION_WS_INACTIVITY_TIMEOUT | Timeout (in seconds) after which the user is considered inactive when there is no activity. The WebSocket is closed after this inactivity period. `None` means disabled. | None | -| COLLABORATION_WS_NOT_CONNECTED_READ_ONLY | Users not connected to the collaboration server cannot edit | false | | COLLABORATION_WS_URL | Collaboration websocket url | | | CONVERSION_API_CONTENT_FIELD | Conversion api content field | content | | CONVERSION_API_ENDPOINT | Conversion API endpoint | convert | @@ -98,7 +95,6 @@ These are the environment variables you can set for the `impress-backend` contai | MALWARE_DETECTION_PARAMETERS | A dict containing all the parameters to initiate the malware detection backend | {"callback_path": "core.malware_detection.malware_detection_callback",} | | MEDIA_BASE_URL | | | | MEDIA_AUTH_ORIGINAL_URL_HEADER | Parameter containing the original request URL, as seen at the media auth endpoint, in CGI/WSGI form (HTTP_HEADER_NAME_ALL_CAPS_WITH_UNDERSCORES) | HTTP_X_ORIGINAL_URL | -| NO_WEBSOCKET_CACHE_TIMEOUT | Cache used to store current editor session key when only users without websocket are editing a document | 120 | | OIDC_ALLOW_DUPLICATE_EMAILS | Allow duplicate emails | false | | OIDC_AUTH_REQUEST_EXTRA_PARAMS | OIDC extra auth parameters | {} | | OIDC_CREATE_USER | Create used on OIDC | false | diff --git a/documentation/examples/helm/impress.values.yaml b/documentation/examples/helm/impress.values.yaml index 9f07fbe584..ba965ade5e 100644 --- a/documentation/examples/helm/impress.values.yaml +++ b/documentation/examples/helm/impress.values.yaml @@ -15,7 +15,6 @@ image: backend: replicas: 1 envVars: - COLLABORATION_SERVER_SECRET: my-secret DJANGO_CSRF_TRUSTED_ORIGINS: https://docs.127.0.0.1.nip.io DJANGO_CONFIGURATION: Feature DJANGO_ALLOWED_HOSTS: docs.127.0.0.1.nip.io diff --git a/env.d/development/common b/env.d/development/common index c1a438c74f..498bcf367e 100644 --- a/env.d/development/common +++ b/env.d/development/common @@ -81,7 +81,6 @@ USER_RECONCILIATION_FORM_URL=http://localhost:3000 COLLABORATION_BACKEND_BASE_URL=http://app-dev:8000 COLLABORATION_SERVER_ORIGIN=http://localhost:3000 COLLABORATION_SERVER_SECRET=my-secret -COLLABORATION_WS_NOT_CONNECTED_READ_ONLY=true COLLABORATION_WS_URL=ws://localhost:3002/ws/docs COLLABORATION_WS_INACTIVITY_TIMEOUT=15 # Seconds diff --git a/env.d/production.dist/yprovider b/env.d/production.dist/yprovider index 58e4d02fa8..5761d7ae21 100644 --- a/env.d/production.dist/yprovider +++ b/env.d/production.dist/yprovider @@ -2,6 +2,5 @@ Y_PROVIDER_API_BASE_URL=http://${YPROVIDER_HOST}:4444/api/ Y_PROVIDER_API_KEY= COLLABORATION_SERVER_SECRET= COLLABORATION_SERVER_ORIGIN=https://${DOCS_HOST} -COLLABORATION_API_URL=https://${DOCS_HOST}/collaboration/api/ COLLABORATION_BACKEND_BASE_URL=https://${DOCS_HOST} COLLABORATION_LOGGING=true \ No newline at end of file diff --git a/src/backend/core/api/serializers.py b/src/backend/core/api/serializers.py index fb35ca9abf..d6b2b422c2 100644 --- a/src/backend/core/api/serializers.py +++ b/src/backend/core/api/serializers.py @@ -180,7 +180,6 @@ class Meta: class DocumentSerializer(ListDocumentSerializer): """Serialize documents with all fields for display in detail views.""" - websocket = serializers.BooleanField(required=False, write_only=True) file = serializers.FileField( required=False, write_only=True, allow_null=True, max_length=255 ) @@ -210,7 +209,6 @@ class Meta: "title", "updated_at", "user_role", - "websocket", ] read_only_fields = [ "id", @@ -312,7 +310,6 @@ class DocumentContentSerializer(serializers.Serializer): """Serializer for updating only the raw content of a document stored in S3.""" content = serializers.CharField(required=True) - websocket = serializers.BooleanField(required=False) def validate_content(self, value): """Validate the content field.""" diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index 292aa9084d..e97be4e157 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -53,7 +53,6 @@ from core.services import mime_types from core.services.ai_services.blocknote import AIService from core.services.ai_services.legacy import get_legacy_ai_service -from core.services.collaboration_services import CollaborationService from core.services.converter_services import ( ConversionError, Converter, @@ -72,7 +71,6 @@ get_document_indexer, get_visited_document_ids_of, ) -from core.tasks.access import reset_service_connections_in_cascade from core.tasks.mail import send_ask_for_access_mail from core.utils.analytics import PosthogEventName, posthog_capture from core.utils.paths import filter_descendants @@ -761,81 +759,6 @@ def perform_destroy(self, instance): PosthogEventName.DOC_DELETED, self.request.user, {}, document=instance ) - def _can_user_edit_document(self, document_id, set_cache=False): - """Check if the user can edit the document.""" - try: - count, exists = CollaborationService().get_document_connection_info( - document_id, - self.request.session.session_key, - ) - except requests.HTTPError as e: - logger.exception("Failed to call collaboration server: %s", e) - count = 0 - exists = False - - if count == 0: - # Nobody is connected to the websocket server - logger.debug("update without connection found in the websocket server") - cache_key = f"docs:no-websocket:{document_id}" - current_editor = cache.get(cache_key) - - if not current_editor: - if set_cache: - cache.set( - cache_key, - self.request.session.session_key, - settings.NO_WEBSOCKET_CACHE_TIMEOUT, - ) - return True - - if current_editor != self.request.session.session_key: - return False - - if set_cache: - cache.touch(cache_key, settings.NO_WEBSOCKET_CACHE_TIMEOUT) - return True - - if exists: - # Current user is connected to the websocket server - logger.debug("session key found in the websocket server") - return True - - logger.debug( - "Users connected to the websocket but current editor not connected to it. Can not edit." - ) - - return False - - def perform_update(self, serializer): - """Check rules about collaboration.""" - if ( - not serializer.validated_data.get("websocket", False) - and settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY - and not self._can_user_edit_document(serializer.instance.id, set_cache=True) - ): - raise drf.exceptions.PermissionDenied( - "You are not allowed to edit this document." - ) - - return super().perform_update(serializer) - - @drf.decorators.action( - detail=True, - methods=["get"], - url_path="can-edit", - ) - def can_edit(self, request, *args, **kwargs): - """Check if the current user can edit the document.""" - document = self.get_object() - - can_edit = ( - True - if not settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY - else self._can_user_edit_document(document.id) - ) - - return drf.response.Response({"can_edit": can_edit}) - @drf.decorators.action( detail=False, methods=["get"], @@ -1829,9 +1752,6 @@ def link_configuration(self, request, *args, **kwargs): serializer.save() - # Notify collaboration server about the link updated - reset_service_connections_in_cascade.delay(str(document.id)) - return drf.response.Response(serializer.data, status=drf.status.HTTP_200_OK) @drf.decorators.action(detail=True, methods=["post", "delete"], url_path="favorite") @@ -2060,15 +1980,6 @@ def content(self, request, *args, **kwargs): serializer = serializers.DocumentContentSerializer(data=request.data) serializer.is_valid(raise_exception=True) - if ( - not serializer.validated_data.get("websocket", False) - and settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY - and not self._can_user_edit_document(document.id, set_cache=True) - ): - raise drf.exceptions.PermissionDenied( - "You are not allowed to edit this document." - ) - content = serializer.validated_data["content"] try: extracted_attachments = set(extract_attachments(content)) @@ -2835,26 +2746,12 @@ def perform_create(self, serializer): or settings.LANGUAGE_CODE, ) - def perform_update(self, serializer): - """Update an access to the document and notify the collaboration server.""" - access = serializer.save() - - access_user_id = None - if access.user: - access_user_id = str(access.user.id) - - # Notify collaboration server about the access change - reset_service_connections_in_cascade.delay( - str(access.document.id), access_user_id - ) - def perform_destroy(self, instance): - """Delete an access to the document and notify the collaboration server.""" + """Delete an access to the document.""" # Snapshot the identifiers before deletion as Django resets the primary key # on the instance once it is deleted. access_id = str(instance.id) document_id = str(instance.document_id) - user_id = str(instance.user.id) instance.delete() @@ -2864,9 +2761,6 @@ def perform_destroy(self, instance): {"access_id": access_id, "document_id": document_id}, ) - # Notify collaboration server about the access removed - reset_service_connections_in_cascade.delay(document_id, user_id) - class InvitationViewset( drf.mixins.CreateModelMixin, @@ -3084,7 +2978,6 @@ def get(self, request): "AI_FEATURE_LEGACY_ENABLED", "API_USERS_SEARCH_QUERY_MIN_LENGTH", "COLLABORATION_WS_URL", - "COLLABORATION_WS_NOT_CONNECTED_READ_ONLY", "COLLABORATION_WS_INACTIVITY_TIMEOUT", "CONVERSION_FILE_EXTENSIONS_ALLOWED", "CONVERSION_FILE_MAX_SIZE", diff --git a/src/backend/core/models.py b/src/backend/core/models.py index 158ec2c44f..e112ab27f0 100644 --- a/src/backend/core/models.py +++ b/src/backend/core/models.py @@ -1386,7 +1386,6 @@ def get_abilities(self, user): # pylint: disable=too-many-locals "ai_translate": ai_access, "attachment_upload": can_update, "media_check": can_get, - "can_edit": can_update, "children_list": can_get, "children_create": can_create_children, "collaboration_auth": can_get, diff --git a/src/backend/core/services/collaboration_services.py b/src/backend/core/services/collaboration_services.py deleted file mode 100644 index 0bf891580a..0000000000 --- a/src/backend/core/services/collaboration_services.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Collaboration services.""" - -from logging import getLogger - -logger = getLogger(__name__) - - -class CollaborationService: - """Service class for Collaboration related operations.""" - - def reset_connections(self, document_id, user_id=None): - """ - Reset the connections of a document and all its descendants in the - collaboration server. - - TODO(yhub): yhub exposes no kick API, so this is a no-op. The regression - is stronger than losing the hocuspocus disconnect: a revoked user keeps - their already-authorized websocket until it closes on its own, and the - edits they push in the meantime are durably persisted and re-served by - yhub (hocuspocus lost them with the room). Until yhub grows a kick API, - the manual remediation is yhub's rollback endpoint — per document: - `POST /rollback/{org}/{docid}` with a lib0-encoded body containing - `{"by": ""}` (see yhub API.md "Rollback"), authenticated as a - user with update ability on the document. - """ - logger.info( - "reset_connections is a no-op (no yhub kick API), document %s, user %s", - document_id, - user_id, - ) - - # pylint: disable=unused-argument - def get_document_connection_info(self, room, session_key): - """ - Get the connection info for a document. - - TODO(yhub): yhub exposes no connection-info API, so pretend nobody is - connected. Callers fall back to the cache-lock no-websocket path. - """ - return 0, False diff --git a/src/backend/core/tasks/access.py b/src/backend/core/tasks/access.py deleted file mode 100644 index 1f7dd1ea8f..0000000000 --- a/src/backend/core/tasks/access.py +++ /dev/null @@ -1,14 +0,0 @@ -"""Tasks dedicated to document's accesses.""" - -from core.services.collaboration_services import CollaborationService - -from impress.celery_app import app - - -@app.task -def reset_service_connections_in_cascade(document_id, user_id=None): - """ - For a given document_id, reset the connections of the document and all its - descendants by delegating to the CollaborationService. - """ - CollaborationService().reset_connections(document_id, user_id) diff --git a/src/backend/core/tests/documents/test_api_document_accesses.py b/src/backend/core/tests/documents/test_api_document_accesses.py index 83c4b392f2..96e3a98def 100644 --- a/src/backend/core/tests/documents/test_api_document_accesses.py +++ b/src/backend/core/tests/documents/test_api_document_accesses.py @@ -4,7 +4,6 @@ # pylint: disable=too-many-lines import random -from contextlib import contextmanager from unittest import mock from uuid import uuid4 @@ -19,25 +18,6 @@ pytestmark = pytest.mark.django_db -@pytest.fixture(name="mock_reset_connections") -def mock_reset_connections_fixture(): - """ - Provide a context manager that patches the ``reset_service_connections_in_cascade`` - Celery task and asserts its ``delay`` method is called exactly once for the given - document and user when leaving the context. - """ - - @contextmanager - def _mock_reset_connections(document_id, user_id=None): - with mock.patch( - "core.api.viewsets.reset_service_connections_in_cascade.delay" - ) as mock_delay: - yield mock_delay - mock_delay.assert_called_once_with(str(document_id), user_id) - - return _mock_reset_connections - - def test_api_document_accesses_list_anonymous(): """Anonymous users should not be allowed to list document accesses.""" document = factories.DocumentFactory() @@ -754,7 +734,6 @@ def test_api_document_accesses_update_administrator_except_owner( create_for, via, mock_user_teams, - mock_reset_connections, ): """ A user who is a direct administrator in a document should be allowed to update a user @@ -793,13 +772,12 @@ def test_api_document_accesses_update_administrator_except_owner( for field, value in new_values.items(): new_data = {**old_values, field: value} - with mock_reset_connections(document.id, str(access.user_id)): - response = client.put( - f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", - data=new_data, - format="json", - ) - assert response.status_code == 200 + response = client.put( + f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", + data=new_data, + format="json", + ) + assert response.status_code == 200 access.refresh_from_db() updated_values = serializers.DocumentAccessSerializer(instance=access).data @@ -864,7 +842,6 @@ def test_api_document_accesses_update_administrator_from_owner(via, mock_user_te def test_api_document_accesses_update_administrator_to_owner( via, mock_user_teams, - mock_reset_connections, ): """ A user who is an administrator in a document, should not be allowed to update @@ -912,13 +889,12 @@ def test_api_document_accesses_update_administrator_to_owner( assert response.status_code == 403 else: - with mock_reset_connections(document.id, str(access.user_id)): - response = client.put( - f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", - data=new_data, - format="json", - ) - assert response.status_code == 200 + response = client.put( + f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", + data=new_data, + format="json", + ) + assert response.status_code == 200 access.refresh_from_db() updated_values = serializers.DocumentAccessSerializer(instance=access).data @@ -931,7 +907,6 @@ def test_api_document_accesses_update_owner( create_for, via, mock_user_teams, - mock_reset_connections, ): """ A user who is an owner in a document should be allowed to update @@ -968,14 +943,13 @@ def test_api_document_accesses_update_owner( for field, value in new_values.items(): new_data = {**old_values, field: value} - with mock_reset_connections(document.id, str(access.user_id)): - response = client.put( - f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", - data=new_data, - format="json", - ) + response = client.put( + f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", + data=new_data, + format="json", + ) - assert response.status_code == 200 + assert response.status_code == 200 access.refresh_from_db() updated_values = serializers.DocumentAccessSerializer(instance=access).data @@ -994,7 +968,6 @@ def test_api_document_accesses_update_owner( def test_api_document_accesses_update_owner_self_root( via, mock_user_teams, - mock_reset_connections, ): """ A user who is owner of a document should be allowed to update @@ -1033,30 +1006,27 @@ def test_api_document_accesses_update_owner_self_root( # Add another owner and it should now work factories.UserDocumentAccessFactory(document=document, role="owner") - user_id = str(access.user_id) if via == USER else None - with mock_reset_connections(document.id, user_id): - response = client.put( - f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", - data={ - **old_values, - "role": new_role, - "user_id": old_values.get("user", {}).get("id") - if old_values.get("user") is not None - else None, - }, - format="json", - ) + response = client.put( + f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", + data={ + **old_values, + "role": new_role, + "user_id": old_values.get("user", {}).get("id") + if old_values.get("user") is not None + else None, + }, + format="json", + ) - assert response.status_code == 200 - access.refresh_from_db() - assert access.role == new_role + assert response.status_code == 200 + access.refresh_from_db() + assert access.role == new_role @pytest.mark.parametrize("via", VIA) def test_api_document_accesses_update_owner_self_child( via, mock_user_teams, - mock_reset_connections, ): """ A user who is owner of a document should be allowed to update @@ -1084,13 +1054,11 @@ def test_api_document_accesses_update_owner_self_child( old_values = serializers.DocumentAccessSerializer(instance=access).data new_role = random.choice(["administrator", "editor", "reader"]) - user_id = str(access.user_id) if via == USER else None - with mock_reset_connections(document.id, user_id): - response = client.put( - f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", - data={**old_values, "role": new_role}, - format="json", - ) + response = client.put( + f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", + data={**old_values, "role": new_role}, + format="json", + ) assert response.status_code == 200 access.refresh_from_db() @@ -1170,7 +1138,6 @@ def test_api_document_accesses_delete_reader_or_editor(via, role, mock_user_team def test_api_document_accesses_delete_administrators_except_owners( via, mock_user_teams, - mock_reset_connections, ): """ Users who are administrators in a document should be allowed to delete an access @@ -1199,14 +1166,13 @@ def test_api_document_accesses_delete_administrators_except_owners( assert models.DocumentAccess.objects.count() == 2 assert models.DocumentAccess.objects.filter(user=access.user).exists() - with mock_reset_connections(document.id, str(access.user_id)): - with mock.patch("core.api.viewsets.posthog_capture") as mock_capture: - response = client.delete( - f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", - ) + with mock.patch("core.api.viewsets.posthog_capture") as mock_capture: + response = client.delete( + f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", + ) - assert response.status_code == 204 - assert models.DocumentAccess.objects.count() == 1 + assert response.status_code == 204 + assert models.DocumentAccess.objects.count() == 1 # The access deletion should be tracked in PostHog mock_capture.assert_called_once_with( @@ -1255,7 +1221,6 @@ def test_api_document_accesses_delete_administrator_on_owners(via, mock_user_tea def test_api_document_accesses_delete_owners( via, mock_user_teams, - mock_reset_connections, ): """ Users should be able to delete the document access of another user @@ -1280,11 +1245,10 @@ def test_api_document_accesses_delete_owners( assert models.DocumentAccess.objects.count() == 2 assert models.DocumentAccess.objects.filter(user=access.user).exists() - with mock_reset_connections(document.id, str(access.user_id)): - with mock.patch("core.api.viewsets.posthog_capture") as mock_capture: - response = client.delete( - f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", - ) + with mock.patch("core.api.viewsets.posthog_capture") as mock_capture: + response = client.delete( + f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", + ) assert response.status_code == 204 assert models.DocumentAccess.objects.count() == 1 @@ -1327,9 +1291,7 @@ def test_api_document_accesses_delete_owners_last_owner_root(via, mock_user_team assert models.DocumentAccess.objects.count() == 2 -def test_api_document_accesses_delete_owners_last_owner_child_user( - mock_reset_connections, -): +def test_api_document_accesses_delete_owners_last_owner_child_user(): """ It should be possible to delete the last owner access from a document that is not a root. """ @@ -1345,10 +1307,9 @@ def test_api_document_accesses_delete_owners_last_owner_child_user( ) assert models.DocumentAccess.objects.count() == 2 - with mock_reset_connections(document.id, str(access.user_id)): - response = client.delete( - f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", - ) + response = client.delete( + f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", + ) assert response.status_code == 204 assert models.DocumentAccess.objects.count() == 1 @@ -1359,7 +1320,6 @@ def test_api_document_accesses_delete_owners_last_owner_child_user( ) def test_api_document_accesses_delete_owners_last_owner_child_team( mock_user_teams, - mock_reset_connections, ): """ It should be possible to delete the last owner access from a document that @@ -1378,10 +1338,9 @@ def test_api_document_accesses_delete_owners_last_owner_child_team( ) assert models.DocumentAccess.objects.count() == 2 - with mock_reset_connections(document.id, str(access.user_id)): - response = client.delete( - f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", - ) + response = client.delete( + f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", + ) assert response.status_code == 204 assert models.DocumentAccess.objects.count() == 1 diff --git a/src/backend/core/tests/documents/test_api_documents_can_edit.py b/src/backend/core/tests/documents/test_api_documents_can_edit.py deleted file mode 100644 index 4755bd93ba..0000000000 --- a/src/backend/core/tests/documents/test_api_documents_can_edit.py +++ /dev/null @@ -1,203 +0,0 @@ -"""Test the can_edit endpoint in the viewset DocumentViewSet.""" - -from django.core.cache import cache - -import pytest -from rest_framework.test import APIClient - -from core import factories - -pytestmark = pytest.mark.django_db - - -@pytest.mark.parametrize("ws_not_connected_ready_only", [True, False]) -@pytest.mark.parametrize("role", ["editor", "reader"]) -def test_api_documents_can_edit_anonymous(settings, ws_not_connected_ready_only, role): - """Anonymous users can edit documents when link_role is editor.""" - document = factories.DocumentFactory(link_reach="public", link_role=role) - client = APIClient() - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = ws_not_connected_ready_only - - response = client.get(f"/api/v1.0/documents/{document.id!s}/can-edit/") - - if role == "reader": - assert response.status_code == 401 - else: - assert response.status_code == 200 - assert response.json() == {"can_edit": True} - - -@pytest.mark.parametrize("ws_not_connected_ready_only", [True, False]) -def test_api_documents_can_edit_authenticated_no_websocket( - settings, ws_not_connected_ready_only -): - """ - A user not connected to the websocket and no other user have already updated the document, - the document can be updated. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = ws_not_connected_ready_only - - assert cache.get(f"docs:no-websocket:{document.id}") is None - - response = client.get( - f"/api/v1.0/documents/{document.id!s}/can-edit/", - ) - assert response.status_code == 200 - - assert response.json() == {"can_edit": True} - - -def test_api_documents_can_edit_authenticated_no_websocket_user_already_editing( - settings, -): - """ - A user not connected to the websocket and another user have already updated the document, - the document can not be updated. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - cache.set(f"docs:no-websocket:{document.id}", "other_session_key") - - response = client.get( - f"/api/v1.0/documents/{document.id!s}/can-edit/", - ) - assert response.status_code == 200 - assert response.json() == {"can_edit": False} - - -# TODO(yhub): removed test_api_documents_can_edit_no_websocket_other_user_connected_to_websocket -# here. yhub has no connection-info API: get_document_connection_info is stubbed to report -# nobody connected, so another user connected to the websocket can no longer block edition. -# Re-add the test once yhub exposes a connection-info API. - - -def test_api_documents_can_edit_user_connected_to_websocket(settings): - """ - A user connected to the websocket, the document can be updated. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - assert cache.get(f"docs:no-websocket:{document.id}") is None - - response = client.get( - f"/api/v1.0/documents/{document.id!s}/can-edit/", - ) - assert response.status_code == 200 - assert response.json() == {"can_edit": True} - assert cache.get(f"docs:no-websocket:{document.id}") is None - - -def test_api_documents_can_edit_websocket_server_unreachable_fallback_to_no_websocket( - settings, -): - """ - When the websocket server is unreachable, the document can be updated like if the user was - not connected to the websocket. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - assert cache.get(f"docs:no-websocket:{document.id}") is None - - response = client.get( - f"/api/v1.0/documents/{document.id!s}/can-edit/", - ) - assert response.status_code == 200 - assert response.json() == {"can_edit": True} - - -def test_api_documents_can_edit_websocket_server_unreachable_fallback_to_no_websocket_other_users( - settings, -): - """ - When the websocket server is unreachable, the behavior fallback to the no websocket one. - If an other user is already editing, the document can not be updated. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - cache.set(f"docs:no-websocket:{document.id}", "other_session_key") - - response = client.get( - f"/api/v1.0/documents/{document.id!s}/can-edit/", - ) - assert response.status_code == 200 - assert response.json() == {"can_edit": False} - - assert cache.get(f"docs:no-websocket:{document.id}") == "other_session_key" - - -def test_api_documents_can_edit_websocket_server_room_not_found( - settings, -): - """ - When the websocket server returns a 404, the document can be updated like if the user was - not connected to the websocket. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - assert cache.get(f"docs:no-websocket:{document.id}") is None - - response = client.get( - f"/api/v1.0/documents/{document.id!s}/can-edit/", - ) - assert response.status_code == 200 - assert response.json() == {"can_edit": True} - - -def test_api_documents_can_edit_websocket_server_room_not_found_other_already_editing( - settings, -): - """ - When the websocket server returns a 404 and another user is editing the document, - the response should be can-edit=False. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - cache.set(f"docs:no-websocket:{document.id}", "other_session_key") - - response = client.get( - f"/api/v1.0/documents/{document.id!s}/can-edit/", - ) - assert response.status_code == 200 - assert response.json() == {"can_edit": False} diff --git a/src/backend/core/tests/documents/test_api_documents_content_update.py b/src/backend/core/tests/documents/test_api_documents_content_update.py index 5184df4118..79530d9cf0 100644 --- a/src/backend/core/tests/documents/test_api_documents_content_update.py +++ b/src/backend/core/tests/documents/test_api_documents_content_update.py @@ -6,7 +6,6 @@ from functools import cache from uuid import uuid4 -from django.core.cache import cache as django_cache from django.core.files.storage import default_storage import pycrdt @@ -101,7 +100,7 @@ def test_api_documents_content_update_success(role, via, mock_user_teams): response = client.patch( f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_sample_ydoc(), "websocket": True}, + {"content": get_sample_ydoc()}, ) assert response.status_code == status.HTTP_204_NO_CONTENT @@ -180,7 +179,7 @@ def test_api_documents_content_update_replaces_existing(): new_content = get_sample_ydoc() response = client.patch( f"/api/v1.0/documents/{document.id!s}/content/", - {"content": new_content, "websocket": True}, + {"content": new_content}, ) assert response.status_code == status.HTTP_204_NO_CONTENT @@ -245,7 +244,7 @@ def test_api_documents_content_update_link_editor(): response = client.patch( f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_sample_ydoc(), "websocket": True}, + {"content": get_sample_ydoc()}, ) assert response.status_code == status.HTTP_204_NO_CONTENT @@ -253,213 +252,6 @@ def test_api_documents_content_update_link_editor(): assert models.Document.objects.filter(id=document.id).exists() -def test_api_documents_content_update_authenticated_no_websocket(settings): - """ - When a user updates the document content, not connected to the websocket and is the first - to update, the content should be updated. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - session_key = client.session.session_key - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - assert django_cache.get(f"docs:no-websocket:{document.id}") is None - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_sample_ydoc(), "websocket": False}, - ) - assert response.status_code == status.HTTP_204_NO_CONTENT - assert get_s3_content(document) == get_sample_ydoc() - assert django_cache.get(f"docs:no-websocket:{document.id}") == session_key - - -def test_api_documents_content_update_authenticated_no_websocket_user_already_editing( - settings, -): - """ - When a user updates the document content, not connected to the websocket and another session - is already editing, the update should be denied. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - django_cache.set(f"docs:no-websocket:{document.id}", "other_session_key") - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_sample_ydoc(), "websocket": False}, - ) - assert response.status_code == status.HTTP_403_FORBIDDEN - assert response.json() == {"detail": "You are not allowed to edit this document."} - - -# TODO(yhub): removed -# test_api_documents_content_update_no_websocket_other_user_connected_to_websocket -# here. yhub has no connection-info API: get_document_connection_info is stubbed to report -# nobody connected, so another user connected to the websocket can no longer block the update. -# Re-add the test once yhub exposes a connection-info API. - - -def test_api_documents_content_update_user_connected_to_websocket(settings): - """ - When a user updates document content and is connected to the websocket, - the content should be updated. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - session_key = client.session.session_key - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - assert django_cache.get(f"docs:no-websocket:{document.id}") is None - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_sample_ydoc(), "websocket": False}, - ) - assert response.status_code == status.HTTP_204_NO_CONTENT - assert get_s3_content(document) == get_sample_ydoc() - # TODO(yhub): the stubbed connection info reports nobody connected, so the - # no-websocket cache lock is taken even though the user is connected. - assert django_cache.get(f"docs:no-websocket:{document.id}") == session_key - - -def test_api_documents_content_update_websocket_server_unreachable_fallback_to_no_websocket( - settings, -): - """ - When the websocket server is unreachable, the content should be updated like if the user - was not connected to the websocket. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - session_key = client.session.session_key - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - assert django_cache.get(f"docs:no-websocket:{document.id}") is None - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_sample_ydoc(), "websocket": False}, - ) - assert response.status_code == status.HTTP_204_NO_CONTENT - assert get_s3_content(document) == get_sample_ydoc() - assert django_cache.get(f"docs:no-websocket:{document.id}") == session_key - - -def test_api_content_update_websocket_server_unreachable_fallback_to_no_websocket_other_users( - settings, -): - """ - When the websocket server is unreachable, the behavior fallback to the no websocket one. - If another user is already editing, the content update should be denied. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - django_cache.set(f"docs:no-websocket:{document.id}", "other_session_key") - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_sample_ydoc(), "websocket": False}, - ) - assert response.status_code == status.HTTP_403_FORBIDDEN - assert django_cache.get(f"docs:no-websocket:{document.id}") == "other_session_key" - - -def test_api_content_update_websocket_server_room_not_found_fallback_to_no_websocket_other_users( - settings, -): - """ - When the WebSocket server does not have the room created, the logic should fallback to - no-WebSocket. If another user is already editing, the update must be denied. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - django_cache.set(f"docs:no-websocket:{document.id}", "other_session_key") - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_sample_ydoc(), "websocket": False}, - ) - assert response.status_code == status.HTTP_403_FORBIDDEN - assert django_cache.get(f"docs:no-websocket:{document.id}") == "other_session_key" - - -def test_api_documents_content_update_force_websocket_param_to_true(settings): - """ - When the websocket parameter is set to true, the content should be updated without any check. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - assert django_cache.get(f"docs:no-websocket:{document.id}") is None - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_sample_ydoc(), "websocket": True}, - ) - assert response.status_code == status.HTTP_204_NO_CONTENT - assert get_s3_content(document) == get_sample_ydoc() - assert django_cache.get(f"docs:no-websocket:{document.id}") is None - - -def test_api_documents_content_update_feature_flag_disabled(settings): - """ - When the feature flag is disabled, the content should be updated without any check. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = False - - assert django_cache.get(f"docs:no-websocket:{document.id}") is None - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_sample_ydoc(), "websocket": False}, - ) - assert response.status_code == status.HTTP_204_NO_CONTENT - assert get_s3_content(document) == get_sample_ydoc() - assert django_cache.get(f"docs:no-websocket:{document.id}") is None - - def test_api_documents_content_upadte_invalid_yjs_doc(): """sending an invalid yjs doc as content should return a 400.""" user = factories.UserFactory() @@ -473,10 +265,7 @@ def test_api_documents_content_upadte_invalid_yjs_doc(): response = client.patch( f"/api/v1.0/documents/{document.id!s}/content/", - { - "content": base64.b64encode(b"invalid yjs").decode("utf-8"), - "websocket": True, - }, + {"content": base64.b64encode(b"invalid yjs").decode("utf-8")}, ) assert response.status_code == status.HTTP_400_BAD_REQUEST diff --git a/src/backend/core/tests/documents/test_api_documents_link_configuration.py b/src/backend/core/tests/documents/test_api_documents_link_configuration.py index 9a3a3c8fdf..7252b1fc05 100644 --- a/src/backend/core/tests/documents/test_api_documents_link_configuration.py +++ b/src/backend/core/tests/documents/test_api_documents_link_configuration.py @@ -1,8 +1,5 @@ """Tests for link configuration of documents on API endpoint""" -from contextlib import contextmanager -from unittest import mock - import pytest from rest_framework.test import APIClient @@ -13,25 +10,6 @@ pytestmark = pytest.mark.django_db -@pytest.fixture(name="mock_reset_connections") -def mock_reset_connections_fixture(): - """ - Provide a context manager that patches the ``reset_service_connections_in_cascade`` - Celery task and asserts its ``delay`` method is called exactly once for the given - document when leaving the context. - """ - - @contextmanager - def _mock_reset_connections(document_id): - with mock.patch( - "core.api.viewsets.reset_service_connections_in_cascade.delay" - ) as mock_delay: - yield mock_delay - mock_delay.assert_called_once_with(str(document_id)) - - return _mock_reset_connections - - @pytest.mark.parametrize("role", models.LinkRoleChoices.values) @pytest.mark.parametrize("reach", models.LinkReachChoices.values) def test_api_documents_link_configuration_update_anonymous(reach, role): @@ -141,7 +119,6 @@ def test_api_documents_link_configuration_update_authenticated_related_success( via, role, mock_user_teams, - mock_reset_connections, # pylint: disable=redefined-outer-name ): """ A user who is administrator or owner of a document should be allowed to update @@ -171,18 +148,17 @@ def test_api_documents_link_configuration_update_authenticated_related_success( ) ).data - with mock_reset_connections(document.id): - response = client.put( - f"/api/v1.0/documents/{document.id!s}/link-configuration/", - new_document_values, - format="json", - ) - assert response.status_code == 200 + response = client.put( + f"/api/v1.0/documents/{document.id!s}/link-configuration/", + new_document_values, + format="json", + ) + assert response.status_code == 200 - document = models.Document.objects.get(pk=document.pk) - document_values = serializers.LinkDocumentSerializer(instance=document).data - for key, value in document_values.items(): - assert value == new_document_values[key] + document = models.Document.objects.get(pk=document.pk) + document_values = serializers.LinkDocumentSerializer(instance=document).data + for key, value in document_values.items(): + assert value == new_document_values[key] def test_api_documents_link_configuration_update_role_restricted_forbidden(): @@ -254,9 +230,7 @@ def test_api_documents_link_configuration_update_link_reach_required(): assert "This field is required" in response.json()["link_reach"][0] -def test_api_documents_link_configuration_update_restricted_without_role_success( - mock_reset_connections, # pylint: disable=redefined-outer-name -): +def test_api_documents_link_configuration_update_restricted_without_role_success(): """ Test that setting link_reach to restricted without specifying link_role succeeds. """ @@ -278,16 +252,15 @@ def test_api_documents_link_configuration_update_restricted_without_role_success "link_reach": models.LinkReachChoices.RESTRICTED, } - with mock_reset_connections(document.id): - response = client.put( - f"/api/v1.0/documents/{document.id!s}/link-configuration/", - new_data, - format="json", - ) + response = client.put( + f"/api/v1.0/documents/{document.id!s}/link-configuration/", + new_data, + format="json", + ) - assert response.status_code == 200 - document.refresh_from_db() - assert document.link_reach == models.LinkReachChoices.RESTRICTED + assert response.status_code == 200 + document.refresh_from_db() + assert document.link_reach == models.LinkReachChoices.RESTRICTED @pytest.mark.parametrize( @@ -297,7 +270,6 @@ def test_api_documents_link_configuration_update_restricted_without_role_success def test_api_documents_link_configuration_update_non_restricted_with_valid_role_success( reach, role, - mock_reset_connections, # pylint: disable=redefined-outer-name ): """ Test that setting non-restricted link_reach with valid link_role succeeds. @@ -320,17 +292,16 @@ def test_api_documents_link_configuration_update_non_restricted_with_valid_role_ "link_role": role, } - with mock_reset_connections(document.id): - response = client.put( - f"/api/v1.0/documents/{document.id!s}/link-configuration/", - new_data, - format="json", - ) + response = client.put( + f"/api/v1.0/documents/{document.id!s}/link-configuration/", + new_data, + format="json", + ) - assert response.status_code == 200 - document.refresh_from_db() - assert document.link_reach == reach - assert document.link_role == role + assert response.status_code == 200 + document.refresh_from_db() + assert document.link_reach == reach + assert document.link_role == role def test_api_documents_link_configuration_update_with_ancestor_constraints(): diff --git a/src/backend/core/tests/documents/test_api_documents_retrieve.py b/src/backend/core/tests/documents/test_api_documents_retrieve.py index feb0b600ce..b3105afe6c 100644 --- a/src/backend/core/tests/documents/test_api_documents_retrieve.py +++ b/src/backend/core/tests/documents/test_api_documents_retrieve.py @@ -33,7 +33,6 @@ def test_api_documents_retrieve_anonymous_public_standalone(): "ai_transform": False, "ai_translate": False, "attachment_upload": document.link_role == "editor", - "can_edit": document.link_role == "editor", "children_create": False, "children_list": True, "collaboration_auth": True, @@ -114,7 +113,6 @@ def test_api_documents_retrieve_anonymous_public_parent(): "ai_transform": False, "ai_translate": False, "attachment_upload": grand_parent.link_role == "editor", - "can_edit": grand_parent.link_role == "editor", "children_create": False, "children_list": True, "collaboration_auth": True, @@ -225,7 +223,6 @@ def test_api_documents_retrieve_authenticated_unrelated_public_or_authenticated( "ai_transform": document.link_role == "editor", "ai_translate": document.link_role == "editor", "attachment_upload": document.link_role == "editor", - "can_edit": document.link_role == "editor", "children_create": document.link_role == "editor", "children_list": True, "collaboration_auth": True, @@ -313,7 +310,6 @@ def test_api_documents_retrieve_authenticated_public_or_authenticated_parent(rea "ai_transform": grand_parent.link_role == "editor", "ai_translate": grand_parent.link_role == "editor", "attachment_upload": grand_parent.link_role == "editor", - "can_edit": grand_parent.link_role == "editor", "children_create": grand_parent.link_role == "editor", "children_list": True, "collaboration_auth": True, @@ -513,7 +509,6 @@ def test_api_documents_retrieve_authenticated_related_parent(): "ai_transform": access.role not in ["reader", "commenter"], "ai_translate": access.role not in ["reader", "commenter"], "attachment_upload": access.role not in ["reader", "commenter"], - "can_edit": access.role not in ["reader", "commenter"], "children_create": access.role not in ["reader", "commenter"], "children_list": True, "collaboration_auth": True, diff --git a/src/backend/core/tests/documents/test_api_documents_trashbin.py b/src/backend/core/tests/documents/test_api_documents_trashbin.py index a6f5668eb9..b32da42e3d 100644 --- a/src/backend/core/tests/documents/test_api_documents_trashbin.py +++ b/src/backend/core/tests/documents/test_api_documents_trashbin.py @@ -83,7 +83,6 @@ def test_api_documents_trashbin_format(): "ai_transform": False, "ai_translate": False, "attachment_upload": False, - "can_edit": False, "children_create": False, "children_list": False, "collaboration_auth": False, @@ -151,7 +150,6 @@ def test_api_documents_trashbin_format(): "ai_transform": False, "ai_translate": False, "attachment_upload": False, - "can_edit": False, "children_create": False, "children_list": False, "collaboration_auth": False, diff --git a/src/backend/core/tests/documents/test_api_documents_update.py b/src/backend/core/tests/documents/test_api_documents_update.py index 358753b496..1b89996b68 100644 --- a/src/backend/core/tests/documents/test_api_documents_update.py +++ b/src/backend/core/tests/documents/test_api_documents_update.py @@ -7,7 +7,6 @@ from unittest.mock import patch from django.contrib.auth.models import AnonymousUser -from django.core.cache import cache import pytest from rest_framework.test import APIClient @@ -47,7 +46,6 @@ def test_api_documents_update_anonymous_forbidden(reach, role, via_parent): new_document_values = serializers.DocumentSerializer( instance=factories.DocumentFactory() ).data - new_document_values["websocket"] = True response = APIClient().put( f"/api/v1.0/documents/{document.id!s}/", new_document_values, @@ -96,7 +94,6 @@ def test_api_documents_update_authenticated_unrelated_forbidden( new_document_values = serializers.DocumentSerializer( instance=factories.DocumentFactory(), ).data - new_document_values["websocket"] = True response = client.put( f"/api/v1.0/documents/{document.id!s}/", new_document_values, @@ -148,7 +145,6 @@ def test_api_documents_update_anonymous_or_authenticated_unrelated( new_document_values = serializers.DocumentSerializer( instance=factories.DocumentFactory(), ).data - new_document_values["websocket"] = True response = client.put( f"/api/v1.0/documents/{document.id!s}/", new_document_values, @@ -216,7 +212,6 @@ def test_api_documents_update_authenticated_reader(via, via_parent, mock_user_te new_document_values = serializers.DocumentSerializer( instance=factories.DocumentFactory() ).data - new_document_values["websocket"] = True response = client.put( f"/api/v1.0/documents/{document.id!s}/", new_document_values, @@ -269,7 +264,6 @@ def test_api_documents_update_authenticated_editor_administrator_or_owner( new_document_values = serializers.DocumentSerializer( instance=factories.DocumentFactory() ).data - new_document_values["websocket"] = True response = client.put( f"/api/v1.0/documents/{document.id!s}/", new_document_values, @@ -303,265 +297,6 @@ def test_api_documents_update_authenticated_editor_administrator_or_owner( assert value == new_document_values[key] -def test_api_documents_update_authenticated_no_websocket(settings): - """ - When a user updates the document, not connected to the websocket and is the first to update, - the document should be updated. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - session_key = client.session.session_key - - document = factories.DocumentFactory(users=[(user, "editor")]) - - new_document_values = serializers.DocumentSerializer( - instance=factories.DocumentFactory() - ).data - new_document_values["websocket"] = False - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - assert cache.get(f"docs:no-websocket:{document.id}") is None - old_path = document.path - - response = client.put( - f"/api/v1.0/documents/{document.id!s}/", - new_document_values, - format="json", - ) - assert response.status_code == 200 - - document.refresh_from_db() - assert document.path == old_path - assert cache.get(f"docs:no-websocket:{document.id}") == session_key - - -def test_api_documents_update_authenticated_no_websocket_user_already_editing(settings): - """ - When a user updates the document, not connected to the websocket and is not the first to update, - the document should not be updated. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - new_document_values = serializers.DocumentSerializer( - instance=factories.DocumentFactory() - ).data - new_document_values["websocket"] = False - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - cache.set(f"docs:no-websocket:{document.id}", "other_session_key") - - response = client.put( - f"/api/v1.0/documents/{document.id!s}/", - new_document_values, - format="json", - ) - assert response.status_code == 403 - assert response.json() == {"detail": "You are not allowed to edit this document."} - - -# TODO(yhub): removed test_api_documents_update_no_websocket_other_user_connected_to_websocket -# here. yhub has no connection-info API: get_document_connection_info is stubbed to report -# nobody connected, so another user connected to the websocket can no longer block the update. -# Re-add the test once yhub exposes a connection-info API. - - -def test_api_documents_update_user_connected_to_websocket(settings): - """ - When a user updates the document, connected to the websocket, the document should be updated. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - session_key = client.session.session_key - - document = factories.DocumentFactory(users=[(user, "editor")]) - - new_document_values = serializers.DocumentSerializer( - instance=factories.DocumentFactory() - ).data - new_document_values["websocket"] = False - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - assert cache.get(f"docs:no-websocket:{document.id}") is None - old_path = document.path - - response = client.put( - f"/api/v1.0/documents/{document.id!s}/", - new_document_values, - format="json", - ) - assert response.status_code == 200 - - document.refresh_from_db() - assert document.path == old_path - # TODO(yhub): the stubbed connection info reports nobody connected, so the - # no-websocket cache lock is taken even though the user is connected. - assert cache.get(f"docs:no-websocket:{document.id}") == session_key - - -def test_api_documents_update_websocket_server_unreachable_fallback_to_no_websocket( - settings, -): - """ - When the websocket server is unreachable, the document should be updated like if the user was - not connected to the websocket. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - session_key = client.session.session_key - - document = factories.DocumentFactory(users=[(user, "editor")]) - - new_document_values = serializers.DocumentSerializer( - instance=factories.DocumentFactory() - ).data - new_document_values["websocket"] = False - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - assert cache.get(f"docs:no-websocket:{document.id}") is None - old_path = document.path - - response = client.put( - f"/api/v1.0/documents/{document.id!s}/", - new_document_values, - format="json", - ) - assert response.status_code == 200 - - document.refresh_from_db() - assert document.path == old_path - assert cache.get(f"docs:no-websocket:{document.id}") == session_key - - -def test_api_documents_update_websocket_server_unreachable_fallback_to_no_websocket_other_users( - settings, -): - """ - When the websocket server is unreachable, the behavior fallback to the no websocket one. - If an other user is already editing, the document should not be updated. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - new_document_values = serializers.DocumentSerializer( - instance=factories.DocumentFactory() - ).data - new_document_values["websocket"] = False - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - cache.set(f"docs:no-websocket:{document.id}", "other_session_key") - - response = client.put( - f"/api/v1.0/documents/{document.id!s}/", - new_document_values, - format="json", - ) - assert response.status_code == 403 - - assert cache.get(f"docs:no-websocket:{document.id}") == "other_session_key" - - -def test_api_documents_update_websocket_server_room_not_found_fallback_to_no_websocket_other_users( - settings, -): - """ - When the WebSocket server does not have the room created, the logic should fallback to - no-WebSocket. If another user is already editing, the update must be denied. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - new_document_values = serializers.DocumentSerializer( - instance=factories.DocumentFactory() - ).data - new_document_values["websocket"] = False - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - cache.set(f"docs:no-websocket:{document.id}", "other_session_key") - - response = client.put( - f"/api/v1.0/documents/{document.id!s}/", - new_document_values, - format="json", - ) - assert response.status_code == 403 - - assert cache.get(f"docs:no-websocket:{document.id}") == "other_session_key" - - -def test_api_documents_update_force_websocket_param_to_true(): - """ - When the websocket parameter is set to true, the document should be updated without any check. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - new_document_values = serializers.DocumentSerializer( - instance=factories.DocumentFactory() - ).data - new_document_values["websocket"] = True - - assert cache.get(f"docs:no-websocket:{document.id}") is None - old_path = document.path - - response = client.put( - f"/api/v1.0/documents/{document.id!s}/", - new_document_values, - format="json", - ) - assert response.status_code == 200 - - document.refresh_from_db() - assert document.path == old_path - assert cache.get(f"docs:no-websocket:{document.id}") is None - - -def test_api_documents_update_feature_flag_disabled(settings): - """ - When the feature flag is disabled, the document should be updated without any check. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - new_document_values = serializers.DocumentSerializer( - instance=factories.DocumentFactory() - ).data - new_document_values["websocket"] = False - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = False - - assert cache.get(f"docs:no-websocket:{document.id}") is None - old_path = document.path - - response = client.put( - f"/api/v1.0/documents/{document.id!s}/", - new_document_values, - format="json", - ) - assert response.status_code == 200 - - document.refresh_from_db() - assert document.path == old_path - assert cache.get(f"docs:no-websocket:{document.id}") is None - - @pytest.mark.parametrize("via", VIA) def test_api_documents_update_administrator_or_owner_of_another(via, mock_user_teams): """ @@ -592,7 +327,6 @@ def test_api_documents_update_administrator_or_owner_of_another(via, mock_user_t new_document_values = serializers.DocumentSerializer( instance=factories.DocumentFactory() ).data - new_document_values["websocket"] = True response = client.put( f"/api/v1.0/documents/{other_document.id!s}/", new_document_values, @@ -728,7 +462,7 @@ def test_api_documents_patch_anonymous_or_authenticated_unrelated( response = client.patch( f"/api/v1.0/documents/{document.id!s}/", - {"title": "new title", "websocket": True}, + {"title": "new title"}, format="json", ) assert response.status_code == 200 @@ -832,7 +566,7 @@ def test_api_documents_patch_authenticated_editor_administrator_or_owner( response = client.patch( f"/api/v1.0/documents/{document.id!s}/", - {"title": "new title", "websocket": True}, + {"title": "new title"}, format="json", ) assert response.status_code == 200 @@ -857,248 +591,6 @@ def test_api_documents_patch_authenticated_editor_administrator_or_owner( assert document_values[key] == old_document_values[key] -def test_api_documents_patch_authenticated_no_websocket(settings): - """ - When a user patches the document, not connected to the websocket and is the first to update, - the document should be updated. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - session_key = client.session.session_key - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - assert cache.get(f"docs:no-websocket:{document.id}") is None - old_path = document.path - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/", - {"title": "new title"}, - format="json", - ) - assert response.status_code == 200 - - # Using document.refresh_from_db does not work because the content is cached. - # Force reloading it by fetching the document from the database. - document = models.Document.objects.get(id=document.id) - assert document.path == old_path - assert document.title == "new title" - assert cache.get(f"docs:no-websocket:{document.id}") == session_key - - -def test_api_documents_patch_authenticated_no_websocket_user_already_editing(settings): - """ - When a user patches the document, not connected to the websocket and is not the first to - update, the document should not be updated. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - cache.set(f"docs:no-websocket:{document.id}", "other_session_key") - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/", - {"title": "new title"}, - format="json", - ) - assert response.status_code == 403 - assert response.json() == {"detail": "You are not allowed to edit this document."} - - -# TODO(yhub): removed test_api_documents_patch_no_websocket_other_user_connected_to_websocket -# here. yhub has no connection-info API: get_document_connection_info is stubbed to report -# nobody connected, so another user connected to the websocket can no longer block the patch. -# Re-add the test once yhub exposes a connection-info API. - - -def test_api_documents_patch_user_connected_to_websocket(settings): - """ - When a user patches the document while connected to the websocket, the document should be - updated. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - session_key = client.session.session_key - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - assert cache.get(f"docs:no-websocket:{document.id}") is None - old_path = document.path - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/", - {"title": "new title"}, - format="json", - ) - assert response.status_code == 200 - - # Using document.refresh_from_db does not wirk because the content is in cache. - # Force reloading it by fetching the document in the database. - document = models.Document.objects.get(id=document.id) - assert document.path == old_path - assert document.title == "new title" - # TODO(yhub): the stubbed connection info reports nobody connected, so the - # no-websocket cache lock is taken even though the user is connected. - assert cache.get(f"docs:no-websocket:{document.id}") == session_key - - -def test_api_documents_patch_websocket_server_unreachable_fallback_to_no_websocket( - settings, -): - """ - When the websocket server is unreachable, the patch should be applied like if the user was - not connected to the websocket. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - session_key = client.session.session_key - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - assert cache.get(f"docs:no-websocket:{document.id}") is None - old_path = document.path - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/", - {"title": "new title"}, - format="json", - ) - assert response.status_code == 200 - - # Using document.refresh_from_db does not work because the content is cached. - # Force reloading it by fetching the document from the database. - document = models.Document.objects.get(id=document.id) - assert document.path == old_path - assert document.title == "new title" - assert cache.get(f"docs:no-websocket:{document.id}") == session_key - - -def test_api_documents_patch_websocket_server_unreachable_fallback_to_no_websocket_other_users( - settings, -): - """ - When the websocket server is unreachable, the behavior falls back to no-websocket. - If another user is already editing, the patch must be denied. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - cache.set(f"docs:no-websocket:{document.id}", "other_session_key") - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/", - {"title": "new title"}, - format="json", - ) - assert response.status_code == 403 - - assert cache.get(f"docs:no-websocket:{document.id}") == "other_session_key" - - -def test_api_documents_patch_websocket_server_room_not_found_fallback_to_no_websocket_other_users( - settings, -): - """ - When the WebSocket server does not have the room created, the logic should fallback to - no-WebSocket. If another user is already editing, the patch must be denied. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - cache.set(f"docs:no-websocket:{document.id}", "other_session_key") - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/", - {"title": "new title"}, - format="json", - ) - assert response.status_code == 403 - - assert cache.get(f"docs:no-websocket:{document.id}") == "other_session_key" - - -def test_api_documents_patch_force_websocket_param_to_true(): - """ - When the websocket parameter is set to true, the patch should be applied without any check. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - assert cache.get(f"docs:no-websocket:{document.id}") is None - old_path = document.path - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/", - {"title": "new title", "websocket": True}, - format="json", - ) - assert response.status_code == 200 - - # Using document.refresh_from_db does not work because the content is cached. - # Force reloading it by fetching the document from the database. - document = models.Document.objects.get(id=document.id) - assert document.path == old_path - assert document.title == "new title" - assert cache.get(f"docs:no-websocket:{document.id}") is None - - -def test_api_documents_patch_feature_flag_disabled(settings): - """ - When the feature flag is disabled, the patch should be applied without any check. - """ - user = factories.UserFactory(with_owned_document=True) - client = APIClient() - client.force_login(user) - - document = factories.DocumentFactory(users=[(user, "editor")]) - - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = False - - assert cache.get(f"docs:no-websocket:{document.id}") is None - old_path = document.path - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/", - {"title": "new title"}, - format="json", - ) - assert response.status_code == 200 - - # Using document.refresh_from_db does not work because the content is cached. - # Force reloading it by fetching the document from the database. - document = models.Document.objects.get(id=document.id) - assert document.path == old_path - assert document.title == "new title" - assert cache.get(f"docs:no-websocket:{document.id}") is None - - @pytest.mark.parametrize("via", VIA) def test_api_documents_patch_administrator_or_owner_of_another(via, mock_user_teams): """ @@ -1141,7 +633,7 @@ def test_api_documents_patch_administrator_or_owner_of_another(via, mock_user_te ) -def test_api_documents_patch_empty_body(settings): +def test_api_documents_patch_empty_body(): """ Test when data is empty the document should not be updated. The `updated_at` property should not change asserting that no update in the database is made. @@ -1150,15 +642,10 @@ def test_api_documents_patch_empty_body(settings): client = APIClient() client.force_login(user) - session_key = client.session.session_key document = factories.DocumentFactory(users=[(user, "owner")], creator=user) document_updated_at = document.updated_at - settings.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = True - - assert cache.get(f"docs:no-websocket:{document.id}") is None - old_document_values = serializers.DocumentSerializer(instance=document).data with patch("core.models.Document.save") as mock_document_save: @@ -1173,6 +660,3 @@ def test_api_documents_patch_empty_body(settings): new_document_values = serializers.DocumentSerializer(instance=document).data assert new_document_values == old_document_values assert document_updated_at == document.updated_at - # TODO(yhub): the stubbed connection info reports nobody connected, so the - # no-websocket cache lock is taken even for an empty body. - assert cache.get(f"docs:no-websocket:{document.id}") == session_key diff --git a/src/backend/core/tests/external_api/test_external_api_documents_link_configuration.py b/src/backend/core/tests/external_api/test_external_api_documents_link_configuration.py index 7c1e6a3086..38f7f732a4 100644 --- a/src/backend/core/tests/external_api/test_external_api_documents_link_configuration.py +++ b/src/backend/core/tests/external_api/test_external_api_documents_link_configuration.py @@ -6,8 +6,6 @@ """ -from unittest.mock import patch - from django.test import override_settings import pytest @@ -61,9 +59,8 @@ def test_external_api_documents_link_configuration_not_allowed( }, }, ) -@patch("core.api.viewsets.reset_service_connections_in_cascade.delay") def test_external_api_documents_link_configuration_can_be_allowed( - mock_reset, user_token, resource_server_backend, user_specific_sub + user_token, resource_server_backend, user_specific_sub ): """ Connected users SHOULD be allowed to update the link configuration of a document @@ -101,6 +98,3 @@ def test_external_api_documents_link_configuration_can_be_allowed( document.refresh_from_db() assert document.link_reach == models.LinkReachChoices.PUBLIC assert document.link_role == models.LinkRoleChoices.EDITOR - - # the collaboration server should be notified through the Celery task - mock_reset.assert_called_once_with(str(document.id)) diff --git a/src/backend/core/tests/test_api_config.py b/src/backend/core/tests/test_api_config.py index 5f7fef4536..4eb8799109 100644 --- a/src/backend/core/tests/test_api_config.py +++ b/src/backend/core/tests/test_api_config.py @@ -25,7 +25,6 @@ AI_FEATURE_LEGACY_ENABLED=False, API_USERS_SEARCH_QUERY_MIN_LENGTH=6, COLLABORATION_WS_URL="http://testcollab/", - COLLABORATION_WS_NOT_CONNECTED_READ_ONLY=True, COLLABORATION_WS_INACTIVITY_TIMEOUT=300, CONVERSION_UPLOAD_ENABLED=False, FRONTEND_CSS_URL="http://testcss/", @@ -56,7 +55,6 @@ def test_api_config(is_authenticated): "AI_FEATURE_LEGACY_ENABLED": False, "API_USERS_SEARCH_QUERY_MIN_LENGTH": 6, "COLLABORATION_WS_URL": "http://testcollab/", - "COLLABORATION_WS_NOT_CONNECTED_READ_ONLY": True, "COLLABORATION_WS_INACTIVITY_TIMEOUT": 300, "CONVERSION_FILE_EXTENSIONS_ALLOWED": [".docx", ".md"], "CONVERSION_FILE_MAX_SIZE": 20971520, diff --git a/src/backend/core/tests/test_models_documents.py b/src/backend/core/tests/test_models_documents.py index bc63b122a8..5c124d15fa 100644 --- a/src/backend/core/tests/test_models_documents.py +++ b/src/backend/core/tests/test_models_documents.py @@ -159,7 +159,6 @@ def test_models_documents_get_abilities_forbidden( "ai_transform": False, "ai_translate": False, "attachment_upload": False, - "can_edit": False, "children_create": False, "children_list": False, "collaboration_auth": False, @@ -228,7 +227,6 @@ def test_models_documents_get_abilities_reader( "ai_transform": False, "ai_translate": False, "attachment_upload": False, - "can_edit": False, "children_create": False, "children_list": True, "collaboration_auth": True, @@ -302,7 +300,6 @@ def test_models_documents_get_abilities_commenter( "ai_transform": False, "ai_translate": False, "attachment_upload": False, - "can_edit": False, "children_create": False, "children_list": True, "collaboration_auth": True, @@ -373,7 +370,6 @@ def test_models_documents_get_abilities_editor( "ai_transform": is_authenticated, "ai_translate": is_authenticated, "attachment_upload": True, - "can_edit": True, "children_create": is_authenticated, "children_list": True, "collaboration_auth": True, @@ -433,7 +429,6 @@ def test_models_documents_get_abilities_owner(django_assert_num_queries): "ai_transform": True, "ai_translate": True, "attachment_upload": True, - "can_edit": True, "children_create": True, "children_list": True, "collaboration_auth": True, @@ -479,7 +474,6 @@ def test_models_documents_get_abilities_owner(django_assert_num_queries): "ai_transform": False, "ai_translate": False, "attachment_upload": False, - "can_edit": False, "children_create": False, "children_list": False, "collaboration_auth": False, @@ -529,7 +523,6 @@ def test_models_documents_get_abilities_administrator(django_assert_num_queries) "ai_transform": True, "ai_translate": True, "attachment_upload": True, - "can_edit": True, "children_create": True, "children_list": True, "collaboration_auth": True, @@ -589,7 +582,6 @@ def test_models_documents_get_abilities_editor_user(django_assert_num_queries): "ai_transform": True, "ai_translate": True, "attachment_upload": True, - "can_edit": True, "children_create": True, "children_list": True, "collaboration_auth": True, @@ -656,7 +648,6 @@ def test_models_documents_get_abilities_reader_user( "ai_transform": access_from_link and ai_access_setting != "restricted", "ai_translate": access_from_link and ai_access_setting != "restricted", "attachment_upload": access_from_link, - "can_edit": access_from_link, "children_create": access_from_link, "children_list": True, "collaboration_auth": True, @@ -726,7 +717,6 @@ def test_models_documents_get_abilities_commenter_user( "ai_transform": access_from_link and ai_access_setting != "restricted", "ai_translate": access_from_link and ai_access_setting != "restricted", "attachment_upload": access_from_link, - "can_edit": access_from_link, "children_create": access_from_link, "children_list": True, "collaboration_auth": True, @@ -791,7 +781,6 @@ def test_models_documents_get_abilities_preset_role(django_assert_num_queries): "ai_transform": False, "ai_translate": False, "attachment_upload": False, - "can_edit": False, "children_create": False, "children_list": True, "collaboration_auth": True, diff --git a/src/backend/core/tests/test_services_collaboration_services.py b/src/backend/core/tests/test_services_collaboration_services.py deleted file mode 100644 index 1eec2f1772..0000000000 --- a/src/backend/core/tests/test_services_collaboration_services.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -This module contains tests for the CollaborationService class in the -core.services.collaboration_services module. -""" - -import responses - -from core.services.collaboration_services import CollaborationService - - -def test_reset_connections_makes_no_http_call(): - """ - TODO(yhub): yhub has no kick API, so reset_connections is a no-op. It must - neither make any HTTP call nor raise, even without any collaboration - settings configured. - """ - with responses.RequestsMock(): - CollaborationService().reset_connections("document-id") - CollaborationService().reset_connections("document-id", user_id="user-id") - - -def test_get_document_connection_info_makes_no_http_call(): - """ - TODO(yhub): yhub has no connection-info API, so get_document_connection_info - always reports nobody connected, without making any HTTP call. - """ - with responses.RequestsMock(): - assert CollaborationService().get_document_connection_info( - "room", "session-key" - ) == (0, False) diff --git a/src/backend/core/tests/test_tasks_access.py b/src/backend/core/tests/test_tasks_access.py deleted file mode 100644 index b5368c453e..0000000000 --- a/src/backend/core/tests/test_tasks_access.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Tests for the `reset_service_connections_in_cascade` Celery task in the -core.tasks.access module. -""" - -from unittest import mock - -from core.tasks.access import reset_service_connections_in_cascade - - -@mock.patch("core.tasks.access.CollaborationService") -def test_reset_service_connections_delegates_to_service(mock_service): - """ - The task should delegate the whole reset to the CollaborationService, - forwarding both the document id and the user id. - """ - reset_service_connections_in_cascade("document-id", "user-id") - - mock_service.return_value.reset_connections.assert_called_once_with( - "document-id", "user-id" - ) - - -@mock.patch("core.tasks.access.CollaborationService") -def test_reset_service_connections_defaults_user_id_to_none(mock_service): - """When no user id is provided, the task should forward None to the service.""" - reset_service_connections_in_cascade("document-id") - - mock_service.return_value.reset_connections.assert_called_once_with( - "document-id", None - ) diff --git a/src/backend/impress/settings.py b/src/backend/impress/settings.py index 173bf5c2da..4c4823da31 100755 --- a/src/backend/impress/settings.py +++ b/src/backend/impress/settings.py @@ -512,28 +512,9 @@ class Base(Configuration): SENTRY_DSN = values.Value(None, environ_name="SENTRY_DSN", environ_prefix=None) # Collaboration - # TODO(yhub): unused since the yhub migration — yhub has no management API - # (reset-connections / get-connections). Kept until a yhub kick and - # connection-info API exist and CollaborationService is reinstated. - COLLABORATION_API_URL = values.Value( - None, environ_name="COLLABORATION_API_URL", environ_prefix=None - ) - # TODO(yhub): unused since the yhub migration, see COLLABORATION_API_URL. - COLLABORATION_SERVER_SECRET = SecretFileValue( - None, environ_name="COLLABORATION_SERVER_SECRET", environ_prefix=None - ) COLLABORATION_WS_URL = values.Value( None, environ_name="COLLABORATION_WS_URL", environ_prefix=None ) - COLLABORATION_WS_NOT_CONNECTED_READ_ONLY = values.BooleanValue( - default=values.BooleanValue( # COLLABORATION_WS_NOT_CONNECTED_READY_ONLY compat - default=False, - environ_name="COLLABORATION_WS_NOT_CONNECTED_READY_ONLY", - environ_prefix=None, - ), - environ_name="COLLABORATION_WS_NOT_CONNECTED_READ_ONLY", - environ_prefix=None, - ) COLLABORATION_WS_INACTIVITY_TIMEOUT = values.IntegerValue( None, environ_name="COLLABORATION_WS_INACTIVITY_TIMEOUT", @@ -983,12 +964,6 @@ class Base(Configuration): environ_prefix=None, ) - NO_WEBSOCKET_CACHE_TIMEOUT = values.Value( - default=120, - environ_name="NO_WEBSOCKET_CACHE_TIMEOUT", - environ_prefix=None, - ) - # Logging # We want to make it easy to log to console but by default we log production # to Sentry and don't want to log to console. diff --git a/src/helm/env.d/dev/values.impress.yaml.gotmpl b/src/helm/env.d/dev/values.impress.yaml.gotmpl index 981f4c84a5..85c319785f 100644 --- a/src/helm/env.d/dev/values.impress.yaml.gotmpl +++ b/src/helm/env.d/dev/values.impress.yaml.gotmpl @@ -15,9 +15,6 @@ image: backend: replicas: 1 envVars: - COLLABORATION_SERVER_SECRET: my-secret - COLLABORATION_API_URL: https://docs.127.0.0.1.nip.io/collaboration/api/ - COLLABORATION_WS_NOT_CONNECTED_READ_ONLY: False CONVERSION_UPLOAD_ENABLED: True DJANGO_CSRF_TRUSTED_ORIGINS: https://docs.127.0.0.1.nip.io DJANGO_CONFIGURATION: Feature diff --git a/src/helm/env.d/feature/values.impress.yaml.gotmpl b/src/helm/env.d/feature/values.impress.yaml.gotmpl index e61d741541..3da4264866 100644 --- a/src/helm/env.d/feature/values.impress.yaml.gotmpl +++ b/src/helm/env.d/feature/values.impress.yaml.gotmpl @@ -16,10 +16,7 @@ image: backend: replicas: 1 envVars: - COLLABORATION_SERVER_SECRET: my-secret CONVERSION_UPLOAD_ENABLED: True - COLLABORATION_API_URL: https://{{ .Values.feature }}-docs.{{ .Values.domain }}/collaboration/api/ - COLLABORATION_WS_NOT_CONNECTED_READ_ONLY: True DJANGO_CSRF_TRUSTED_ORIGINS: https://{{ .Values.feature }}-docs.{{ .Values.domain }} DJANGO_CONFIGURATION: Feature DJANGO_ALLOWED_HOSTS: {{ .Values.feature }}-docs.{{ .Values.domain }} From 49089c3b3be0c447d105d021469b50c1a34f1a34 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Wed, 5 Aug 2026 11:11:38 +0200 Subject: [PATCH 09/59] =?UTF-8?q?=E2=9C=A8(collaboration)=20add=20admin=20?= =?UTF-8?q?reset-connections=20endpoint=20on=20yhub=200.4.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add POST /collaboration/reset-connections/v1/{org}/{docid} (optional X-User-Id header) to yhub-server. It distributes yhub recheckAuth: every server re-runs the access check per matching connection and closes only those whose access actually changed (close code 4401), so unaffected clients see no reconnect churn. The endpoint authenticates with the RS256 admin JWT issued by JWTService, verified against the backend JWKS (new jose dependency); the admin token acts as the "system" user and is the only principal granted the reset-connections access purpose. The backend does not trigger it on permission changes yet - that wiring comes separately, now that CollaborationService is gone. yhub is upgraded to 0.4.0 and serves every route under the /collaboration/ prefix (server.apiPrefix): the websocket moves to /collaboration/ws/v1/docs, and the built-in document apis are meant to be publicly exposed alongside it, with reset-connections as the one backend-internal exception. Also harden websocket auth: fail closed when the backend errors (only a genuine 401/403 falls back to the anonymous identity, so a signed-in editor can never hide from a targeted recheck under an anon userid) and tolerate small clock skew when verifying the cached admin token. Co-Authored-By: Claude Fable 5 Signed-off-by: Kevin Jahns --- CHANGELOG.md | 11 ++ env.d/development/common | 3 +- .../core/config/hooks/useCollaborationUrl.tsx | 2 +- src/yhub-server/README.md | 16 +++ src/yhub-server/package-lock.json | 26 +++-- src/yhub-server/package.json | 3 +- src/yhub-server/server.js | 105 +++++++++++++++++- 7 files changed, 148 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 185474fedd..9a9300968b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,17 @@ and this project adheres to ### Added +- ✨(collaboration) add an admin reset-connections endpoint on yhub: + `POST /collaboration/reset-connections/v1/docs/{id}` re-checks the + authorization of the document's connected clients and disconnects (close + code 4401) only those whose access changed. Authenticated with the admin + JWT verified against the backend JWKS; not yet triggered by the backend on + permission changes (follow-up) +- ⬆️(collaboration) upgrade yhub to 0.4.0 and serve all its routes under the + `/collaboration/` prefix (`server.apiPrefix`): the websocket moves to + `/collaboration/ws/v1/docs`. All `/collaboration/` routes are meant to be + publicly exposed except `reset-connections`, which stays backend-internal + (admin JWT only) - ✨(backend) add a service generating cached RS256 JWT tokens - ✨(backend) publish the JWT public key on a JWKS endpoint - 🔧(dev) generate the JWT signing key when bootstrapping the dev stack diff --git a/env.d/development/common b/env.d/development/common index 498bcf367e..60072b5fdc 100644 --- a/env.d/development/common +++ b/env.d/development/common @@ -77,11 +77,10 @@ OIDC_RS_ALLOWED_AUDIENCES="" USER_RECONCILIATION_FORM_URL=http://localhost:3000 # Collaboration -# TODO(yhub): no management API yet COLLABORATION_BACKEND_BASE_URL=http://app-dev:8000 COLLABORATION_SERVER_ORIGIN=http://localhost:3000 COLLABORATION_SERVER_SECRET=my-secret -COLLABORATION_WS_URL=ws://localhost:3002/ws/docs +COLLABORATION_WS_URL=ws://localhost:3002/collaboration/ws/v1/docs COLLABORATION_WS_INACTIVITY_TIMEOUT=15 # Seconds DJANGO_SERVER_TO_SERVER_API_TOKENS=server-api-token diff --git a/src/frontend/apps/impress/src/core/config/hooks/useCollaborationUrl.tsx b/src/frontend/apps/impress/src/core/config/hooks/useCollaborationUrl.tsx index feee4ab031..d87974c1c3 100644 --- a/src/frontend/apps/impress/src/core/config/hooks/useCollaborationUrl.tsx +++ b/src/frontend/apps/impress/src/core/config/hooks/useCollaborationUrl.tsx @@ -12,7 +12,7 @@ export const useCollaborationUrl = (room?: string) => { conf?.COLLABORATION_WS_URL || (typeof window !== 'undefined' ? // TODO(yhub): no prod ingress route yet - `wss://${window.location.host}/ws/docs` + `wss://${window.location.host}/collaboration/ws/v1/docs` : '') ); }; diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index 229c5b9826..248a52d9ae 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -11,9 +11,25 @@ It is not a fork of yhub — it is a thin wrapper (`server.js`) that: - plugs in an auth plugin that resolves users and per-document access rights by calling the Docs Django backend (`/api/v1.0/users/me/` and `/api/v1.0/documents/{id}/`), +- serves every route under the `/collaboration/` prefix + (`server.apiPrefix`), including the websocket sync route + `/collaboration/ws/v1/{org}/{docid}`, +- exposes `POST /collaboration/reset-connections/v1/{org}/{docid}` (optional + `X-User-Id` header), for the Django backend to re-check the authorization + of a document's connected clients when permissions change (backend wiring + pending) — authenticated with an RS256 admin JWT issued by Django and + verified against its JWKS (`/api/v1.0/jwks`); the `reset-connections` + purpose is granted only to that admin token, never to regular users, - mirrors the environment conventions used elsewhere in this repository (`*_FILE` secret indirection, `COLLABORATION_SERVER_ORIGIN` allowlist, …). +Public exposure: route the whole `/collaboration/` prefix to this server — +the websocket and the built-in document APIs (`ydoc`, `rollback`, `prune`, +`changeset`, `activity`) are all guarded by the same cookie-based document +authorization and are meant to be reachable by browsers. The one exception +is `/collaboration/reset-connections/`, which is backend-internal and should +not be routed through the public ingress. + The `Dockerfile` builds the container image used by the `yhub` service in `compose.yml`. diff --git a/src/yhub-server/package-lock.json b/src/yhub-server/package-lock.json index 050099a8d6..9808e2ab1d 100644 --- a/src/yhub-server/package-lock.json +++ b/src/yhub-server/package-lock.json @@ -6,7 +6,8 @@ "": { "name": "yhub-server", "dependencies": { - "@y/hub": "0.3.1" + "@y/hub": "0.4.0", + "jose": "6.2.8" }, "engines": { "node": ">=22" @@ -109,15 +110,15 @@ "license": "ISC" }, "node_modules/@y/hub": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/@y/hub/-/hub-0.3.1.tgz", - "integrity": "sha512-gauvqZ2XwOi7c/Fu13xbOCV3uR+OCHH5QUMyIPXJfDvdTwILEo0hatQWaLK1NhfOZCcaxTPbpgeu2GdLGs7xhg==", + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@y/hub/-/hub-0.4.0.tgz", + "integrity": "sha512-LejJTSrBt86DI88pMO8rck4tQcHHR1SPLrZrOIfb5Ws/zNd601x5SXX0MOuTwKqoNOweT4esUX0Eh/2ER4sSsA==", "license": "AGPL-3.0 OR PROPRIETARY", "dependencies": { "@y-crdt/yn": "^0.1.4", "@y/protocols": "^1.0.6-rc.1", "@y/y": "^14.0.0-rc.24", - "lib0": "^1.0.0-rc.22", + "lib0": "^1.0.0-rc.23", "minio": "^8.0.6", "pino": "^10.3.1", "postgres": "^3.4.3", @@ -323,10 +324,19 @@ ], "license": "MIT" }, + "node_modules/jose": { + "version": "6.2.8", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.8.tgz", + "integrity": "sha512-Bsdjwm3Qsd/P0jR+BHDe3LytDfY7WBq2HmCCLIwuVRHMuEC9ae7/R474GIUdF1NgCyZjzVo/A9DOiOBtXq8ZoQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, "node_modules/lib0": { - "version": "1.0.0-rc.22", - "resolved": "https://registry.npmjs.org/lib0/-/lib0-1.0.0-rc.22.tgz", - "integrity": "sha512-KNefJloRQIsWncTF2tIcRqQXSQ7bDRYHwVSUhf1lY2P65Rej4WWFnen6L8L+odJQIo1ZNJGVVjK2WzqB9a+B/g==", + "version": "1.0.0-rc.23", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-1.0.0-rc.23.tgz", + "integrity": "sha512-JPomcbwgKoTIDoXP61DFZV+Yvkw8bCyQhr9QYxps0fmHzsliEw+mhbUP1/nyVmk6ugzapJw1hUpFEMFRA4sRIg==", "license": "MIT", "bin": { "0ecdsa-generate-keypair": "src/bin/0ecdsa-generate-keypair.js", diff --git a/src/yhub-server/package.json b/src/yhub-server/package.json index 0d5e5d0d90..f4ec5cb513 100644 --- a/src/yhub-server/package.json +++ b/src/yhub-server/package.json @@ -6,7 +6,8 @@ "start": "node server.js" }, "dependencies": { - "@y/hub": "0.3.1" + "@y/hub": "0.4.0", + "jose": "6.2.8" }, "engines": { "node": ">=22" diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index 8577706a55..5eade43e3a 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -1,7 +1,8 @@ import { createHash } from 'node:crypto'; import { readFileSync } from 'node:fs'; -import { createAuthPlugin, createYHub } from '@y/hub'; +import { createApiEndpoint, createAuthPlugin, createYHub } from '@y/hub'; +import { createRemoteJWKSet, jwtVerify } from 'jose'; // mirror y-provider's env.ts secret-file support const secret = (name, dflt) => @@ -23,6 +24,13 @@ const ORG = process.env.YHUB_ORG || 'docs'; const UUID4 = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +// Public keys verifying the RS256 admin tokens Django issues (JWTService). +// Lazily fetched on first use; jose caches the keys and refetches on unknown +// "kid", so Django can rotate the signing key without a yhub restart. +const JWKS = createRemoteJWKSet( + new URL(`${COLLABORATION_BACKEND_BASE_URL}/api/v1.0/jwks`), +); + const backendFetch = async (path, { cookie, origin }) => { const res = await fetch(`${COLLABORATION_BACKEND_BASE_URL}${path}`, { headers: { @@ -32,7 +40,9 @@ const backendFetch = async (path, { cookie, origin }) => { }, }); if (!res.ok) { - throw new Error(`Failed to fetch ${path}: ${res.status}`); + const err = new Error(`Failed to fetch ${path}: ${res.status}`); + err.status = res.status; + throw err; } return res.json(); }; @@ -40,9 +50,37 @@ const backendFetch = async (path, { cookie, origin }) => { const auth = createAuthPlugin({ // uws req is only valid synchronously — read headers AND query before first await. async readAuthInfo(req) { + const authorization = req.getHeader('authorization'); const cookie = req.getHeader('cookie'); const origin = req.getHeader('origin'); const gcOff = req.getQuery('gc') === 'false'; + if (authorization !== '') { + // backend-to-server call: RS256 JWT signed by Django, verified against + // its JWKS. A browser cannot attach an Authorization header to a ws + // upgrade or a credentialed cross-origin fetch, so this never shadows a + // real user session. present-but-invalid fails here (401) instead of + // falling through to the cookie flow, which would mask a + // misconfiguration as an origin error. + const token = authorization.startsWith('Bearer ') + ? authorization.slice('Bearer '.length) + : authorization; + try { + // clockTolerance absorbs Django's cache-at-exp race (the admin token + // is cached for exactly its lifetime, so it can arrive here moments + // after exp) plus small clock skew — without it a kick would be + // silently dropped as a 401. + const { payload } = await jwtVerify(token, JWKS, { + algorithms: ['RS256'], + clockTolerance: 5, + }); + // admin tokens act as the "system" user (no per-user admin identities yet) + return payload.admin === true + ? { userid: 'system', admin: true } + : null; + } catch { + return null; // bad signature / expired / JWKS unreachable — fail closed + } + } if (gcOff) return null; // full-history connections: not for Docs users if (!origin || !allowedOrigins.includes(origin)) return null; // was 4001 'Origin not allowed' if (!cookie) return null; // was 4001 'No cookies' @@ -52,7 +90,13 @@ const auth = createAuthPlugin({ origin, }); return { userid: String(user.id), cookie, origin }; // MUST be string (yhub server.js:667) - } catch { + } catch (err) { + // Only a genuine "not signed in" falls back to the anonymous identity. + // On backend failure (5xx/network) fail closed: a signed-in editor + // authorized under an anon userid would be invisible to the targeted + // reset-connections recheck (users: []) for the connection's + // whole lifetime. + if (err?.status !== 401 && err?.status !== 403) return null; // anonymous (public docs): stable per-session id — random ids would mint a new // permanent attribution identity per reconnect const anon = createHash('sha256') @@ -62,8 +106,17 @@ const auth = createAuthPlugin({ return { userid: `anon:${anon}`, cookie, origin }; } }, - async getAccessType(authInfo, { org, docid, branch }) { - if (org !== ORG || branch !== 'main' || !UUID4.test(docid)) { + async getAccessType(authInfo, { org, docid, branch }, purpose) { + if (authInfo.admin === true) return 'rw'; // Django's admin token: full access + // Regular users only get access for the default purpose — custom-endpoint + // purposes (reset-connections) are backend-internal. Loose != on purpose: + // ws upgrades and rechecks pass undefined, built-in rest endpoints null. + if ( + org !== ORG || + branch !== 'main' || + !UUID4.test(docid) || + purpose != null + ) { return null; } try { @@ -81,6 +134,43 @@ const auth = createAuthPlugin({ }, }); +// Mimic the old y-provider REST responses (JSON, not yhub's lib0-any +// encoding) so the Django caller keeps its historical contract. +const jsonResponse = (status, body) => + new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); + +const api = [ + // POST /collaboration/reset-connections/v1/{org}/{docid} — replaces + // y-provider's /collaboration/api/reset-connections/?room=. Doc-scoped, so + // the room comes from the path; access is gated to the admin token via the + // 'reset-connections' purpose in getAccessType. uws routes are exact: a + // trailing slash 404s. + createApiEndpoint('reset-connections', { + accessPurpose: 'reset-connections', + post: { + handler: async (req) => { + const userId = req.headers['x-user-id'] || null; + if (req.org !== ORG) { + return jsonResponse(400, { error: 'Unknown org' }); + } + if (!UUID4.test(req.docid)) { + return jsonResponse(400, { error: 'Room name is invalid' }); + } + // in-place recheck: every yhub server re-runs getAccessType per + // matching connection and closes 4401 only when the access changed — + // no reconnect churn for unaffected clients + await req.yhub.recheckAuth(req.room, { + users: userId ? [userId] : null, + }); + return jsonResponse(200, { message: 'Connections reset' }); + }, + }, + }), +]; + await createYHub({ redis: { url: REDIS, @@ -90,7 +180,10 @@ await createYHub({ }, postgres: POSTGRES, persistence: [], // blobs live in yhub's postgres - server: { port: PORT, auth }, + // apiPrefix mounts every route — built-ins, reset-connections, and the + // websocket (/collaboration/ws/v1/{org}/{docid}) — under /collaboration/, + // matching the URL scheme Docs already routes to the collaboration server. + server: { port: PORT, auth, api, apiPrefix: 'collaboration' }, worker: { taskConcurrency: 5 }, // TODO(yhub): worker.events.docUpdate could push snapshots to Django and replace the // client useSaveDoc PATCH flow — blocked upstream: the payload is a DocTable without From 2a46427c86aba46bef2aa9a6157f72949625185a Mon Sep 17 00:00:00 2001 From: Anthony LC Date: Tue, 4 Aug 2026 18:16:24 +0200 Subject: [PATCH 10/59] =?UTF-8?q?=F0=9F=9B=82(django)=20use=20jwt=20token?= =?UTF-8?q?=20for=20converter=20services?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Y_PROVIDER_API_KEY shared secret is replaced by a signed admin JWT when Django calls the y-provider conversion endpoint. --- .../core/services/converter_services.py | 4 +-- src/backend/core/tests/test_api_jwks.py | 21 ++---------- .../tests/test_services_converter_services.py | 32 ++++++++++++++----- .../core/tests/test_services_jwt_services.py | 23 +------------ src/backend/core/tests/utils/jwt.py | 23 +++++++++++++ 5 files changed, 53 insertions(+), 50 deletions(-) create mode 100644 src/backend/core/tests/utils/jwt.py diff --git a/src/backend/core/services/converter_services.py b/src/backend/core/services/converter_services.py index 3cd4498da0..ee542b1c7f 100644 --- a/src/backend/core/services/converter_services.py +++ b/src/backend/core/services/converter_services.py @@ -9,6 +9,7 @@ import requests from core.services import mime_types +from core.services.jwt_services import JWTService logger = logging.getLogger(__name__) @@ -109,8 +110,7 @@ class YdocConverter: @property def auth_header(self): """Build microservice authentication header.""" - # Note: Yprovider microservice accepts only raw token, which is not recommended - return f"Bearer {settings.Y_PROVIDER_API_KEY}" + return f"Bearer {JWTService().get_admin_token()}" def _request(self, url, data, content_type, accept): """Make a request to the Y-Provider API.""" diff --git a/src/backend/core/tests/test_api_jwks.py b/src/backend/core/tests/test_api_jwks.py index 3c76bd1fb5..348d55a6c1 100644 --- a/src/backend/core/tests/test_api_jwks.py +++ b/src/backend/core/tests/test_api_jwks.py @@ -6,11 +6,10 @@ import jwt import pytest -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import rsa from rest_framework.test import APIClient from core.services.jwt_services import JWTService +from core.tests.utils.jwt import generate_key_pair from core.tests.utils.urls import reload_urls pytestmark = pytest.mark.django_db @@ -18,23 +17,9 @@ # Private members of a RSA JWK, none of them may ever leak in the JWKS PRIVATE_JWK_MEMBERS = {"d", "p", "q", "dp", "dq", "qi", "oth"} - -def generate_private_key(): - """Generate a PEM encoded RSA private key.""" - return ( - rsa.generate_private_key(public_exponent=65537, key_size=2048) - .private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.PKCS8, - encryption_algorithm=serialization.NoEncryption(), - ) - .decode("utf-8") - ) - - # Generating RSA keys is expensive, do it once for the whole module -PRIVATE_KEY = generate_private_key() -OTHER_PRIVATE_KEY = generate_private_key() +PRIVATE_KEY, _ = generate_key_pair() +OTHER_PRIVATE_KEY, _ = generate_key_pair() @pytest.fixture(name="jwt_settings") diff --git a/src/backend/core/tests/test_services_converter_services.py b/src/backend/core/tests/test_services_converter_services.py index 760504cec1..345e4fb249 100644 --- a/src/backend/core/tests/test_services_converter_services.py +++ b/src/backend/core/tests/test_services_converter_services.py @@ -3,6 +3,7 @@ from base64 import b64decode from unittest.mock import MagicMock, patch +import jwt import pytest import requests @@ -12,13 +13,28 @@ ValidationError, YdocConverter, ) +from core.tests.utils.jwt import generate_key_pair +# Generating an RSA key is expensive, do it once for the whole module +PRIVATE_KEY, PUBLIC_KEY = generate_key_pair() -def test_auth_header(settings): - """Test authentication header generation.""" - settings.Y_PROVIDER_API_KEY = "test-key" + +@pytest.fixture(autouse=True) +def jwt_settings(settings): + """Setup valid settings for the JWT service used to sign the auth header.""" + settings.JWT_PRIVATE_KEY = PRIVATE_KEY + settings.JWT_TOKEN_LIFETIME = 3600 + + +def test_auth_header(): + """The auth header carries an admin JWT signed with the configured key.""" converter = YdocConverter() - assert converter.auth_header == "Bearer test-key" + + scheme, token = converter.auth_header.split(" ") + + assert scheme == "Bearer" + payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + assert payload["admin"] is True def test_convert_empty_text(): @@ -63,12 +79,12 @@ def test_convert_full_integration(mock_post, settings): """Test full integration with all settings.""" settings.Y_PROVIDER_API_BASE_URL = "http://test.com/" - settings.Y_PROVIDER_API_KEY = "test-key" settings.CONVERSION_API_ENDPOINT = "conversion-endpoint" settings.CONVERSION_API_TIMEOUT = 5 settings.CONVERSION_API_CONTENT_FIELD = "content" converter = YdocConverter() + auth_header = converter.auth_header expected_content = b"converted content" mock_response = MagicMock() @@ -83,7 +99,7 @@ def test_convert_full_integration(mock_post, settings): "http://test.com/conversion-endpoint/", data="test markdown", headers={ - "Authorization": "Bearer test-key", + "Authorization": auth_header, "Content-Type": mime_types.MARKDOWN, "Accept": mime_types.YJS, }, @@ -96,12 +112,12 @@ def test_convert_full_integration(mock_post, settings): def test_convert_full_integration_with_specific_headers(mock_post, settings): """Test successful conversion with specific content type and accept headers.""" settings.Y_PROVIDER_API_BASE_URL = "http://test.com/" - settings.Y_PROVIDER_API_KEY = "test-key" settings.CONVERSION_API_ENDPOINT = "conversion-endpoint" settings.CONVERSION_API_TIMEOUT = 5 settings.CONVERSION_API_SECURE = False converter = YdocConverter() + auth_header = converter.auth_header expected_response = "# Test Document\n\nThis is test content." mock_response = MagicMock() @@ -116,7 +132,7 @@ def test_convert_full_integration_with_specific_headers(mock_post, settings): "http://test.com/conversion-endpoint/", data=b"test_content", headers={ - "Authorization": "Bearer test-key", + "Authorization": auth_header, "Content-Type": mime_types.YJS, "Accept": mime_types.MARKDOWN, }, diff --git a/src/backend/core/tests/test_services_jwt_services.py b/src/backend/core/tests/test_services_jwt_services.py index be8b584183..f2bc8f2b2a 100644 --- a/src/backend/core/tests/test_services_jwt_services.py +++ b/src/backend/core/tests/test_services_jwt_services.py @@ -10,8 +10,6 @@ import jwt import pytest -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import rsa from freezegun import freeze_time from core.services.jwt_services import ( @@ -19,26 +17,7 @@ JWTService, TokenGenerationError, ) - - -def generate_key_pair(): - """Generate a PEM encoded RSA key pair to sign and verify test tokens.""" - private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) - private_pem = private_key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.PKCS8, - encryption_algorithm=serialization.NoEncryption(), - ).decode("utf-8") - public_pem = ( - private_key.public_key() - .public_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PublicFormat.SubjectPublicKeyInfo, - ) - .decode("utf-8") - ) - return private_pem, public_pem - +from core.tests.utils.jwt import generate_key_pair # Generating RSA keys is expensive, do it once for the whole module PRIVATE_KEY, PUBLIC_KEY = generate_key_pair() diff --git a/src/backend/core/tests/utils/jwt.py b/src/backend/core/tests/utils/jwt.py new file mode 100644 index 0000000000..f97bb18ea2 --- /dev/null +++ b/src/backend/core/tests/utils/jwt.py @@ -0,0 +1,23 @@ +"""Utils for testing JWT-signed tokens.""" + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + + +def generate_key_pair(): + """Generate a PEM encoded RSA key pair to sign and verify test tokens.""" + private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + private_pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ).decode("utf-8") + public_pem = ( + private_key.public_key() + .public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ) + .decode("utf-8") + ) + return private_pem, public_pem From 23e03cc69d4ce13c1baa0e2ace92b86b828fdba6 Mon Sep 17 00:00:00 2001 From: Anthony LC Date: Tue, 4 Aug 2026 18:24:00 +0200 Subject: [PATCH 11/59] =?UTF-8?q?=F0=9F=9B=82(y-provider)=20verify=20jwt?= =?UTF-8?q?=20token=20instead=20of=20the=20shared=20api=20key?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /api/convert route no longer accepts the Y_PROVIDER_API_KEY shared secret. It now verifies the admin JWT signed by Django against the JWKS published on its /api/v1.0/jwks endpoint. --- .../y-provider/__tests__/convert.test.ts | 17 ++- .../y-provider/__tests__/middlewares.test.ts | 114 ++++++++++++++++++ .../y-provider/__tests__/server.test.ts | 24 ++-- .../__tests__/testUtils/adminJwt.ts | 75 ++++++++++++ src/frontend/servers/y-provider/package.json | 1 + src/frontend/servers/y-provider/src/env.ts | 11 +- .../servers/y-provider/src/middlewares.ts | 32 +++-- src/frontend/yarn.lock | 32 ++--- 8 files changed, 256 insertions(+), 50 deletions(-) create mode 100644 src/frontend/servers/y-provider/__tests__/middlewares.test.ts create mode 100644 src/frontend/servers/y-provider/__tests__/testUtils/adminJwt.ts diff --git a/src/frontend/servers/y-provider/__tests__/convert.test.ts b/src/frontend/servers/y-provider/__tests__/convert.test.ts index 30b0a43c17..bbbec7bf6f 100644 --- a/src/frontend/servers/y-provider/__tests__/convert.test.ts +++ b/src/frontend/servers/y-provider/__tests__/convert.test.ts @@ -6,7 +6,7 @@ import { import { ServerBlockNoteEditor } from '@blocknote/server-util'; import { Fragment, Node as PMNode } from 'prosemirror-model'; import request from 'supertest'; -import { afterEach, describe, expect, test, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; import { prosemirrorToYXmlFragment } from 'y-prosemirror'; import * as Y from 'yjs'; @@ -14,17 +14,17 @@ vi.mock('../src/env', async (importOriginal) => { return { ...(await importOriginal()), COLLABORATION_SERVER_ORIGIN: 'http://localhost:3000', - Y_PROVIDER_API_KEY: 'yprovider-api-key', }; }); import { docsBlockNoteSchema } from '@/blockSpecs'; import { initApp } from '@/servers'; -import { - Y_PROVIDER_API_KEY as apiKey, - COLLABORATION_SERVER_ORIGIN as origin, -} from '../src/env'; +import { JWKS_URL, COLLABORATION_SERVER_ORIGIN as origin } from '../src/env'; + +import { mockJwksEndpoint, signAdminToken } from './testUtils/adminJwt'; + +const apiKey = await signAdminToken(); const expectedMarkdown = '# Example document\n\nLorem ipsum dolor sit amet.'; const expectedHTML = @@ -141,8 +141,13 @@ const buildYjsUpdateWithComment = (): Buffer => { console.error = vi.fn(); describe('Conversion Testing', () => { + beforeEach(() => { + mockJwksEndpoint(JWKS_URL); + }); + afterEach(() => { vi.clearAllMocks(); + vi.unstubAllGlobals(); }); test('POST /api/convert with incorrect API key responds with 401', async () => { diff --git a/src/frontend/servers/y-provider/__tests__/middlewares.test.ts b/src/frontend/servers/y-provider/__tests__/middlewares.test.ts new file mode 100644 index 0000000000..cc782f9fab --- /dev/null +++ b/src/frontend/servers/y-provider/__tests__/middlewares.test.ts @@ -0,0 +1,114 @@ +import express from 'express'; +import request from 'supertest'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const { JWKS_URL } = vi.hoisted(() => ({ + JWKS_URL: 'http://app-dev:8000/api/v1.0/jwks', +})); + +vi.mock('../src/env', async (importOriginal) => { + return { + ...(await importOriginal()), + JWKS_URL, + }; +}); + +import { httpSecurity } from '@/middlewares'; + +import { + mockJwksEndpoint, + signAdminToken, + signAdminTokenWithWrongKey, + signExpiredAdminToken, + signToken, +} from './testUtils/adminJwt'; + +const buildApp = () => { + const app = express(); + app.get('/protected', httpSecurity, (req, res) => { + res.status(200).json({ ok: true }); + }); + return app; +}; + +describe('httpSecurity', () => { + beforeEach(() => { + mockJwksEndpoint(JWKS_URL); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('rejects requests without an authorization header', async () => { + const response = await request(buildApp()).get('/protected'); + + expect(response.status).toBe(401); + expect(response.body).toStrictEqual({ + error: 'Unauthorized: No credentials given', + }); + expect(fetch).not.toHaveBeenCalled(); + }); + + it('accepts a valid admin JWT signed by the Django backend', async () => { + const token = await signAdminToken(); + + const response = await request(buildApp()) + .get('/protected') + .set('authorization', `Bearer ${token}`); + + expect(response.status).toBe(200); + // Verified against the real JWKS document served over the (mocked) network. + expect(fetch).toHaveBeenCalledWith(JWKS_URL, expect.anything()); + }); + + it('rejects a token signed with a key that is not in the JWKS', async () => { + const token = await signAdminTokenWithWrongKey(); + + const response = await request(buildApp()) + .get('/protected') + .set('authorization', `Bearer ${token}`); + + expect(response.status).toBe(401); + expect(response.body).toStrictEqual({ + error: 'Unauthorized: Invalid API Key', + }); + }); + + it('rejects an expired admin JWT', async () => { + const token = await signExpiredAdminToken(); + + const response = await request(buildApp()) + .get('/protected') + .set('authorization', `Bearer ${token}`); + + expect(response.status).toBe(401); + expect(response.body).toStrictEqual({ + error: 'Unauthorized: Invalid API Key', + }); + }); + + it('rejects a validly signed JWT missing the admin claim', async () => { + const token = await signToken({ sub: 'someone' }); + + const response = await request(buildApp()) + .get('/protected') + .set('authorization', `Bearer ${token}`); + + expect(response.status).toBe(401); + expect(response.body).toStrictEqual({ + error: 'Unauthorized: Invalid API Key', + }); + }); + + it('rejects a bearer token that is not a valid JWT', async () => { + const response = await request(buildApp()) + .get('/protected') + .set('authorization', 'Bearer wrong-token'); + + expect(response.status).toBe(401); + expect(response.body).toStrictEqual({ + error: 'Unauthorized: Invalid API Key', + }); + }); +}); diff --git a/src/frontend/servers/y-provider/__tests__/server.test.ts b/src/frontend/servers/y-provider/__tests__/server.test.ts index 5454f80197..6d76bda620 100644 --- a/src/frontend/servers/y-provider/__tests__/server.test.ts +++ b/src/frontend/servers/y-provider/__tests__/server.test.ts @@ -1,5 +1,5 @@ import request from 'supertest'; -import { describe, expect, it, test, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, test, vi } from 'vitest'; import { routes } from '@/routes'; import { initApp } from '@/servers'; @@ -8,19 +8,25 @@ vi.mock('../src/env', async (importOriginal) => { return { ...(await importOriginal()), COLLABORATION_SERVER_ORIGIN: 'http://localhost:3000', - Y_PROVIDER_API_KEY: 'yprovider-api-key', CONVERSION_FILE_MAX_SIZE: 500 * 1024, // 500kb }; }); -import { - Y_PROVIDER_API_KEY as apiKey, - COLLABORATION_SERVER_ORIGIN as origin, -} from '../src/env'; +import { JWKS_URL, COLLABORATION_SERVER_ORIGIN as origin } from '../src/env'; + +import { mockJwksEndpoint, signAdminToken } from './testUtils/adminJwt'; console.error = vi.fn(); describe('Server Tests', () => { + beforeEach(() => { + mockJwksEndpoint(JWKS_URL); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + test('Ping Pong', async () => { const app = initApp(); @@ -43,12 +49,13 @@ describe('Server Tests', () => { it('allows payloads up to 500kb for the CONVERT route', async () => { const app = initApp(); + const apiKey = await signAdminToken(); const largePayload = 'a'.repeat(400 * 1024); // 400kb payload const response = await request(app) .post(routes.CONVERT) .set('origin', origin) - .set('authorization', apiKey) + .set('authorization', `Bearer ${apiKey}`) .set('content-type', 'text/markdown') .send(largePayload); @@ -57,12 +64,13 @@ describe('Server Tests', () => { it('rejects payloads larger than CONVERSION_FILE_MAX_SIZE for the CONVERT route', async () => { const app = initApp(); + const apiKey = await signAdminToken(); const oversizedPayload = 'a'.repeat(501 * 1024); // 501kb payload const response = await request(app) .post(routes.CONVERT) .set('origin', origin) - .set('authorization', apiKey) + .set('authorization', `Bearer ${apiKey}`) .set('content-type', 'text/markdown') .send(oversizedPayload); diff --git a/src/frontend/servers/y-provider/__tests__/testUtils/adminJwt.ts b/src/frontend/servers/y-provider/__tests__/testUtils/adminJwt.ts new file mode 100644 index 0000000000..66d522aa65 --- /dev/null +++ b/src/frontend/servers/y-provider/__tests__/testUtils/adminJwt.ts @@ -0,0 +1,75 @@ +import { + SignJWT, + calculateJwkThumbprint, + exportJWK, + generateKeyPair, +} from 'jose'; +import { vi } from 'vitest'; + +import { JWT_ALGORITHM } from '@/middlewares'; + +const { privateKey, publicKey } = await generateKeyPair(JWT_ALGORITHM, { + extractable: true, +}); +const publicJwk = await exportJWK(publicKey); +const kid = await calculateJwkThumbprint(publicJwk); + +/** JWKS document shaped like the one Django's JWKSView publishes. */ +export const JWKS = { + keys: [{ ...publicJwk, kid, alg: JWT_ALGORITHM, use: 'sig' }], +}; + +/** Sign a token the way Django's JWTService would, for tests only. */ +export const signToken = (claims: Record) => + new SignJWT(claims) + .setProtectedHeader({ alg: JWT_ALGORITHM, kid }) + .setIssuedAt() + .setExpirationTime('1h') + .sign(privateKey); + +export const signAdminToken = () => signToken({ admin: true }); + +/** An admin token signed correctly but already past its expiry. */ +export const signExpiredAdminToken = () => + new SignJWT({ admin: true }) + .setProtectedHeader({ alg: JWT_ALGORITHM, kid }) + .setIssuedAt(Math.floor(Date.now() / 1000) - 3600) + .setExpirationTime(Math.floor(Date.now() / 1000) - 60) + .sign(privateKey); + +// A second, unrelated key pair: never published in the test JWKS, so a token +// signed with it must fail signature verification. +const { privateKey: roguePrivateKey } = await generateKeyPair(JWT_ALGORITHM, { + extractable: true, +}); + +/** + * An admin token carrying the real "kid" (so key lookup succeeds) but signed + * with a key that isn't the one published in the JWKS. + */ +export const signAdminTokenWithWrongKey = () => + new SignJWT({ admin: true }) + .setProtectedHeader({ alg: JWT_ALGORITHM, kid }) + .setIssuedAt() + .setExpirationTime('1h') + .sign(roguePrivateKey); + +/** + * Stub global fetch so jose's createRemoteJWKSet resolves our test JWKS + * instead of making a real network call to the Django backend. The real + * "jose" verification code still runs against a real signed token. + */ +export const mockJwksEndpoint = (jwksUrl: string) => { + vi.stubGlobal( + 'fetch', + vi.fn(async (input: string | URL) => { + if (input.toString() !== jwksUrl) { + throw new Error(`Unexpected fetch to ${input.toString()}`); + } + return new Response(JSON.stringify(JWKS), { + status: 200, + headers: { 'content-type': 'application/json' }, + }); + }), + ); +}; diff --git a/src/frontend/servers/y-provider/package.json b/src/frontend/servers/y-provider/package.json index f11df4b152..69f7279873 100644 --- a/src/frontend/servers/y-provider/package.json +++ b/src/frontend/servers/y-provider/package.json @@ -23,6 +23,7 @@ "@tiptap/extensions": "*", "cors": "2.8.6", "express": "5.2.1", + "jose": "6.2.8", "yjs": "*" }, "devDependencies": { diff --git a/src/frontend/servers/y-provider/src/env.ts b/src/frontend/servers/y-provider/src/env.ts index 13dfcb577f..564c72927c 100644 --- a/src/frontend/servers/y-provider/src/env.ts +++ b/src/frontend/servers/y-provider/src/env.ts @@ -1,9 +1,14 @@ import { readFileSync } from 'fs'; +export const COLLABORATION_BACKEND_BASE_URL = + process.env.COLLABORATION_BACKEND_BASE_URL || 'http://app-dev:8000'; export const COLLABORATION_LOGGING = process.env.COLLABORATION_LOGGING || 'false'; export const COLLABORATION_SERVER_ORIGIN = process.env.COLLABORATION_SERVER_ORIGIN || 'http://localhost:3000'; +// TODO(yhub): unused since the yhub migration, mirrors the Django setting of +// the same name (see impress/settings.py). Kept until CollaborationService is +// reinstated. export const COLLABORATION_SERVER_SECRET = process.env .COLLABORATION_SERVER_SECRET_FILE ? readFileSync(process.env.COLLABORATION_SERVER_SECRET_FILE, 'utf-8') @@ -11,8 +16,8 @@ export const COLLABORATION_SERVER_SECRET = process.env export const CONVERSION_FILE_MAX_SIZE = process.env.CONVERSION_FILE_MAX_SIZE ? Number(process.env.CONVERSION_FILE_MAX_SIZE) : 20971520; // 20 MB default -export const Y_PROVIDER_API_KEY = process.env.Y_PROVIDER_API_KEY_FILE - ? readFileSync(process.env.Y_PROVIDER_API_KEY_FILE, 'utf-8') - : process.env.Y_PROVIDER_API_KEY || 'yprovider-api-key'; +// JWKS of the Django backend, used to verify the JWT it signs when calling us. +export const JWKS_URL = + process.env.JWKS_URL || `${COLLABORATION_BACKEND_BASE_URL}/api/v1.0/jwks`; export const PORT = Number(process.env.PORT || 4444); export const SENTRY_DSN = process.env.SENTRY_DSN || ''; diff --git a/src/frontend/servers/y-provider/src/middlewares.ts b/src/frontend/servers/y-provider/src/middlewares.ts index 11a9e1df56..c7364ddacb 100644 --- a/src/frontend/servers/y-provider/src/middlewares.ts +++ b/src/frontend/servers/y-provider/src/middlewares.ts @@ -1,13 +1,9 @@ import cors from 'cors'; import { NextFunction, Request, Response } from 'express'; +import { createRemoteJWKSet, jwtVerify } from 'jose'; -import { - COLLABORATION_SERVER_ORIGIN, - COLLABORATION_SERVER_SECRET, - Y_PROVIDER_API_KEY, -} from '@/env'; +import { COLLABORATION_SERVER_ORIGIN, JWKS_URL } from '@/env'; -const VALID_API_KEYS = [COLLABORATION_SERVER_SECRET, Y_PROVIDER_API_KEY]; const allowedOrigins = COLLABORATION_SERVER_ORIGIN.split(','); export const corsMiddleware = cors({ @@ -16,11 +12,29 @@ export const corsMiddleware = cors({ credentials: true, }); -export const httpSecurity = ( +// Cached across requests: fetches the Django backend's public keys lazily and +// keeps them until their "kid" no longer matches a token, per jose's own policy. +const jwks = createRemoteJWKSet(new URL(JWKS_URL)); + +export const JWT_ALGORITHM = 'RS256'; + +/** + * Verify that the given token is an admin JWT signed by the Django backend. + */ +const isValidAdminToken = async (token: string): Promise => { + try { + const { payload } = await jwtVerify(token, jwks, { algorithms: [JWT_ALGORITHM] }); + return payload.admin === true; + } catch { + return false; + } +}; + +export const httpSecurity = async ( req: Request, res: Response, next: NextFunction, -): void => { +): Promise => { let apiKey = req.headers['authorization']; if (!apiKey) { @@ -32,7 +46,7 @@ export const httpSecurity = ( apiKey = apiKey.slice('Bearer '.length); } - if (!VALID_API_KEYS.includes(apiKey)) { + if (!(await isValidAdminToken(apiKey))) { res.status(401).json({ error: 'Unauthorized: Invalid API Key' }); return; } diff --git a/src/frontend/yarn.lock b/src/frontend/yarn.lock index 8f776ad9f6..5a26fd54a0 100644 --- a/src/frontend/yarn.lock +++ b/src/frontend/yarn.lock @@ -11291,6 +11291,11 @@ jest@30.4.2: import-local "^3.2.0" jest-cli "30.4.2" +jose@6.2.8: + version "6.2.8" + resolved "https://registry.yarnpkg.com/jose/-/jose-6.2.8.tgz#39c1459fe5eac84eb39b1623b8077dcf9ca6c506" + integrity sha512-Bsdjwm3Qsd/P0jR+BHDe3LytDfY7WBq2HmCCLIwuVRHMuEC9ae7/R474GIUdF1NgCyZjzVo/A9DOiOBtXq8ZoQ== + "js-tokens@^3.0.0 || ^4.0.0", js-tokens@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" @@ -13193,7 +13198,7 @@ react-intersection-observer@10.1.0: resolved "https://registry.yarnpkg.com/react-intersection-observer/-/react-intersection-observer-10.1.0.tgz#4aedf418c793f2bcf27353263f43553d12afba71" integrity sha512-V8HDu3+Llg6OEhOxx8LnUSS0t4VS+1Xk9ZatkI8Jct/H0CwKnqFTCu8NT3q7ghJTghTdIrEMPSWr2dkKPG+gdQ== -"react-is-18@npm:react-is@^18.3.1": +"react-is-18@npm:react-is@^18.3.1", react-is@^18.3.1: version "18.3.1" resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.3.1.tgz#e83557dc12eae63a99e003a46388b1dcbb44db7e" integrity sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg== @@ -13218,11 +13223,6 @@ react-is@^17.0.1: resolved "https://registry.yarnpkg.com/react-is/-/react-is-17.0.2.tgz#e691d4a8e9c789365655539ab372762b0efb54f0" integrity sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w== -react-is@^18.3.1: - version "18.3.1" - resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.3.1.tgz#e83557dc12eae63a99e003a46388b1dcbb44db7e" - integrity sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg== - react-lifecycles-compat@^3.0.0, react-lifecycles-compat@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/react-lifecycles-compat/-/react-lifecycles-compat-3.0.4.tgz#4f1a273afdfc8f3488a8c516bfda78f872352362" @@ -14273,16 +14273,7 @@ string-length@^4.0.2: char-regex "^1.0.2" strip-ansi "^6.0.0" -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -14424,14 +14415,7 @@ stringify-object@^3.3.0: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - -strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== From 65e27305d2272468b84cc1f7e30afcb8c076da2b Mon Sep 17 00:00:00 2001 From: Anthony LC Date: Tue, 4 Aug 2026 18:27:13 +0200 Subject: [PATCH 12/59] =?UTF-8?q?=F0=9F=94=A5(helm)=20remove=20occurences?= =?UTF-8?q?=20of=20Y=5FPROVIDER=5FAPI=5FKEY?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Y_PROVIDER_API_KEY is no longer used in the codebase, so we can remove it from the helm chart and the documentation. We adapt the documentation to use the new JWT conversion mechanism instead. --- documentation/examples/helm/impress.values.yaml | 1 - documentation/format_conversion.md | 11 +++++++---- src/helm/env.d/dev/values.impress.yaml.gotmpl | 1 - src/helm/env.d/feature/values.impress.yaml.gotmpl | 1 - 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/documentation/examples/helm/impress.values.yaml b/documentation/examples/helm/impress.values.yaml index ba965ade5e..05c8c2feda 100644 --- a/documentation/examples/helm/impress.values.yaml +++ b/documentation/examples/helm/impress.values.yaml @@ -135,7 +135,6 @@ yProvider: COLLABORATION_LOGGING: true COLLABORATION_SERVER_ORIGIN: https://docs.127.0.0.1.nip.io COLLABORATION_SERVER_SECRET: my-secret - Y_PROVIDER_API_KEY: my-secret ingress: enabled: true diff --git a/documentation/format_conversion.md b/documentation/format_conversion.md index 030e498e9f..828b82b1e4 100644 --- a/documentation/format_conversion.md +++ b/documentation/format_conversion.md @@ -8,21 +8,24 @@ To make it work, some configuration should be made and another service enabled i The first configuration to make is related to converting a docs in multiple format. This will be used by the `formatted-content` endpoint (`/api/v1.0/documents/{document_id}/formatted-content/?content_format=(json|html|markdown)`). This service is also used by the `create-for-owner` endpoint and in the import of markdown file. -To configure it, use this environment variables in the Django service: +To configure it, use this environment variable in the Django service: ```yaml Y_PROVIDER_API_BASE_URL: http://{y-provider-service}:443/api/ -Y_PROVIDER_API_KEY: a-shared-private-key-with-y-provider ``` For the `Y_PROVIDER_API_BASE_URL`, it can be the FQDN of your docs instance if you have configured a reverse proxy in front of the y-provider service and created a route to the `/api` for this service. It can also be the internal `y-provider` service url if Django can access it directly. In the case you deploy in a Kubernetes cluster, you can use the `y-provider` service url. We prefer the usage of internal url. -You also have to add an environment variable in your `y-provider` configuration, to share the same `Y_PROVIDER_API_KEY`: +Requests to the y-provider service are authenticated with a short-lived admin JWT that Django signs itself (see `core.services.jwt_services.JWTService`), instead of a shared secret. The y-provider service verifies the signature against the public key Django publishes on its JWKS endpoint (`/api/v1.0/jwks`), so there is nothing to configure on the Django side beyond `JWT_PRIVATE_KEY` (see the JWT section of [env.md](env.md)). + +On the `y-provider` side, point it at the Django backend so it can fetch the JWKS: ```yaml -Y_PROVIDER_API_KEY: a-shared-private-key-with-y-provider +COLLABORATION_BACKEND_BASE_URL: http://{django-service}:8000 ``` +The JWKS url defaults to `{COLLABORATION_BACKEND_BASE_URL}/api/v1.0/jwks`; override it with `JWKS_URL` if Django is not reachable at that base url from the y-provider service. + ### Splitting conversion service The conversion service is present in the `y-provider` server. The same server used to manage websockets. You can split in one side the websocket server and in an other side the converter service. diff --git a/src/helm/env.d/dev/values.impress.yaml.gotmpl b/src/helm/env.d/dev/values.impress.yaml.gotmpl index 85c319785f..a35727a259 100644 --- a/src/helm/env.d/dev/values.impress.yaml.gotmpl +++ b/src/helm/env.d/dev/values.impress.yaml.gotmpl @@ -176,7 +176,6 @@ yProvider: COLLABORATION_LOGGING: true COLLABORATION_SERVER_ORIGIN: https://docs.127.0.0.1.nip.io COLLABORATION_SERVER_SECRET: my-secret - Y_PROVIDER_API_KEY: my-secret NODE_EXTRA_CA_CERTS: /cert/cacert.pem # Extra volume mounts to manage our local custom CA and avoid to set ssl_verify: false diff --git a/src/helm/env.d/feature/values.impress.yaml.gotmpl b/src/helm/env.d/feature/values.impress.yaml.gotmpl index 3da4264866..884fee0931 100644 --- a/src/helm/env.d/feature/values.impress.yaml.gotmpl +++ b/src/helm/env.d/feature/values.impress.yaml.gotmpl @@ -154,7 +154,6 @@ yProvider: COLLABORATION_LOGGING: true COLLABORATION_SERVER_ORIGIN: https://{{ .Values.feature }}-docs.{{ .Values.domain }} COLLABORATION_SERVER_SECRET: my-secret - Y_PROVIDER_API_KEY: my-secret NODE_OPTIONS: "--max-old-space-size=1024" docSpec: From 921e4c3968a4ec16cf2806fdc1bf8c4dd6690adc Mon Sep 17 00:00:00 2001 From: Anthony LC Date: Wed, 5 Aug 2026 11:33:12 +0200 Subject: [PATCH 13/59] =?UTF-8?q?=F0=9F=9B=82(backend)=20add=20audience=20?= =?UTF-8?q?to=20jwt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add audience to the jwt, scoping the token to it prevents an admin JWT issued for another backend service from being replayed against y-provider. --- src/backend/core/services/converter_services.py | 8 +++++++- .../core/tests/test_services_converter_services.py | 7 +++++-- .../y-provider/__tests__/middlewares.test.ts | 14 ++++++++++++++ .../y-provider/__tests__/testUtils/adminJwt.ts | 7 ++++++- src/frontend/servers/y-provider/src/middlewares.ts | 11 +++++++++-- 5 files changed, 41 insertions(+), 6 deletions(-) diff --git a/src/backend/core/services/converter_services.py b/src/backend/core/services/converter_services.py index ee542b1c7f..ce527dcb71 100644 --- a/src/backend/core/services/converter_services.py +++ b/src/backend/core/services/converter_services.py @@ -13,6 +13,11 @@ logger = logging.getLogger(__name__) +# Audience of the admin token y-provider expects. Scoping the token to it +# prevents an admin JWT issued for another backend service from being +# replayed against y-provider. +Y_CONVERTER_AUDIENCE = "y-converter" + class ConversionError(Exception): """Base exception for conversion-related errors.""" @@ -110,7 +115,8 @@ class YdocConverter: @property def auth_header(self): """Build microservice authentication header.""" - return f"Bearer {JWTService().get_admin_token()}" + token = JWTService().get_admin_token({"aud": Y_CONVERTER_AUDIENCE}) + return f"Bearer {token}" def _request(self, url, data, content_type, accept): """Make a request to the Y-Provider API.""" diff --git a/src/backend/core/tests/test_services_converter_services.py b/src/backend/core/tests/test_services_converter_services.py index 345e4fb249..0b6f5183bf 100644 --- a/src/backend/core/tests/test_services_converter_services.py +++ b/src/backend/core/tests/test_services_converter_services.py @@ -27,14 +27,17 @@ def jwt_settings(settings): def test_auth_header(): - """The auth header carries an admin JWT signed with the configured key.""" + """The auth header carries an admin JWT scoped to the y-converter audience.""" converter = YdocConverter() scheme, token = converter.auth_header.split(" ") assert scheme == "Bearer" - payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + payload = jwt.decode( + token, PUBLIC_KEY, algorithms=["RS256"], audience="y-converter" + ) assert payload["admin"] is True + assert payload["aud"] == "y-converter" def test_convert_empty_text(): diff --git a/src/frontend/servers/y-provider/__tests__/middlewares.test.ts b/src/frontend/servers/y-provider/__tests__/middlewares.test.ts index cc782f9fab..935abb73c8 100644 --- a/src/frontend/servers/y-provider/__tests__/middlewares.test.ts +++ b/src/frontend/servers/y-provider/__tests__/middlewares.test.ts @@ -18,6 +18,7 @@ import { httpSecurity } from '@/middlewares'; import { mockJwksEndpoint, signAdminToken, + signAdminTokenForAudience, signAdminTokenWithWrongKey, signExpiredAdminToken, signToken, @@ -88,6 +89,19 @@ describe('httpSecurity', () => { }); }); + it('rejects a valid admin JWT issued for another audience', async () => { + const token = await signAdminTokenForAudience('some-other-service'); + + const response = await request(buildApp()) + .get('/protected') + .set('authorization', `Bearer ${token}`); + + expect(response.status).toBe(401); + expect(response.body).toStrictEqual({ + error: 'Unauthorized: Invalid API Key', + }); + }); + it('rejects a validly signed JWT missing the admin claim', async () => { const token = await signToken({ sub: 'someone' }); diff --git a/src/frontend/servers/y-provider/__tests__/testUtils/adminJwt.ts b/src/frontend/servers/y-provider/__tests__/testUtils/adminJwt.ts index 66d522aa65..e5ffe3288d 100644 --- a/src/frontend/servers/y-provider/__tests__/testUtils/adminJwt.ts +++ b/src/frontend/servers/y-provider/__tests__/testUtils/adminJwt.ts @@ -27,7 +27,12 @@ export const signToken = (claims: Record) => .setExpirationTime('1h') .sign(privateKey); -export const signAdminToken = () => signToken({ admin: true }); +export const signAdminToken = () => + signToken({ admin: true, aud: 'y-converter' }); + +/** An admin token correctly signed but scoped to another service's audience. */ +export const signAdminTokenForAudience = (aud: string) => + signToken({ admin: true, aud }); /** An admin token signed correctly but already past its expiry. */ export const signExpiredAdminToken = () => diff --git a/src/frontend/servers/y-provider/src/middlewares.ts b/src/frontend/servers/y-provider/src/middlewares.ts index c7364ddacb..863afa3b45 100644 --- a/src/frontend/servers/y-provider/src/middlewares.ts +++ b/src/frontend/servers/y-provider/src/middlewares.ts @@ -16,14 +16,21 @@ export const corsMiddleware = cors({ // keeps them until their "kid" no longer matches a token, per jose's own policy. const jwks = createRemoteJWKSet(new URL(JWKS_URL)); +// Requiring this audience stops a valid admin JWT issued for another service +// from being replayed against y-provider. +const Y_CONVERTER_AUDIENCE = 'y-converter'; export const JWT_ALGORITHM = 'RS256'; /** - * Verify that the given token is an admin JWT signed by the Django backend. + * Verify that the given token is an admin JWT signed by the Django backend + * for the y-converter audience. */ const isValidAdminToken = async (token: string): Promise => { try { - const { payload } = await jwtVerify(token, jwks, { algorithms: [JWT_ALGORITHM] }); + const { payload } = await jwtVerify(token, jwks, { + algorithms: [JWT_ALGORITHM], + audience: Y_CONVERTER_AUDIENCE, + }); return payload.admin === true; } catch { return false; From 349f0928fd1de4025d46192859afe48090ea50fd Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Wed, 5 Aug 2026 12:28:39 +0200 Subject: [PATCH 14/59] =?UTF-8?q?=E2=9C=A8(collaboration)=20add=20create-y?= =?UTF-8?q?doc=20endpoint=20on=20yhub?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Python cannot call yhub's built-in PATCH ydoc api because its body must be lib0-any encoded - a lib0-specific binary framing with no implementation outside javascript. The new endpoint POST /collaboration/create-ydoc/v1/{org}/{docid} accepts the raw binary Yjs update (pycrdt get_update() / Y.encodeStateAsUpdate output) as application/octet-stream, so Django can seed a document's initial state with a plain requests.post(url, data=raw_bytes) - needed by the server-side creation flows (file import, create-for-owner, duplication, template instantiation) whose yhub rooms currently stay empty until the first browser connects. Strict create semantics: 409 when the room already has content (checked via getDoc, covering persisted state and uncompacted stream messages; yhub has no atomic create, concurrent creates merge via CRDT and never corrupt). The initial content is attributed to the optional X-User-Id header, else to the caller's identity. Access uses the default purpose, i.e. standard document write access like the built-in ydoc routes: the admin JWT, or a user session with update ability. Malformed updates map to 400 (the compute worker rejects them and the pool replaces the thread), empty updates to 400, bodies over 10MiB to 413. Gotcha worth noting: req.bytes() resolves to a Node Buffer, but yhub's compute-task schema validates with lib0's exact-constructor Uint8Array check, so the body is re-viewed as a plain Uint8Array before it is handed to the compute pool. Co-Authored-By: Claude Fable 5 Signed-off-by: Kevin Jahns --- CHANGELOG.md | 7 +++ src/yhub-server/README.md | 9 ++++ src/yhub-server/server.js | 97 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a9300968b..ee04d3c86e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ and this project adheres to ### Added +- ✨(collaboration) add a create-ydoc endpoint on yhub: + `POST /collaboration/create-ydoc/v1/docs/{id}` seeds a document's initial + Yjs state from a raw binary update posted as `application/octet-stream`, + so the Django backend can create documents without speaking yhub's lib0 + wire encoding. Strict create (409 when the document already has content), + initial content attributed to the optional `X-User-Id` header; guarded by + standard document write access (admin JWT or user session) - ✨(collaboration) add an admin reset-connections endpoint on yhub: `POST /collaboration/reset-connections/v1/docs/{id}` re-checks the authorization of the document's connected clients and disconnects (close diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index 248a52d9ae..cc907d9ea4 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -20,6 +20,15 @@ It is not a fork of yhub — it is a thin wrapper (`server.js`) that: pending) — authenticated with an RS256 admin JWT issued by Django and verified against its JWKS (`/api/v1.0/jwks`); the `reset-connections` purpose is granted only to that admin token, never to regular users, +- exposes `POST /collaboration/create-ydoc/v1/{org}/{docid}` (optional + `X-User-Id` header naming the user the initial content is attributed to), + which seeds a document's initial Yjs state from a raw binary update + (`Y.encodeStateAsUpdate` / pycrdt `get_update()` output posted as + `application/octet-stream` — no lib0 encoding, unlike yhub's built-in + `PATCH .../ydoc/`), so the Django backend can create documents + server-side. Strict create: 409 when the document already has content. + Guarded by standard document write access (the admin JWT, or a user + session with update ability), - mirrors the environment conventions used elsewhere in this repository (`*_FILE` secret indirection, `COLLABORATION_SERVER_ORIGIN` allowlist, …). diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index 5eade43e3a..20f940583d 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -23,6 +23,15 @@ const Y_PROVIDER_API_KEY = secret('Y_PROVIDER_API_KEY', 'yprovider-api-key'); const ORG = process.env.YHUB_ORG || 'docs'; const UUID4 = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +// an empty Yjs update (what `Y.encodeStateAsUpdate(new Y.Doc())` encodes to) — +// hardcoded so we don't import @y/y for two bytes +const EMPTY_YDOC = new Uint8Array([0, 0]); +// uws buffers the whole body before the handler sees it, so this cap does not +// bound upload memory — it bounds what a single create hands to a compute +// worker and writes to the valkey stream as one message. Creates carry one +// freshly-converted snapshot (typically KBs); anything bigger belongs on the +// websocket path. +const MAX_CREATE_BYTES = 10 * 1024 * 1024; // Public keys verifying the RS256 admin tokens Django issues (JWTService). // Lazily fetched on first use; jose caches the keys and refetches on unknown @@ -169,6 +178,94 @@ const api = [ }, }, }), + // POST /collaboration/create-ydoc/v1/{org}/{docid} — create a document's + // initial Yjs state from a RAW binary update (`Y.encodeStateAsUpdate` / + // pycrdt `get_update()` output) posted as application/octet-stream. Unlike + // yhub's built-in `PATCH ydoc`, the body is not lib0-any encoded, so Django + // can call it with a plain `requests.post(url, data=raw_bytes)`. Strict + // create: 409 when the room already has content. Default access purpose: + // guarded like the built-in ydoc routes (write access on the doc — the + // admin JWT, or a user session with update ability). + createApiEndpoint('create-ydoc', { + post: { + handler: async (req) => { + if (req.org !== ORG) { + return jsonResponse(400, { error: 'Unknown org' }); + } + if (!UUID4.test(req.docid)) { + return jsonResponse(400, { error: 'Room name is invalid' }); + } + const body = await req.bytes(); + // req.bytes() resolves to a Node Buffer, but the compute-task schema + // requires an exact Uint8Array (lib0 $constructedBy compares the + // constructor) — re-view the same bytes without copying + const update = new Uint8Array( + body.buffer, + body.byteOffset, + body.byteLength, + ); + if (update.byteLength > MAX_CREATE_BYTES) { + // 413 is missing from yhub's status-line map, so the reason phrase + // is empty ("HTTP/1.1 413 ") — legal, and callers switch on the code + return jsonResponse(413, { error: 'Update too large' }); + } + // <= 3 bytes is yhub's "no effective content" convention (an empty + // update encodes to 2 bytes) — reject before it reaches a worker + if (update.byteLength <= 3) { + return jsonResponse(400, { error: 'Empty update' }); + } + // covers persisted state AND uncompacted stream messages. Not atomic + // with addMessage below (yhub has no atomic create): two concurrent + // creates can both pass — acceptable, Yjs merges both updates; worst + // case is a doubly-attributed first revision, never corruption. + const { gcDoc } = await req.yhub.getDoc( + req.room, + { gc: true, nongc: false }, + { gcOnMerge: false }, + ); + if (gcDoc != null && gcDoc.byteLength > 3) { + return jsonResponse(409, { error: 'Document already exists' }); + } + // attribute the initial content to the acting user when the caller + // names one, else to the caller's identity ('system' for admin tokens) + const userid = req.headers['x-user-id'] || req.authInfo.userid; + let result; + try { + // diffs the posted update against the (empty) current doc and + // stamps the attribution contentmap + result = await req.yhub.computePool.patchYdoc( + { + update, + currentDoc: gcDoc ?? EMPTY_YDOC, + userid, + customAttributions: [], + }, + { room: req.room }, + ); + } catch { + // a malformed update makes the compute worker throw (yhub logs + // 'worker failed' and replaces the thread). The update is the only + // untrusted input here, so a rejection maps to 400; getDoc / + // addMessage failures stay generic 500s. + return jsonResponse(400, { error: 'Invalid Yjs update' }); + } + if (result == null) { + // structurally valid but no effective content (e.g. delete-set + // only). A "successful" create that leaves the room nonexistent + // would lie to the caller — a later create would not 409. + return jsonResponse(400, { error: 'Empty update' }); + } + // on a fresh room this creates the stream, schedules compaction, and + // fans out to any live subscribers — nothing else to do + await req.yhub.stream.addMessage(req.room, { + type: 'ydoc:update:v1', + contentmap: result.contentmap, + update: result.update, + }); + return jsonResponse(201, { message: 'Document created' }); + }, + }, + }), ]; await createYHub({ From dc76f9001b11d9e1e73159fcced4d298c050c5d2 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Wed, 5 Aug 2026 12:31:41 +0200 Subject: [PATCH 15/59] =?UTF-8?q?=F0=9F=94=92=EF=B8=8F(collaboration)=20ha?= =?UTF-8?q?rden=20the=20create-ydoc=20endpoint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address the findings of an adversarial review of the new endpoint: - Only the backend admin token may attribute content to another user via the X-User-Id header. The endpoint uses the default access purpose, so any editor with update ability can call it — honoring the header for them would let an editor forge the attribution history of the first revision (the websocket path likewise stamps the server-side identity). Regular callers now always author as themselves; verified: an editor session posting X-User-Id gets its own userid stamped. - Reject non-main ?branch= requests (400). Cookie users are main-only via getAccessType, but the admin token bypasses it and could seed an orphan (org, docid, branch) room no user-facing path reads — while dodging the branch-scoped 409 existence check. - Correct the concurrent-create comment: two racing creates merge as independently generated updates (fresh clientIDs), so the seeded content appears twice — user-visible duplication, not merely a doubly-attributed revision. Still accepted (Django creates each doc once and a duplicated seed is user-fixable), but the tradeoff is now stated accurately. Co-Authored-By: Claude Fable 5 Signed-off-by: Kevin Jahns --- src/yhub-server/server.js | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index 20f940583d..4f7f303f2f 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -195,6 +195,13 @@ const api = [ if (!UUID4.test(req.docid)) { return jsonResponse(400, { error: 'Room name is invalid' }); } + if (req.branch !== 'main') { + // cookie users are main-only via getAccessType, but the admin + // token bypasses it — reject explicitly so an admin create can't + // seed an orphan non-main room (and dodge the 409 check, which is + // branch-scoped) + return jsonResponse(400, { error: 'Unknown branch' }); + } const body = await req.bytes(); // req.bytes() resolves to a Node Buffer, but the compute-task schema // requires an exact Uint8Array (lib0 $constructedBy compares the @@ -216,8 +223,10 @@ const api = [ } // covers persisted state AND uncompacted stream messages. Not atomic // with addMessage below (yhub has no atomic create): two concurrent - // creates can both pass — acceptable, Yjs merges both updates; worst - // case is a doubly-attributed first revision, never corruption. + // creates can both pass the check and their updates merge — with + // independently generated updates (fresh clientIDs) the seeded + // content then appears twice. Accepted: Django creates each doc + // once, and a duplicated seed is user-fixable, unlike corruption. const { gcDoc } = await req.yhub.getDoc( req.room, { gc: true, nongc: false }, @@ -226,9 +235,13 @@ const api = [ if (gcDoc != null && gcDoc.byteLength > 3) { return jsonResponse(409, { error: 'Document already exists' }); } - // attribute the initial content to the acting user when the caller - // names one, else to the caller's identity ('system' for admin tokens) - const userid = req.headers['x-user-id'] || req.authInfo.userid; + // Only the backend admin token may attribute the content to another + // user; regular callers always author as themselves — honoring a + // client-supplied header would let any editor forge the attribution + // history (the ws path likewise stamps the server-side identity). + const userid = + (req.authInfo.admin === true && req.headers['x-user-id']) || + req.authInfo.userid; let result; try { // diffs the posted update against the (empty) current doc and From 6f0bdc7fb3d876470f1220b2dcafcb552c3ff29d Mon Sep 17 00:00:00 2001 From: Anthony LC Date: Wed, 5 Aug 2026 14:11:14 +0200 Subject: [PATCH 16/59] =?UTF-8?q?=F0=9F=94=A5(frontend)=20remove=20"can-ed?= =?UTF-8?q?it"=20mechanism?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We will not block anymore the users not connected to the collaboration server from editing the document, we will have an HTTP fallback instead, so we can remove the "can-edit" mechanism and the related code. --- .../app-impress/doc-collaboration.spec.ts | 143 +----------------- .../doc-editor/__tests__/DocEditor.spec.tsx | 1 - .../docs/doc-editor/components/DocEditor.tsx | 5 +- .../doc-header/components/AlertNetwork.tsx | 125 --------------- .../docs/doc-header/components/DocHeader.tsx | 6 +- .../doc-header/components/DocHeaderInfo.tsx | 10 +- .../docs/doc-header/components/DocTitle.tsx | 4 +- .../docs/doc-management/api/useDocCanEdit.tsx | 32 ---- .../api/useDocContentUpdate.tsx | 7 - .../docs/doc-management/hooks/index.ts | 1 - .../hooks/useIsCollaborativeEditable.tsx | 80 ---------- 11 files changed, 10 insertions(+), 404 deletions(-) delete mode 100644 src/frontend/apps/impress/src/features/docs/doc-header/components/AlertNetwork.tsx delete mode 100644 src/frontend/apps/impress/src/features/docs/doc-management/api/useDocCanEdit.tsx delete mode 100644 src/frontend/apps/impress/src/features/docs/doc-management/hooks/useIsCollaborativeEditable.tsx diff --git a/src/frontend/apps/e2e/__tests__/app-impress/doc-collaboration.spec.ts b/src/frontend/apps/e2e/__tests__/app-impress/doc-collaboration.spec.ts index 526e25a747..fb237161ad 100644 --- a/src/frontend/apps/e2e/__tests__/app-impress/doc-collaboration.spec.ts +++ b/src/frontend/apps/e2e/__tests__/app-impress/doc-collaboration.spec.ts @@ -5,7 +5,6 @@ import { expect, test } from '@playwright/test'; import { createDoc, overrideConfig, verifyDocName } from './utils-common'; import { openSuggestionMenu, writeInEditor } from './utils-editor'; import { connectOtherUserToDoc, updateShareLink } from './utils-share'; -import { createRootSubPage } from './utils-sub-pages'; test.beforeEach(async ({ page }) => { await page.goto('/'); @@ -111,147 +110,7 @@ test.describe('Doc Collaboration', () => { await cleanup(); }); - // TODO(yhub): re-enable when yhub exposes a connection-info API - the test - // asserts `can_edit=false` while another user is connected to the - // collaborative server, but `get_document_connection_info` is currently - // stubbed to report no connections. - test.skip('it checks block editing when not connected to collab server', async ({ - page, - browserName, - }) => { - test.slow(); - - /** - * The good port is 3002, but we want to simulate a not connected - * collaborative server. - * So we use a port that is not used by the collaborative server. - * The server will not be able to connect to the collaborative server. - */ - await overrideConfig(page, { - COLLABORATION_WS_URL: 'ws://localhost:5555/ws/docs', - COLLABORATION_WS_NOT_CONNECTED_READ_ONLY: true, - }); - - await page.goto('/'); - - const [parentTitle] = await createDoc( - page, - 'editing-blocking', - browserName, - 1, - ); - - const card = page.getByLabel('It is the card information'); - await expect( - card.getByText('Others are editing. Your network prevent changes.'), - ).toBeHidden(); - const editor = page.locator('.ProseMirror'); - - await expect(editor).toHaveAttribute('contenteditable', 'true'); - - let responseCanEditPromise = page.waitForResponse( - (response) => - response.url().includes(`/can-edit/`) && response.status() === 200, - ); - - await page.getByRole('button', { name: 'Share' }).click(); - - await updateShareLink(page, 'Public', 'Editing'); - - // Close the modal - await page.getByRole('button', { name: 'close' }).first().click(); - - const urlParentDoc = page.url(); - - const { name: childTitle } = await createRootSubPage( - page, - browserName, - 'editing-blocking - child', - ); - - let responseCanEdit = await responseCanEditPromise; - expect(responseCanEdit.ok()).toBeTruthy(); - let jsonCanEdit = (await responseCanEdit.json()) as { can_edit: boolean }; - expect(jsonCanEdit.can_edit).toBeTruthy(); - - const urlChildDoc = page.url(); - - /** - * We open another browser that will connect to the collaborative server - * and will block the current browser to edit the doc. - */ - const { otherPage, cleanup } = await connectOtherUserToDoc({ - browserName, - docUrl: urlChildDoc, - docTitle: childTitle, - withoutSignIn: true, - }); - - const webSocketPromise = otherPage.waitForEvent( - 'websocket', - (webSocket) => { - return webSocket.url().includes(`${process.env.COLLABORATION_WS_URL}/`); - }, - ); - - await otherPage.goto(urlChildDoc); - - const webSocket = await webSocketPromise; - expect(webSocket.url()).toContain(`${process.env.COLLABORATION_WS_URL}/`); - - await verifyDocName(otherPage, childTitle); - - await page.reload(); - - responseCanEdit = await page.waitForResponse( - (response) => - response.url().includes(`/can-edit/`) && response.status() === 200, - ); - expect(responseCanEdit.ok()).toBeTruthy(); - - jsonCanEdit = (await responseCanEdit.json()) as { can_edit: boolean }; - expect(jsonCanEdit.can_edit).toBeFalsy(); - - await expect( - card.getByText('Others are editing. Your network prevent changes.'), - ).toBeVisible({ - timeout: 10000, - }); - - await expect(editor).toHaveAttribute('contenteditable', 'false'); - - await expect( - page.getByRole('textbox', { name: 'Document title' }), - ).toBeHidden(); - await expect(page.getByRole('heading', { name: childTitle })).toBeVisible(); - - await page.goto(urlParentDoc); - - await verifyDocName(page, parentTitle); - - await page.getByRole('button', { name: 'Share' }).click(); - - await page.getByTestId('doc-access-mode').click(); - await page.getByRole('menuitemradio', { name: 'Reading' }).click(); - - // Close the modal - await page.getByRole('button', { name: 'close' }).first().click(); - - await page.goto(urlChildDoc); - - await expect(editor).toHaveAttribute('contenteditable', 'true'); - - await expect( - page.getByRole('textbox', { name: 'Document title' }), - ).toContainText(childTitle); - await expect(page.getByRole('heading', { name: childTitle })).toBeHidden(); - - await expect( - card.getByText('Others are editing. Your network prevent changes.'), - ).toBeHidden(); - - await cleanup(); - }); + // TODO(yhub): Add test to check that no connected websocket users can collaborate test('checks disconnection and reconnection when changing tab visibility', async ({ page, diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/__tests__/DocEditor.spec.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/__tests__/DocEditor.spec.tsx index e27870b94d..9db8681311 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-editor/__tests__/DocEditor.spec.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/__tests__/DocEditor.spec.tsx @@ -25,7 +25,6 @@ vi.mock('../../doc-management', async () => { const actual = await vi.importActual('../../doc-management'); return { ...actual, - useIsCollaborativeEditable: () => ({ isEditable: true, isLoading: false }), useProviderStore: () => ({ provider: { roomname: 'test-doc-id', diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/components/DocEditor.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/components/DocEditor.tsx index f24db166bc..79bedea1f8 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-editor/components/DocEditor.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/components/DocEditor.tsx @@ -8,7 +8,6 @@ import { Doc, LinkReach, getDocLinkReach, - useIsCollaborativeEditable, useProviderStore, } from '@/docs/doc-management'; import { useAuth } from '@/features/auth/'; @@ -85,10 +84,8 @@ interface DocEditorProps { export const DocEditor = ({ doc }: DocEditorProps) => { useCollaboration(doc.id); - const { isEditable, isLoading } = useIsCollaborativeEditable(doc); const isDeletedDoc = !!doc.deleted_at; - const readOnly = - !doc.abilities.partial_update || !isEditable || isLoading || isDeletedDoc; + const readOnly = !doc.abilities.partial_update || isDeletedDoc; const { trackEvent } = useAnalytics(); const [hasTracked, setHasTracked] = useState(false); const { authenticated } = useAuth(); diff --git a/src/frontend/apps/impress/src/features/docs/doc-header/components/AlertNetwork.tsx b/src/frontend/apps/impress/src/features/docs/doc-header/components/AlertNetwork.tsx deleted file mode 100644 index a0837feebd..0000000000 --- a/src/frontend/apps/impress/src/features/docs/doc-header/components/AlertNetwork.tsx +++ /dev/null @@ -1,125 +0,0 @@ -import { Button, Modal, ModalSize } from '@gouvfr-lasuite/cunningham-react'; -import { t } from 'i18next'; -import { useState } from 'react'; -import { useTranslation } from 'react-i18next'; - -import { Box, BoxButton, Card, Icon, Text } from '@/components'; -import { useCunninghamTheme } from '@/cunningham'; - -export const AlertNetwork = () => { - const { t } = useTranslation(); - const { spacingsTokens } = useCunninghamTheme(); - const [isModalOpen, setIsModalOpen] = useState(false); - - return ( - <> - - - - - - {t('Others are editing. Your network prevent changes.')} - - - setIsModalOpen(true)} - $withThemeInherited - > - - - {t('Learn more')} - - - - - {isModalOpen && ( - setIsModalOpen(false)} /> - )} - - ); -}; - -interface AlertNetworkModalProps { - onClose: () => void; -} - -export const AlertNetworkModal = ({ onClose }: AlertNetworkModalProps) => { - return ( - onClose()} - aria-label={t("Why you can't edit the document?")} - rightActions={ - <> - - - } - size={ModalSize.MEDIUM} - title={ - - {t("Why you can't edit the document?")} - - } - > - - - {t( - 'Others are editing this document. Unfortunately your network blocks WebSockets, the technology enabling real-time co-editing.', - )} - - - {t("This means you can't edit until others leave.")}{' '} - - {t( - 'If you wish to be able to co-edit in real-time, contact your Information Systems Security Manager about allowing WebSockets.', - )} - - - - - ); -}; diff --git a/src/frontend/apps/impress/src/features/docs/doc-header/components/DocHeader.tsx b/src/frontend/apps/impress/src/features/docs/doc-header/components/DocHeader.tsx index 6e24e17fbd..6e0e1e72c9 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-header/components/DocHeader.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-header/components/DocHeader.tsx @@ -10,10 +10,8 @@ import { getEmojiAndTitle, useDocTitleUpdate, useDocUtils, - useIsCollaborativeEditable, } from '@/docs/doc-management'; -import { AlertNetwork } from './AlertNetwork'; import { AlertRestore } from './AlertRestore'; import { DocHeaderInfo } from './DocHeaderInfo'; import { DocTitle } from './DocTitle'; @@ -24,7 +22,6 @@ interface DocHeaderProps { export const DocHeader = ({ doc }: DocHeaderProps) => { const { t } = useTranslation(); - const { isEditable } = useIsCollaborativeEditable(doc); const isDeletedDoc = !!doc.deleted_at; // Emoji Management const { emoji } = getEmojiAndTitle(doc.title ?? ''); @@ -57,11 +54,10 @@ export const DocHeader = ({ doc }: DocHeaderProps) => { {isDeletedDoc && } - {!isEditable && } diff --git a/src/frontend/apps/impress/src/features/docs/doc-header/components/DocHeaderInfo.tsx b/src/frontend/apps/impress/src/features/docs/doc-header/components/DocHeaderInfo.tsx index c5f7f8d739..530e0e83ba 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-header/components/DocHeaderInfo.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-header/components/DocHeaderInfo.tsx @@ -9,7 +9,6 @@ import { LinkReach, Role, getDocLinkReach, - useIsCollaborativeEditable, useTrans, } from '@/docs/doc-management'; import { useDate } from '@/hooks'; @@ -20,7 +19,6 @@ interface DocHeaderInfoProps { export const DocHeaderInfo = ({ doc }: DocHeaderInfoProps) => { const { transRole } = useTrans(); - const { isEditable } = useIsCollaborativeEditable(doc); const { relativeDate, calculateDaysLeft } = useDate(); const { data: config } = useConfig(); @@ -50,12 +48,16 @@ export const DocHeaderInfo = ({ doc }: DocHeaderInfoProps) => { $variation="tertiary" $size="s" $weight="bold" - $theme={isEditable ? 'neutral' : 'warning'} + $theme={doc.abilities.partial_update ? 'neutral' : 'warning'} $direction="row" $margin="0" > - {transRole(isEditable ? doc.user_role || doc.link_role : Role.READER)} + {transRole( + doc.abilities.partial_update + ? doc.user_role || doc.link_role + : Role.READER, + )}  ·  diff --git a/src/frontend/apps/impress/src/features/docs/doc-header/components/DocTitle.tsx b/src/frontend/apps/impress/src/features/docs/doc-header/components/DocTitle.tsx index 23a06c1075..b7e38b8308 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-header/components/DocTitle.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-header/components/DocTitle.tsx @@ -11,7 +11,6 @@ import { useDocStore, useDocTitleUpdate, useDocUtils, - useIsCollaborativeEditable, useTrans, } from '@/docs/doc-management'; import SimpleFileIcon from '@/features/docs/doc-management/assets/simple-document.svg'; @@ -24,8 +23,7 @@ interface DocTitleProps { } export const DocTitle = ({ doc }: DocTitleProps) => { - const { isEditable, isLoading } = useIsCollaborativeEditable(doc); - const readOnly = !doc.abilities.partial_update || !isEditable || isLoading; + const readOnly = !doc.abilities.partial_update; if (readOnly) { return ; diff --git a/src/frontend/apps/impress/src/features/docs/doc-management/api/useDocCanEdit.tsx b/src/frontend/apps/impress/src/features/docs/doc-management/api/useDocCanEdit.tsx deleted file mode 100644 index 8847ef94e3..0000000000 --- a/src/frontend/apps/impress/src/features/docs/doc-management/api/useDocCanEdit.tsx +++ /dev/null @@ -1,32 +0,0 @@ -import { UseQueryOptions, useQuery } from '@tanstack/react-query'; - -import { APIError, errorCauses, fetchAPI } from '@/api'; - -type DocCanEditResponse = { can_edit: boolean }; - -export const docCanEdit = async (id: string): Promise => { - const response = await fetchAPI(`documents/${id}/can-edit/`); - - if (!response.ok) { - throw new APIError('Failed to get the doc', await errorCauses(response)); - } - - return response.json() as Promise; -}; - -export const KEY_CAN_EDIT = 'doc-can-edit'; - -export function useDocCanEdit( - param: string, - queryConfig?: UseQueryOptions< - DocCanEditResponse, - APIError, - DocCanEditResponse - >, -) { - return useQuery({ - queryKey: [KEY_CAN_EDIT, param], - queryFn: () => docCanEdit(param), - ...queryConfig, - }); -} diff --git a/src/frontend/apps/impress/src/features/docs/doc-management/api/useDocContentUpdate.tsx b/src/frontend/apps/impress/src/features/docs/doc-management/api/useDocContentUpdate.tsx index 23cb7402ef..8f845a29ba 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-management/api/useDocContentUpdate.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-management/api/useDocContentUpdate.tsx @@ -9,7 +9,6 @@ import { APIError, errorCauses, fetchAPI } from '@/api'; import { Doc } from '../types'; -import { KEY_CAN_EDIT } from './useDocCanEdit'; import { KEY_DOC_CONTENT } from './useDocContent'; export interface UpdateDocContentParams { @@ -112,12 +111,6 @@ export function useDocContentUpdate(queryConfig?: UseDocContentUpdate) { ); } - // If error it means the user is probably not allowed to edit the doc - // so we invalidate the canEdit query to update the UI accordingly - void queryClient.invalidateQueries({ - queryKey: [KEY_CAN_EDIT], - }); - if (queryConfig?.onError) { queryConfig.onError(error, variables, onMutateResult, context); } diff --git a/src/frontend/apps/impress/src/features/docs/doc-management/hooks/index.ts b/src/frontend/apps/impress/src/features/docs/doc-management/hooks/index.ts index ba5d9640d7..eb2fc20ea6 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-management/hooks/index.ts +++ b/src/frontend/apps/impress/src/features/docs/doc-management/hooks/index.ts @@ -2,5 +2,4 @@ export * from './useCopyDocLink'; export * from './useCreateChildDocTree'; export * from './useDocTitleUpdate'; export * from './useDocUtils'; -export * from './useIsCollaborativeEditable'; export * from './useTrans'; diff --git a/src/frontend/apps/impress/src/features/docs/doc-management/hooks/useIsCollaborativeEditable.tsx b/src/frontend/apps/impress/src/features/docs/doc-management/hooks/useIsCollaborativeEditable.tsx deleted file mode 100644 index d2d2f172af..0000000000 --- a/src/frontend/apps/impress/src/features/docs/doc-management/hooks/useIsCollaborativeEditable.tsx +++ /dev/null @@ -1,80 +0,0 @@ -import { useEffect, useRef, useState } from 'react'; - -import { useConfig } from '@/core'; -import { useIsOffline } from '@/features/service-worker'; - -import { KEY_CAN_EDIT, useDocCanEdit } from '../api/useDocCanEdit'; -import { useProviderStore } from '../stores'; -import { Doc, LinkReach, LinkRole } from '../types'; - -export const useIsCollaborativeEditable = (doc: Doc) => { - const { isConnected } = useProviderStore(); - const { data: conf } = useConfig(); - - const docIsPublic = - doc.computed_link_reach === LinkReach.PUBLIC && - doc.computed_link_role === LinkRole.EDITOR; - const docIsAuth = - doc.computed_link_reach === LinkReach.AUTHENTICATED && - doc.computed_link_role === LinkRole.EDITOR; - const docHasMember = - doc.nb_accesses_direct > 1 || doc.nb_accesses_ancestors > 1; - const isUserReader = !doc.abilities.partial_update; - const isShared = docIsPublic || docIsAuth || docHasMember; - const { isOffline } = useIsOffline(); - const _isEditable = isUserReader || isConnected || !isShared || isOffline; - const [isEditable, setIsEditable] = useState(true); - const [isLoading, setIsLoading] = useState(!_isEditable); - const timeout = useRef(null); - const { data: editingRight, isLoading: isLoadingCanEdit } = useDocCanEdit( - doc.id, - { - enabled: !_isEditable, - queryKey: [KEY_CAN_EDIT, doc.id], - staleTime: 0, - }, - ); - - useEffect(() => { - if (isLoadingCanEdit || _isEditable || !editingRight) { - return; - } - - // Connection to the WebSocket can take some time, so we set a timeout to ensure the loading state is cleared after a reasonable time. - timeout.current = setTimeout(() => { - setIsEditable(editingRight.can_edit); - setIsLoading(false); - }, 1500); - - return () => { - if (timeout.current) { - clearTimeout(timeout.current); - } - }; - }, [editingRight, isLoadingCanEdit, _isEditable]); - - useEffect(() => { - if (!_isEditable) { - return; - } - - if (timeout.current) { - clearTimeout(timeout.current); - } - - setIsEditable(true); - setIsLoading(false); - }, [_isEditable]); - - if (!conf?.COLLABORATION_WS_NOT_CONNECTED_READ_ONLY) { - return { - isEditable: true, - isLoading: false, - }; - } - - return { - isEditable, - isLoading, - }; -}; From c4440966ce73153ae33edd419eab1fb465820e0f Mon Sep 17 00:00:00 2001 From: Anthony LC Date: Wed, 5 Aug 2026 15:42:00 +0200 Subject: [PATCH 17/59] =?UTF-8?q?=F0=9F=94=A5(project)=20remove=20occurenc?= =?UTF-8?q?es=20of=20COLLABORATION=5FSERVER=5FSECRET?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit COLLABORATION_SERVER_SECRET is no longer used in the codebase, so we can remove it from the codebase. --- documentation/examples/helm/impress.values.yaml | 1 - documentation/installation/compose.md | 2 +- env.d/development/common | 1 - env.d/production.dist/yprovider | 1 - src/frontend/servers/y-provider/src/env.ts | 12 +----------- src/helm/env.d/dev/values.impress.yaml.gotmpl | 1 - src/helm/env.d/feature/values.impress.yaml.gotmpl | 1 - 7 files changed, 2 insertions(+), 17 deletions(-) diff --git a/documentation/examples/helm/impress.values.yaml b/documentation/examples/helm/impress.values.yaml index 05c8c2feda..a6d26dab53 100644 --- a/documentation/examples/helm/impress.values.yaml +++ b/documentation/examples/helm/impress.values.yaml @@ -134,7 +134,6 @@ yProvider: COLLABORATION_BACKEND_BASE_URL: https://docs.127.0.0.1.nip.io COLLABORATION_LOGGING: true COLLABORATION_SERVER_ORIGIN: https://docs.127.0.0.1.nip.io - COLLABORATION_SERVER_SECRET: my-secret ingress: enabled: true diff --git a/documentation/installation/compose.md b/documentation/installation/compose.md index 014f6e5a17..baf976e6ba 100644 --- a/documentation/installation/compose.md +++ b/documentation/installation/compose.md @@ -93,7 +93,7 @@ If you are using an external service, you need to set `REDIS_URL` environment va The Y provider service enables collaboration through websockets. -Generates a secure key for `Y_PROVIDER_API_KEY` and `COLLABORATION_SERVER_SECRET` in ``env.d/yprovider``. +Generates a secure key for `Y_PROVIDER_API_KEY` in ``env.d/yprovider``. ### Docs diff --git a/env.d/development/common b/env.d/development/common index 60072b5fdc..7a294177d3 100644 --- a/env.d/development/common +++ b/env.d/development/common @@ -79,7 +79,6 @@ USER_RECONCILIATION_FORM_URL=http://localhost:3000 # Collaboration COLLABORATION_BACKEND_BASE_URL=http://app-dev:8000 COLLABORATION_SERVER_ORIGIN=http://localhost:3000 -COLLABORATION_SERVER_SECRET=my-secret COLLABORATION_WS_URL=ws://localhost:3002/collaboration/ws/v1/docs COLLABORATION_WS_INACTIVITY_TIMEOUT=15 # Seconds diff --git a/env.d/production.dist/yprovider b/env.d/production.dist/yprovider index 5761d7ae21..401cf4016d 100644 --- a/env.d/production.dist/yprovider +++ b/env.d/production.dist/yprovider @@ -1,6 +1,5 @@ Y_PROVIDER_API_BASE_URL=http://${YPROVIDER_HOST}:4444/api/ Y_PROVIDER_API_KEY= -COLLABORATION_SERVER_SECRET= COLLABORATION_SERVER_ORIGIN=https://${DOCS_HOST} COLLABORATION_BACKEND_BASE_URL=https://${DOCS_HOST} COLLABORATION_LOGGING=true \ No newline at end of file diff --git a/src/frontend/servers/y-provider/src/env.ts b/src/frontend/servers/y-provider/src/env.ts index 564c72927c..7c607aad91 100644 --- a/src/frontend/servers/y-provider/src/env.ts +++ b/src/frontend/servers/y-provider/src/env.ts @@ -1,23 +1,13 @@ -import { readFileSync } from 'fs'; - export const COLLABORATION_BACKEND_BASE_URL = process.env.COLLABORATION_BACKEND_BASE_URL || 'http://app-dev:8000'; export const COLLABORATION_LOGGING = process.env.COLLABORATION_LOGGING || 'false'; export const COLLABORATION_SERVER_ORIGIN = process.env.COLLABORATION_SERVER_ORIGIN || 'http://localhost:3000'; -// TODO(yhub): unused since the yhub migration, mirrors the Django setting of -// the same name (see impress/settings.py). Kept until CollaborationService is -// reinstated. -export const COLLABORATION_SERVER_SECRET = process.env - .COLLABORATION_SERVER_SECRET_FILE - ? readFileSync(process.env.COLLABORATION_SERVER_SECRET_FILE, 'utf-8') - : process.env.COLLABORATION_SERVER_SECRET || 'secret-api-key'; export const CONVERSION_FILE_MAX_SIZE = process.env.CONVERSION_FILE_MAX_SIZE ? Number(process.env.CONVERSION_FILE_MAX_SIZE) : 20971520; // 20 MB default // JWKS of the Django backend, used to verify the JWT it signs when calling us. -export const JWKS_URL = - process.env.JWKS_URL || `${COLLABORATION_BACKEND_BASE_URL}/api/v1.0/jwks`; +export const JWKS_URL = `${COLLABORATION_BACKEND_BASE_URL}/api/v1.0/jwks`; export const PORT = Number(process.env.PORT || 4444); export const SENTRY_DSN = process.env.SENTRY_DSN || ''; diff --git a/src/helm/env.d/dev/values.impress.yaml.gotmpl b/src/helm/env.d/dev/values.impress.yaml.gotmpl index a35727a259..a4f06e27d9 100644 --- a/src/helm/env.d/dev/values.impress.yaml.gotmpl +++ b/src/helm/env.d/dev/values.impress.yaml.gotmpl @@ -175,7 +175,6 @@ yProvider: COLLABORATION_BACKEND_BASE_URL: https://docs.127.0.0.1.nip.io COLLABORATION_LOGGING: true COLLABORATION_SERVER_ORIGIN: https://docs.127.0.0.1.nip.io - COLLABORATION_SERVER_SECRET: my-secret NODE_EXTRA_CA_CERTS: /cert/cacert.pem # Extra volume mounts to manage our local custom CA and avoid to set ssl_verify: false diff --git a/src/helm/env.d/feature/values.impress.yaml.gotmpl b/src/helm/env.d/feature/values.impress.yaml.gotmpl index 884fee0931..2579b985db 100644 --- a/src/helm/env.d/feature/values.impress.yaml.gotmpl +++ b/src/helm/env.d/feature/values.impress.yaml.gotmpl @@ -153,7 +153,6 @@ yProvider: COLLABORATION_BACKEND_BASE_URL: https://{{ .Values.feature }}-docs.{{ .Values.domain }} COLLABORATION_LOGGING: true COLLABORATION_SERVER_ORIGIN: https://{{ .Values.feature }}-docs.{{ .Values.domain }} - COLLABORATION_SERVER_SECRET: my-secret NODE_OPTIONS: "--max-old-space-size=1024" docSpec: From ba8630bc5e74808d814c1230afe03bf85be98706 Mon Sep 17 00:00:00 2001 From: Anthony LC Date: Wed, 5 Aug 2026 15:43:40 +0200 Subject: [PATCH 18/59] =?UTF-8?q?=F0=9F=94=A5(frontend)=20remove=20content?= =?UTF-8?q?=20GET=20PATCH?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We remove the code related to the content GET and PATCH endpoints, as they are no longer used in the codebase. The yhub server will handle the content management directly, providing the content and managing the updates. This change simplifies the code and reduces the complexity of the frontend application. We will need to reimplement the saving mechanism in the service worker when we are offline. Let's wait that the service is fully developed on the yhub side before we implement this feature. --- .../e2e/__tests__/app-impress/utils-common.ts | 1 - .../impress/src/core/config/api/useConfig.tsx | 1 - .../doc-editor/components/BlockNoteEditor.tsx | 2 - .../hook/__tests__/useSaveDoc.test.tsx | 155 ------------------ .../features/docs/doc-editor/hook/index.ts | 1 - .../docs/doc-editor/hook/useCollaboration.tsx | 24 +-- .../docs/doc-editor/hook/useSaveDoc.tsx | 139 ---------------- .../docs/doc-management/api/useDocContent.tsx | 41 ----- .../api/useDocContentUpdate.tsx | 119 -------------- .../doc-management/api/useDuplicateDoc.tsx | 26 +-- .../components/ModalConfirmationVersion.tsx | 58 +++---- .../service-worker/plugins/ApiPlugin.ts | 119 +------------- .../service-worker/service-worker-api.ts | 36 ---- 13 files changed, 31 insertions(+), 691 deletions(-) delete mode 100644 src/frontend/apps/impress/src/features/docs/doc-editor/hook/__tests__/useSaveDoc.test.tsx delete mode 100644 src/frontend/apps/impress/src/features/docs/doc-editor/hook/useSaveDoc.tsx delete mode 100644 src/frontend/apps/impress/src/features/docs/doc-management/api/useDocContent.tsx delete mode 100644 src/frontend/apps/impress/src/features/docs/doc-management/api/useDocContentUpdate.tsx diff --git a/src/frontend/apps/e2e/__tests__/app-impress/utils-common.ts b/src/frontend/apps/e2e/__tests__/app-impress/utils-common.ts index dccb9e746b..56acf4b50b 100644 --- a/src/frontend/apps/e2e/__tests__/app-impress/utils-common.ts +++ b/src/frontend/apps/e2e/__tests__/app-impress/utils-common.ts @@ -20,7 +20,6 @@ export const CONFIG = { API_USERS_SEARCH_QUERY_MIN_LENGTH: 3, COLLABORATION_WS_INACTIVITY_TIMEOUT: 15, COLLABORATION_WS_URL: process.env.COLLABORATION_WS_URL, - COLLABORATION_WS_NOT_CONNECTED_READ_ONLY: true, CONVERSION_UPLOAD_ENABLED: true, CONVERSION_FILE_EXTENSIONS_ALLOWED: ['.docx', '.md'], CONVERSION_FILE_MAX_SIZE: 20971520, diff --git a/src/frontend/apps/impress/src/core/config/api/useConfig.tsx b/src/frontend/apps/impress/src/core/config/api/useConfig.tsx index b204e5a381..5ac1ee14b9 100644 --- a/src/frontend/apps/impress/src/core/config/api/useConfig.tsx +++ b/src/frontend/apps/impress/src/core/config/api/useConfig.tsx @@ -49,7 +49,6 @@ export interface ConfigResponse { AI_FEATURE_LEGACY_ENABLED?: boolean; API_USERS_SEARCH_QUERY_MIN_LENGTH?: number; COLLABORATION_WS_URL?: string; - COLLABORATION_WS_NOT_CONNECTED_READ_ONLY?: boolean; COLLABORATION_WS_INACTIVITY_TIMEOUT?: number | null; CONVERSION_FILE_EXTENSIONS_ALLOWED: string[]; CONVERSION_FILE_MAX_SIZE: number; diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteEditor.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteEditor.tsx index 894cf006b3..6f285ef843 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteEditor.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/components/BlockNoteEditor.tsx @@ -39,7 +39,6 @@ import { useAnalytics } from '@/libs/Analytics'; import { AI_FEATURE_FLAG, DEFAULT_LOCALE } from '../conf'; import { useHeadings, - useSaveDoc, useShortcuts, useUploadFile, useUploadStatus, @@ -93,7 +92,6 @@ export const BlockNoteEditor = ({ doc, provider }: BlockNoteEditorProps) => { const { setEditor } = useEditorStore(); const { themeTokens } = useCunninghamTheme(); const refEditorContainer = useRef(null); - useSaveDoc(doc.id, provider.doc); const { i18n, t } = useTranslation(); const langLocalesBN = diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/hook/__tests__/useSaveDoc.test.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/hook/__tests__/useSaveDoc.test.tsx deleted file mode 100644 index 8ed670d6b0..0000000000 --- a/src/frontend/apps/impress/src/features/docs/doc-editor/hook/__tests__/useSaveDoc.test.tsx +++ /dev/null @@ -1,155 +0,0 @@ -import { act, renderHook, waitFor } from '@testing-library/react'; -import fetchMock from 'fetch-mock'; -import { useRouter } from 'next/router'; -import { Mock, beforeEach, describe, expect, it, vi } from 'vitest'; -import * as Y from 'yjs'; - -import { AppWrapper } from '@/tests/utils'; - -import { useSaveDoc } from '../useSaveDoc'; - -vi.mock('next/router', () => ({ - useRouter: vi.fn(), -})); - -vi.mock('@/docs/doc-versioning', () => ({ - KEY_LIST_DOC_VERSIONS: 'test-key-list-doc-versions', -})); - -vi.mock('@/docs/doc-management', async () => ({ - useUpdateDoc: ( - await vi.importActual('@/docs/doc-management/api/useUpdateDoc') - ).useUpdateDoc, -})); - -describe('useSaveDoc', () => { - const mockRouterEvents = { - on: vi.fn(), - off: vi.fn(), - }; - - beforeEach(() => { - vi.clearAllMocks(); - fetchMock.hardReset(); - fetchMock.mockGlobal(); - - (useRouter as Mock).mockReturnValue({ - events: mockRouterEvents, - }); - }); - - it('should setup event listeners on mount', () => { - const yDoc = new Y.Doc(); - const docId = 'test-doc-id'; - - const addEventListenerSpy = vi.spyOn(window, 'addEventListener'); - - renderHook(() => useSaveDoc(docId, yDoc), { - wrapper: AppWrapper, - }); - - // Verify router event listeners are set up - expect(mockRouterEvents.on).toHaveBeenCalledWith( - 'routeChangeStart', - expect.any(Function), - ); - - // Verify window event listener is set up - expect(addEventListenerSpy).toHaveBeenCalledWith( - 'beforeunload', - expect.any(Function), - ); - - addEventListenerSpy.mockRestore(); - }); - - it('should save when there are local changes', async () => { - vi.useFakeTimers(); - const yDoc = new Y.Doc(); - const docId = self.crypto.randomUUID(); - - fetchMock.patch(`http://test.jest/api/v1.0/documents/${docId}/content/`, { - body: JSON.stringify({ - id: docId, - content: 'test-content', - }), - }); - - renderHook(() => useSaveDoc(docId, yDoc), { - wrapper: AppWrapper, - }); - - act(() => { - // Trigger a local update - yDoc.getMap('test').set('key', 'value'); - }); - - act(() => { - // Advance timers to trigger the save interval - vi.advanceTimersByTime(61000); - }); - - // Switch to real timers to allow the mutation promise to resolve - vi.useRealTimers(); - - await waitFor(() => { - expect(fetchMock.callHistory.lastCall()?.url).toBe( - `http://test.jest/api/v1.0/documents/${docId}/content/`, - ); - }); - }); - - it('should not save when there are no local changes', () => { - vi.useFakeTimers(); - const yDoc = new Y.Doc(); - const docId = 'test-doc-id'; - - fetchMock.patch( - 'http://test.jest/api/v1.0/documents/test-doc-id/content/', - { - body: JSON.stringify({ - id: 'test-doc-id', - content: 'test-content', - }), - }, - ); - - renderHook(() => useSaveDoc(docId, yDoc), { - wrapper: AppWrapper, - }); - - act(() => { - // Advance timers without triggering any local updates - vi.advanceTimersByTime(61000); - }); - - // Since there are no local changes, no API call should be made - expect(fetchMock.callHistory.calls().length).toBe(0); - - vi.useRealTimers(); - }); - - it('should cleanup event listeners on unmount', () => { - const yDoc = new Y.Doc(); - const docId = 'test-doc-id'; - const removeEventListenerSpy = vi.spyOn(window, 'removeEventListener'); - - const { unmount } = renderHook(() => useSaveDoc(docId, yDoc), { - wrapper: AppWrapper, - }); - - unmount(); - - // Verify router event listeners are cleaned up - expect(mockRouterEvents.off).toHaveBeenCalledWith( - 'routeChangeStart', - expect.any(Function), - ); - - // Verify window event listener is cleaned up - expect(removeEventListenerSpy).toHaveBeenCalledWith( - 'beforeunload', - expect.any(Function), - ); - }); -}); diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/hook/index.ts b/src/frontend/apps/impress/src/features/docs/doc-editor/hook/index.ts index 95a0804b22..f45183574d 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-editor/hook/index.ts +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/hook/index.ts @@ -1,4 +1,3 @@ export * from './useHeadings'; -export * from './useSaveDoc'; export * from './useShortcuts'; export * from './useUploadFile'; diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useCollaboration.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useCollaboration.tsx index 150b7c07bf..eb26a0cb98 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useCollaboration.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useCollaboration.tsx @@ -3,10 +3,6 @@ import { useEffect } from 'react'; import { useCollaborationUrl, useConfig } from '@/core/config'; import { KEY_DOC } from '@/docs/doc-management/api/useDoc'; -import { - KEY_DOC_CONTENT, - useDocContent, -} from '@/docs/doc-management/api/useDocContent'; import { useProviderStore } from '@/docs/doc-management/stores/useProviderStore'; import { useIsOffline } from '@/features/service-worker/hooks/useOffline'; import { useBroadcastStore } from '@/stores/useBroadcastStore'; @@ -33,13 +29,6 @@ export const useCollaboration = (room: string) => { resumeFromInactivity, } = useProviderStore(); const isOffline = useIsOffline((state) => state.isOffline); - const { data: docContent } = useDocContent( - { id: room }, - { - staleTime: 30000, // 30 seconds - We keep the data fresh as it is a highly collaborative page - queryKey: [KEY_DOC_CONTENT, { id: room }], - }, - ); /** * When offline, the WebSocket never connects so the provider would stay @@ -89,20 +78,13 @@ export const useCollaboration = (room: string) => { * Set the provider when the collaboration URL and the document content are available. */ useEffect(() => { - if (!room || !collaborationUrl || provider || docContent === undefined) { + if (!room || !collaborationUrl || provider) { return; } - const newProvider = createProvider(collaborationUrl, room, docContent); + const newProvider = createProvider(collaborationUrl, room); setBroadcastProvider(newProvider); - }, [ - provider, - collaborationUrl, - createProvider, - docContent, - room, - setBroadcastProvider, - ]); + }, [provider, collaborationUrl, createProvider, room, setBroadcastProvider]); /** * Destroy the provider when the component is unmounted diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useSaveDoc.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useSaveDoc.tsx deleted file mode 100644 index 2edd76424a..0000000000 --- a/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useSaveDoc.tsx +++ /dev/null @@ -1,139 +0,0 @@ -import { useRouter } from 'next/router'; -import { useCallback, useEffect, useRef, useState } from 'react'; -import { WebsocketProvider } from 'y-websocket'; -import * as Y from 'yjs'; - -import { useDocContentUpdate } from '@/docs/doc-management/api/useDocContentUpdate'; -import { useProviderStore } from '@/docs/doc-management/stores/useProviderStore'; -import { KEY_LIST_DOC_VERSIONS } from '@/docs/doc-versioning/api/useDocVersions'; -import { COMMENT_UPDATE_ORIGIN } from '@/features/docs/doc-comments/api/DocsThreadStore'; -import { useIsOffline } from '@/features/service-worker'; -import { toBase64 } from '@/utils/string'; -import { isFirefox } from '@/utils/userAgent'; - -const SAVE_INTERVAL = 60000; - -export const useSaveDoc = (docId: string, yDoc: Y.Doc) => { - /** - * isSynced is more reliable than isConnected in this cases - * because it indicates that the content is fully synchronised - * with the yjs server - */ - const { isSynced: isConnectedToCollabServer } = useProviderStore(); - - const { isOffline } = useIsOffline(); - const isSavingRef = useRef(false); - const { mutate: updateDocContent } = useDocContentUpdate({ - listInvalidQueries: [KEY_LIST_DOC_VERSIONS], - isOptimistic: isOffline, // Enable optimistic updates when offline, to update the cache immediately - onSuccess: () => { - isSavingRef.current = false; - setIsLocalChange(false); - }, - onError: () => { - isSavingRef.current = false; - }, - }); - const [isLocalChange, setIsLocalChange] = useState(false); - - /** - * Update initial doc when doc is updated by other users, - * so only the user typing will trigger the save. - * This is to avoid saving the same doc multiple time. - */ - useEffect(() => { - const onUpdate = ( - _uintArray: Uint8Array, - _pluginKey: string, - _updatedDoc: Y.Doc, - transaction: Y.Transaction, - ) => { - /** - * When the AI edit the doc transaction.local is false, - * so we check the transaction origin to know where - * the transaction comes from. - * "PluginKey" origin comes from the current user, but transaction.local is more reliable - * Updates from other users are applied by the collaboration server with - * the provider instance as origin, it seems quite reliable too. - * The AI origin seems to not be reliable enough, but by deduction if it's not local - * and not from other users, it has to be from the AI. - * - * TODO: see if we can get the local changes from the AI - */ - const isAIChange = - !transaction.local && - !(transaction.origin instanceof WebsocketProvider); - - /** - * notifySubscribers generate a transaction that can be - * interpreted as a local change. - * We intercept the update with this origin to - * avoid marking the change as local. - */ - if (transaction.origin === COMMENT_UPDATE_ORIGIN) { - return; - } - - setIsLocalChange(transaction.local || isAIChange); - }; - - yDoc.on('update', onUpdate); - - return () => { - yDoc.off('update', onUpdate); - }; - }, [yDoc]); - - const saveDoc = useCallback(() => { - if (!isLocalChange || isSavingRef.current) { - return false; - } - - isSavingRef.current = true; - updateDocContent({ - id: docId, - content: toBase64(Y.encodeStateAsUpdate(yDoc)), - websocket: isConnectedToCollabServer, - }); - - return true; - }, [isLocalChange, updateDocContent, docId, yDoc, isConnectedToCollabServer]); - - const router = useRouter(); - - useEffect(() => { - const onSave = (e?: Event) => { - const isSaving = saveDoc(); - - /** - * Firefox does not trigger the request every time the user leaves the page. - * Plus the request is not intercepted by the service worker. - * So we prevent the default behavior to have the popup asking the user - * if he wants to leave the page, by adding the popup, we let the time to the - * request to be sent, and intercepted by the service worker (for the offline part). - */ - if ( - isSaving && - typeof e !== 'undefined' && - e.preventDefault && - isFirefox() - ) { - e.preventDefault(); - } - }; - - // Save every minute - const timeout = setInterval(onSave, SAVE_INTERVAL); - // Save when the user leaves the page - addEventListener('beforeunload', onSave); - // Save when the user navigates to another page - router.events.on('routeChangeStart', onSave); - - return () => { - clearInterval(timeout); - - removeEventListener('beforeunload', onSave); - router.events.off('routeChangeStart', onSave); - }; - }, [router.events, saveDoc]); -}; diff --git a/src/frontend/apps/impress/src/features/docs/doc-management/api/useDocContent.tsx b/src/frontend/apps/impress/src/features/docs/doc-management/api/useDocContent.tsx deleted file mode 100644 index 8b9882a6e3..0000000000 --- a/src/frontend/apps/impress/src/features/docs/doc-management/api/useDocContent.tsx +++ /dev/null @@ -1,41 +0,0 @@ -import { UseQueryOptions, useQuery } from '@tanstack/react-query'; -import { validate as uuidValidate } from 'uuid'; - -import { APIError, errorCauses, fetchAPI } from '@/api'; - -export type DocContentParams = { - id: string; -}; - -export const getDocContent = async ({ - id, -}: DocContentParams): Promise => { - if (!uuidValidate(id)) { - throw new Error(`Invalid doc id in getDocContent: ${id}`); - } - - const response = await fetchAPI(`documents/${id}/content/`, { - headers: { - accept: 'text/plain,application/json', - }, - }); - - if (!response.ok) { - throw new APIError('Failed to get the doc', await errorCauses(response)); - } - - return response.text(); -}; - -export const KEY_DOC_CONTENT = 'doc-content'; - -export function useDocContent( - param: DocContentParams, - queryConfig?: UseQueryOptions, -) { - return useQuery({ - queryKey: queryConfig?.queryKey ?? [KEY_DOC_CONTENT, param], - queryFn: () => getDocContent(param), - ...queryConfig, - }); -} diff --git a/src/frontend/apps/impress/src/features/docs/doc-management/api/useDocContentUpdate.tsx b/src/frontend/apps/impress/src/features/docs/doc-management/api/useDocContentUpdate.tsx deleted file mode 100644 index 8f845a29ba..0000000000 --- a/src/frontend/apps/impress/src/features/docs/doc-management/api/useDocContentUpdate.tsx +++ /dev/null @@ -1,119 +0,0 @@ -import { - UseMutationOptions, - useMutation, - useQueryClient, -} from '@tanstack/react-query'; -import { validate as uuidValidate } from 'uuid'; - -import { APIError, errorCauses, fetchAPI } from '@/api'; - -import { Doc } from '../types'; - -import { KEY_DOC_CONTENT } from './useDocContent'; - -export interface UpdateDocContentParams { - id: Doc['id']; - content: string; // Base64 encoded content - websocket?: boolean; -} - -export const updateDocContent = async ({ - id, - content, - websocket, -}: UpdateDocContentParams): Promise => { - if (!uuidValidate(id)) { - throw new Error(`Invalid doc id in updateDocContent: ${id}`); - } - - const response = await fetchAPI(`documents/${id}/content/`, { - method: 'PATCH', - body: JSON.stringify({ - content, - websocket, - }), - }); - - if (!response.ok) { - throw new APIError( - 'Failed to update the doc content', - await errorCauses(response), - ); - } -}; - -type UseDocContentUpdate = UseMutationOptions< - void, - APIError, - UpdateDocContentParams -> & { - isOptimistic?: boolean; - listInvalidQueries?: string[]; -}; - -export function useDocContentUpdate(queryConfig?: UseDocContentUpdate) { - const queryClient = useQueryClient(); - return useMutation({ - mutationFn: updateDocContent, - ...queryConfig, - onMutate: (variables) => { - /** - * If optimistic, we update the content cache immediately with the new content - * It is useful when we are in offline mode because the onSuccess is not always triggered. - */ - if (queryConfig?.isOptimistic) { - const previousContent = queryClient.getQueryData([ - KEY_DOC_CONTENT, - { id: variables.id }, - ]); - - queryClient.setQueryData( - [KEY_DOC_CONTENT, { id: variables.id }], - variables.content, - ); - - return { previousContent }; - } - }, - onSuccess: (data, variables, onMutateResult, context) => { - if (!queryConfig?.isOptimistic) { - /** - * If not optimistic, we need to update the content cache with the new content returned - * from the server - */ - queryClient.setQueryData( - [KEY_DOC_CONTENT, { id: variables.id }], - variables.content, - ); - } - - queryConfig?.listInvalidQueries?.forEach((queryKey) => { - void queryClient.resetQueries({ - queryKey: [queryKey], - }); - }); - - if (queryConfig?.onSuccess) { - void queryConfig.onSuccess(data, variables, onMutateResult, context); - } - }, - onError: (error, variables, onMutateResult, context) => { - if ( - queryConfig?.isOptimistic && - (onMutateResult as { previousContent: unknown })?.previousContent - ) { - const previousContent = (onMutateResult as { previousContent: unknown }) - .previousContent; - - queryClient.setQueryData( - [KEY_DOC_CONTENT, { id: variables.id }], - previousContent, - ); - } - - if (queryConfig?.onError) { - queryConfig.onError(error, variables, onMutateResult, context); - } - }, - }); -} diff --git a/src/frontend/apps/impress/src/features/docs/doc-management/api/useDuplicateDoc.tsx b/src/frontend/apps/impress/src/features/docs/doc-management/api/useDuplicateDoc.tsx index 10f6165467..0b247c20e0 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-management/api/useDuplicateDoc.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-management/api/useDuplicateDoc.tsx @@ -8,16 +8,11 @@ import { useQueryClient, } from '@tanstack/react-query'; import { useTranslation } from 'react-i18next'; -import * as Y from 'yjs'; import { APIError, errorCauses, fetchAPI } from '@/api'; -import { KEY_LIST_DOC_VERSIONS } from '@/docs/doc-versioning/api/useDocVersions'; -import { toBase64 } from '@/utils/string'; -import { useProviderStore } from '../stores'; import { Doc } from '../types'; -import { useDocContentUpdate } from './useDocContentUpdate'; import { KEY_LIST_DOC } from './useDocs'; interface DuplicateDocPayload { @@ -60,27 +55,10 @@ export function useDuplicateDoc(options?: DuplicateDocOptions) { const queryClient = useQueryClient(); const { toast } = useToastProvider(); const { t } = useTranslation(); - const { provider } = useProviderStore(); - - const { mutateAsync: updateDocContent } = useDocContentUpdate({ - listInvalidQueries: [KEY_LIST_DOC_VERSIONS], - }); return useMutation({ - mutationFn: async (variables) => { - // Save the document if we can first, to ensure the latest state is duplicated - const canSave = - variables.canSave && provider && provider.doc.guid === variables.docId; - - if (canSave) { - await updateDocContent({ - id: variables.docId, - content: toBase64(Y.encodeStateAsUpdate(provider.doc)), - }); - } - - return await duplicateDoc(variables); - }, + // TODO(yhub): double check the saving is made correctly from the back so + mutationFn: duplicateDoc, onSuccess: (data, variables, onMutateResult, context) => { void queryClient.resetQueries({ queryKey: [KEY_LIST_DOC], diff --git a/src/frontend/apps/impress/src/features/docs/doc-versioning/components/ModalConfirmationVersion.tsx b/src/frontend/apps/impress/src/features/docs/doc-versioning/components/ModalConfirmationVersion.tsx index 73b2fb9745..f511f841f4 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-versioning/components/ModalConfirmationVersion.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-versioning/components/ModalConfirmationVersion.tsx @@ -1,22 +1,12 @@ -import { - Button, - Modal, - ModalSize, - VariantType, - useToastProvider, -} from '@gouvfr-lasuite/cunningham-react'; +import { Button, Modal, ModalSize } from '@gouvfr-lasuite/cunningham-react'; import { useTranslation } from 'react-i18next'; import { createGlobalStyle } from 'styled-components'; import { Box, Text } from '@/components'; -import { useThreadStore } from '@/docs/doc-comments/stores/useThreadStore'; -import { Doc, base64ToYDoc, useProviderStore } from '@/docs/doc-management/'; -import { useDocContentUpdate } from '@/docs/doc-management/api/useDocContentUpdate'; +import { Doc } from '@/docs/doc-management/'; import { useDocVersion } from '../api'; -import { KEY_LIST_DOC_VERSIONS } from '../api/useDocVersions'; import { Versions } from '../types'; -import { revertUpdate } from '../utils'; const ModalStyle = createGlobalStyle` .c__modal__title { @@ -33,7 +23,7 @@ interface ModalConfirmationVersionProps { export const ModalConfirmationVersion = ({ onClose, - onSuccess, + onSuccess: __onSuccess, docId, versionId, }: ModalConfirmationVersionProps) => { @@ -42,29 +32,28 @@ export const ModalConfirmationVersion = ({ versionId, }); const { t } = useTranslation(); - const { toast } = useToastProvider(); - const { provider } = useProviderStore(); - const { threadStore } = useThreadStore(); - const { mutate: updateDocContent } = useDocContentUpdate({ - listInvalidQueries: [KEY_LIST_DOC_VERSIONS], - onSuccess: () => { - const onDisplaySuccess = () => { - toast(t('Version restored successfully'), VariantType.SUCCESS); - onSuccess(); - }; - if (!provider || !version?.content) { - onDisplaySuccess(); - return; - } + // TODO(yhub) : Revert the doc to a previous state using Y.js / Yhub + // const { mutate: updateDocContent } = useDocContentUpdate({ + // listInvalidQueries: [KEY_LIST_DOC_VERSIONS], + // onSuccess: () => { + // const onDisplaySuccess = () => { + // toast(t('Version restored successfully'), VariantType.SUCCESS); + // onSuccess(); + // }; - revertUpdate(provider.doc, provider.doc, base64ToYDoc(version.content)); + // if (!provider || !version?.content) { + // onDisplaySuccess(); + // return; + // } - threadStore?.refreshThreads(); + // revertUpdate(provider.doc, provider.doc, base64ToYDoc(version.content)); - onDisplaySuccess(); - }, - }); + // threadStore?.refreshThreads(); + + // onDisplaySuccess(); + // }, + // }); if (!version) { return null; @@ -96,11 +85,6 @@ export const ModalConfirmationVersion = ({ return; } - updateDocContent({ - id: docId, - content: version.content, - }); - onClose(); }} > diff --git a/src/frontend/apps/impress/src/features/service-worker/plugins/ApiPlugin.ts b/src/frontend/apps/impress/src/features/service-worker/plugins/ApiPlugin.ts index e381acddc3..500e144efd 100644 --- a/src/frontend/apps/impress/src/features/service-worker/plugins/ApiPlugin.ts +++ b/src/frontend/apps/impress/src/features/service-worker/plugins/ApiPlugin.ts @@ -2,19 +2,20 @@ import { WorkboxPlugin } from 'workbox-core'; import { Doc, DocsResponse } from '@/docs/doc-management'; import { LinkReach, LinkRole, Role } from '@/docs/doc-management/types'; -import { UpdateDocContentParams } from '@/features/docs/doc-management/api/useDocContentUpdate'; import { DBRequest, DocsDB } from '../DocsDB'; import { RequestSerializer } from '../RequestSerializer'; import { SyncManager } from '../SyncManager'; interface OptionsReadonly { - tableName: 'doc-list' | 'doc-item' | 'doc-content'; - type: 'list' | 'item' | 'content'; + tableName: 'doc-list' | 'doc-item'; + type: 'list' | 'item'; } +// TODO(yhub): Used to work offline, we need to implement the patch mechanism +// It will be probably linked to the HTTP fallback mechanism of yhub interface OptionsMutate { - type: 'update' | 'delete' | 'create' | 'content-update'; + type: 'update' | 'delete' | 'create'; } interface OptionsSync { @@ -53,27 +54,6 @@ export class ApiPlugin implements WorkboxPlugin { response, }) => { try { - // For content requests, a 304 means the document hasn't changed: - // transparently serve the cached version from IDB. - if (this.options.type === 'content' && response.status === 304) { - const db = await DocsDB.open(); - const entry = await db.get('doc-content', request.url); - db.close(); - if (entry) { - return new Response(entry.content, { - status: 200, - statusText: 'OK', - headers: { - 'Content-Type': 'text/plain', - ...(entry.etag && { ETag: entry.etag }), - ...(entry.lastModified && { - 'Last-Modified': entry.lastModified, - }), - }, - }); - } - } - if (response.status !== 200) { return response; } @@ -82,17 +62,6 @@ export class ApiPlugin implements WorkboxPlugin { const tableName = this.options.tableName; const body = (await response.clone().json()) as DocsResponse | Doc; await DocsDB.cacheResponse(request.url, body, tableName); - } else if (this.options.type === 'content') { - // Cache the content response with its ETag / Last-Modified to be - // able to use it for conditional requests and offline access. - const content = await response.clone().text(); - const etag = response.headers.get('ETag') ?? ''; - const lastModified = response.headers.get('Last-Modified') ?? ''; - await DocsDB.cacheResponse( - request.url, - { etag, lastModified, content }, - 'doc-content', - ); } else if (this.options.type === 'update') { const db = await DocsDB.open(); const storedResponse = await db.get('doc-item', request.url); @@ -135,7 +104,6 @@ export class ApiPlugin implements WorkboxPlugin { requestWillFetch: WorkboxPlugin['requestWillFetch'] = async ({ request }) => { if ( this.options.type === 'update' || - this.options.type === 'content-update' || this.options.type === 'create' || this.options.type === 'delete' ) { @@ -144,27 +112,6 @@ export class ApiPlugin implements WorkboxPlugin { await this.options.syncManager.sync(); - // For content requests, add If-None-Match / If-Modified-Since from IDB - // so the backend can return a 304 when the document hasn't changed. - if (this.options.type === 'content') { - try { - const db = await DocsDB.open(); - const entry = await db.get('doc-content', request.url); - db.close(); - if (entry?.etag || entry?.lastModified) { - const headers = new Headers(request.headers); - if (entry.etag) { - headers.set('If-None-Match', entry.etag); - } else { - headers.set('If-Modified-Since', entry.lastModified); - } - return new Request(request, { headers }); - } - } catch (error) { - console.error('SW: ApiPlugin requestWillFetch content error', error); - } - } - return Promise.resolve(request); }; @@ -188,13 +135,9 @@ export class ApiPlugin implements WorkboxPlugin { return this.handlerDidErrorDelete(request); case 'update': return this.handlerDidErrorUpdate(request); - case 'content-update': - return this.handlerDidErrorContentUpdate(request); case 'list': case 'item': return this.handlerDidErrorRead(this.options.tableName, request.url); - case 'content': - return this.handlerDidErrorContent(request); } return Promise.resolve(ApiPlugin.getApiCatchHandler()); @@ -492,56 +435,4 @@ export class ApiPlugin implements WorkboxPlugin { }, }); }; - - private handlerDidErrorContent = async (request: Request) => { - const db = await DocsDB.open(); - const entry = await db.get('doc-content', request.url); - db.close(); - - if (!entry) { - return Promise.resolve(ApiPlugin.getApiCatchHandler()); - } - - return new Response(entry.content, { - status: 200, - statusText: 'OK', - headers: { - 'Content-Type': 'text/plain', - ...(entry.etag && { ETag: entry.etag }), - ...(entry.lastModified && { 'Last-Modified': entry.lastModified }), - }, - }); - }; - - /** - * When the content update fails, we save the new content in the cache, and we will sync it later with the SyncManager. - * We return a 204 to the client to say that the update is successful, and we update the content in the cache so the - * client can see the new content while offline. - */ - private handlerDidErrorContentUpdate = async (request: Request) => { - const db = await DocsDB.open(); - const entry = await db.get('doc-content', request.url); - db.close(); - - if (!entry || !this.initialRequest) { - return new Response('Not found', { status: 404 }); - } - - await this.queueMutation(this.initialRequest); - - const bodyMutate = (await this.initialRequest - .clone() - .json()) as Partial; - const newContent = bodyMutate.content ?? entry.content; - await DocsDB.cacheResponse( - request.url, - { etag: '', lastModified: '', content: newContent }, - 'doc-content', - ); - - return new Response(null, { - status: 204, - statusText: 'No Content', - }); - }; } diff --git a/src/frontend/apps/impress/src/features/service-worker/service-worker-api.ts b/src/frontend/apps/impress/src/features/service-worker/service-worker-api.ts index 1de19733e1..80c8be8b66 100644 --- a/src/frontend/apps/impress/src/features/service-worker/service-worker-api.ts +++ b/src/frontend/apps/impress/src/features/service-worker/service-worker-api.ts @@ -62,42 +62,6 @@ registerRoute( 'GET', ); -registerRoute( - ({ url }) => - isApiUrl(url.href) && /\/documents\/[a-z0-9-]+\/content\/$/.test(url.href), - new NetworkOnly({ - plugins: [ - new ApiPlugin({ - tableName: 'doc-content', - type: 'content', - syncManager, - }), - new OfflinePlugin(), - ], - }), - 'GET', -); - -/** - * Mutate routes for the content update - * It will save in cache the request if the content update fails, and will retry - * to sync it later with the SyncManager - */ -registerRoute( - ({ url }) => - isApiUrl(url.href) && /\/documents\/[a-z0-9-]+\/content\/$/.test(url.href), - new NetworkOnly({ - plugins: [ - new ApiPlugin({ - type: 'content-update', - syncManager, - }), - new OfflinePlugin(), - ], - }), - 'PATCH', -); - /** * Mutate routes for the document update * It will save in cache the request if the document update fails, and will retry From a8ebe57671f32fa5aa688c2cc7b403474c481d54 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Wed, 5 Aug 2026 16:58:02 +0200 Subject: [PATCH 19/59] =?UTF-8?q?=F0=9F=99=88(dev)=20ignore=20playwright-m?= =?UTF-8?q?cp=20browser=20artifacts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Playwright MCP browser session writes snapshots and console logs into .playwright-mcp/ at the repository root while driving the app during development — keep them out of version control. Signed-off-by: Kevin Jahns --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 08dfcc4b13..631e0bf094 100644 --- a/.gitignore +++ b/.gitignore @@ -93,3 +93,4 @@ AGENTS.md .aider* .copilot/ .github/copilot-instructions.md +.playwright-mcp From fe8165dfb090368655f21c3b2fbb16ba1101d3f2 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Wed, 5 Aug 2026 16:58:14 +0200 Subject: [PATCH 20/59] =?UTF-8?q?=E2=9C=A8(collaboration)=20soft-migrate?= =?UTF-8?q?=20legacy=20S3=20documents=20into=20yhub?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With SOFT_MIGRATION=true, the first access to a document yhub does not know yet fetches the legacy snapshot from Django's S3 media bucket ({id}/file, UTF-8 base64 of a raw Yjs update), seeds the room through the compute pool - attributed to "system" with a migration=s3 custom attribution - and only then admits the connection, so the initial sync always includes the seed. Now that the frontend no longer bootstraps rooms client-side (content GET/PATCH removal), this is the only path that brings legacy content into yhub; keep the flag on until a batch backfill has migrated the full corpus. A missing S3 object is the brand-new-document case and yields an empty room; every real failure fails closed (opaque 401, y-websocket retries with backoff). Existence is probed postgres-first (bare SELECT, then the valkey stream, then the SELECT again to close the compaction race). Guard rails: a per-docid verdict cache (poison objects cannot sustain an S3 retry storm, transient errors expire in 15s, per-replica seed backpressure denies once without caching), in-flight dedup, a token-owned cross-replica valkey lock released by compare-and-delete, a 10s S3 fetch timeout that also destroys a late-arriving response stream, and the same 10MiB decoded cap as create-ydoc. Concurrent seeds stay correct regardless: the frozen snapshots share one Yjs lineage, so duplicates merge as CRDT no-ops. Also reject non-lowercase docids (Django serializes UUIDs lowercase; a case variant would open a parallel room and miss its S3 object) and refuse to boot when AWS_S3_ENDPOINT_URL carries a path the minio client cannot address. On AWS the read-only credentials must include s3:ListBucket so a missing object surfaces as NoSuchKey rather than AccessDenied - see the README for the full guarantees and ops notes. Signed-off-by: Kevin Jahns --- CHANGELOG.md | 7 + compose.yml | 7 + src/yhub-server/README.md | 82 +++++++ src/yhub-server/package-lock.json | 3 +- src/yhub-server/package.json | 3 +- src/yhub-server/server.js | 366 +++++++++++++++++++++++++++++- 6 files changed, 454 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee04d3c86e..00a6d38376 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,13 @@ and this project adheres to ### Added +- ✨(collaboration) soft-migrate legacy S3 documents into yhub on first access + (`SOFT_MIGRATION=true`): when yhub does not know a document yet, its legacy + snapshot (`{id}/file`, base64 Yjs update) is fetched from the Django S3 + media bucket and seeded server-side (attributed to `system`) before the + connection is admitted. A missing S3 object means a brand-new document and + yields an empty room; any real S3/compute failure fails closed (opaque 401, + the client retries with backoff). Enabled in the dev stack via compose.yml - ✨(collaboration) add a create-ydoc endpoint on yhub: `POST /collaboration/create-ydoc/v1/docs/{id}` seeds a document's initial Yjs state from a raw binary update posted as `application/octet-stream`, diff --git a/compose.yml b/compose.yml index 7f7b95f8b6..02bf221bc3 100644 --- a/compose.yml +++ b/compose.yml @@ -243,6 +243,9 @@ services: REDIS: redis://yhub-valkey:6379 POSTGRES: postgres://yhub:yhub@yhub-postgres:5432/yhub REDIS_PREFIX: yhub + # seed rooms from the legacy Django/S3 document store on first access — + # S3 endpoint/credentials come from env.d/development/common + SOFT_MIGRATION: "true" env_file: - env.d/development/common - env.d/development/common.local @@ -254,6 +257,10 @@ services: condition: service_healthy yhub-postgres: condition: service_healthy + # soft migration reads the legacy document store at startup traffic — + # starting before minio would cache 401s for the first accessed docs + minio: + condition: service_healthy kc_postgresql: image: postgres:14.3 diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index cc907d9ea4..d6e04670f1 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -42,6 +42,88 @@ not be routed through the public ingress. The `Dockerfile` builds the container image used by the `yhub` service in `compose.yml`. +## Soft migration (`SOFT_MIGRATION=true`) + +Documents were historically stored by the Django backend in the S3 media +bucket, as UTF-8 text that is the base64 encoding of a raw Yjs update, at key +`{document-uuid}/file`. With `SOFT_MIGRATION=true`, this server migrates those +documents into yhub lazily, on first access: + +1. After a user's document authorization succeeds, the auth plugin checks + whether yhub already has content for the room — a bare postgres `SELECT` + (persisted rows), then the valkey stream (uncompacted `ydoc:update:v1` + messages), then the `SELECT` again to close the compaction race. Verdicts + are cached in-process (existing docs 10 min, empty docs 60 s, failures + 5 min). +2. If the room is unknown, the legacy object is fetched from S3 (10 s + timeout, 10 MiB decoded cap — the same limit as `create-ydoc`), decoded, + diffed through yhub's compute pool and appended to the room's stream — + attributed to the `system` identity with a `migration=s3` custom + attribution. This completes before the websocket upgrade resolves, so the + initial sync always includes the seeded content. First access to an + unmigrated document is therefore slower by one S3 round-trip plus one + compute pass. +3. Concurrent first-connections are collapsed: an in-process in-flight map, a + per-room valkey lock (`{prefix}:softmigrate:*`, 30 s TTL), and a cap of 4 + concurrent seeds per replica (excess connections fail fast and retry). + +Guarantees and failure behavior: + +- **Missing S3 object is not an error** — that is the brand-new-document case + (Django writes no object until the first content save); the room simply + starts empty. +- **Everything else fails closed**: network/auth errors, timeouts, oversized + objects and corrupt updates deny the connection (an opaque 401 the client + retries with backoff) and log a `soft-migration` error. A cached failure + verdict prevents retry storms from hammering S3 — permanent failures + (corrupt/oversized objects) for 5 minutes, transient ones (network errors, + timeouts) for 15 seconds, and per-replica seed backpressure (more than 4 + concurrent seeds) is denied without caching so the client's next retry goes + through. +- **Seeding is idempotent**: the legacy S3 snapshots are frozen (the + frontend no longer PATCHes content snapshots to Django) and share one Yjs + lineage with everything in yhub, so duplicate or concurrent seeds merge as + CRDT no-ops. Losing valkey before compaction merely makes the next access + re-seed from S3. Note that edits made *after* a document was migrated live + only in yhub — a re-seed after total yhub data loss restores the + pre-migration snapshot, nothing newer. +- **`SOFT_MIGRATION=false` does not undo anything** — migrated documents + stay correct in yhub — but since the frontend's client-side seeding was + removed along with the content GET/PATCH endpoints, an unmigrated legacy + document then opens as an *empty* room. Keep the flag on until a backfill + has migrated the full corpus. + +Configuration: `AWS_S3_ENDPOINT_URL`, `AWS_S3_ACCESS_KEY_ID`, +`AWS_S3_SECRET_ACCESS_KEY` (both with `*_FILE` indirection), optional +`AWS_S3_REGION_NAME`, and `AWS_STORAGE_BUCKET_NAME` (defaults to Django's dev +default `impress-media-storage`; production uses a different bucket name and +must set it explicitly). The server refuses to boot when the flag is set +without endpoint and credentials. In development the values arrive via +`env.d/development/common`. + +Operational notes: + +- Use **read-only, bucket-scoped S3 credentials** in production — never the + backend's read-write keys; this process terminates untrusted traffic. On + AWS the credentials must include `s3:ListBucket` on the bucket in addition + to `s3:GetObject`: without it, S3 reports a missing object as + `403 AccessDenied` instead of `404 NoSuchKey`, and every brand-new document + would fail closed instead of starting empty. +- `AWS_S3_ENDPOINT_URL` must not contain a path (the minio client cannot + address a base path); the server refuses to boot otherwise. +- After manually wiping a room's yhub state (postgres row + stream key), + **restart yhub** so the in-process verdict cache cannot serve a stale + "exists" and suppress the re-seed. +- Lazy migration never finishes on its own: documents that are never opened + stay in S3 forever, and they are only reachable through this flag now that + the frontend's client-side seeding is gone. A batch backfill (Django + posting each document's **exact** S3 bytes to `create-ydoc` with the admin + JWT, treating 409 as success) is the intended completion path and composes + safely with concurrent first-accesses — as long as content is never + re-converted: independently generated updates for the same document would + duplicate its content on merge, while re-posting the stored bytes is a + no-op. Only after the backfill may `SOFT_MIGRATION` be turned off. + ## ⚠️ License warning (AGPL) This directory depends on `@y/hub`, which is licensed under the diff --git a/src/yhub-server/package-lock.json b/src/yhub-server/package-lock.json index 9808e2ab1d..814f8e8a02 100644 --- a/src/yhub-server/package-lock.json +++ b/src/yhub-server/package-lock.json @@ -7,7 +7,8 @@ "name": "yhub-server", "dependencies": { "@y/hub": "0.4.0", - "jose": "6.2.8" + "jose": "6.2.8", + "minio": "8.0.7" }, "engines": { "node": ">=22" diff --git a/src/yhub-server/package.json b/src/yhub-server/package.json index f4ec5cb513..e0129e4904 100644 --- a/src/yhub-server/package.json +++ b/src/yhub-server/package.json @@ -7,7 +7,8 @@ }, "dependencies": { "@y/hub": "0.4.0", - "jose": "6.2.8" + "jose": "6.2.8", + "minio": "8.0.7" }, "engines": { "node": ">=22" diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index 4f7f303f2f..79d2cca023 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -1,8 +1,9 @@ -import { createHash } from 'node:crypto'; +import { createHash, randomUUID } from 'node:crypto'; import { readFileSync } from 'node:fs'; -import { createApiEndpoint, createAuthPlugin, createYHub } from '@y/hub'; +import { createApiEndpoint, createAuthPlugin, createYHub, logger } from '@y/hub'; import { createRemoteJWKSet, jwtVerify } from 'jose'; +import { Client as S3Client } from 'minio'; // mirror y-provider's env.ts secret-file support const secret = (name, dflt) => @@ -21,8 +22,12 @@ const allowedOrigins = ( ).split(','); const Y_PROVIDER_API_KEY = secret('Y_PROVIDER_API_KEY', 'yprovider-api-key'); const ORG = process.env.YHUB_ORG || 'docs'; +// lowercase only (no /i): Django serializes UUIDs lowercase, while yhub rooms +// and S3 keys are case-sensitive strings — accepting case variants would let a +// client open a parallel room for the same document (and, with soft migration, +// miss its S3 object and fork the document's lineage) const UUID4 = - /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; // an empty Yjs update (what `Y.encodeStateAsUpdate(new Y.Doc())` encodes to) — // hardcoded so we don't import @y/y for two bytes const EMPTY_YDOC = new Uint8Array([0, 0]); @@ -33,6 +38,61 @@ const EMPTY_YDOC = new Uint8Array([0, 0]); // websocket path. const MAX_CREATE_BYTES = 10 * 1024 * 1024; +// Soft migration (see README.md): legacy documents live in Django's S3 media +// bucket as UTF-8 base64 of a raw Yjs update at key `{docid}/file`. With +// SOFT_MIGRATION=true, the first access to a room yhub does not know yet +// fetches that object, seeds the room with it (attributed to 'system'), and +// only then admits the connection. +const SOFT_MIGRATION = process.env.SOFT_MIGRATION === 'true'; +const AWS_S3_ENDPOINT_URL = process.env.AWS_S3_ENDPOINT_URL; +const AWS_S3_ACCESS_KEY_ID = secret('AWS_S3_ACCESS_KEY_ID'); +const AWS_S3_SECRET_ACCESS_KEY = secret('AWS_S3_SECRET_ACCESS_KEY'); +const AWS_S3_REGION_NAME = process.env.AWS_S3_REGION_NAME; +// Django's default bucket name (impress settings.py) — prod overrides it +const AWS_STORAGE_BUCKET_NAME = + process.env.AWS_STORAGE_BUCKET_NAME || 'impress-media-storage'; +// base64 inflates 3 bytes to 4 — cap the streamed read at the encoded size of +// MAX_CREATE_BYTES (the same effective limit as create-ydoc) plus padding slack +const MAX_LEGACY_B64_BYTES = Math.ceil(MAX_CREATE_BYTES / 3) * 4 + 1024; +const S3_FETCH_TIMEOUT_MS = 10000; +const MIGRATE_LOCK_TTL_MS = 30000; +const MAX_CONCURRENT_SEEDS = 4; + +if ( + SOFT_MIGRATION && + (!AWS_S3_ENDPOINT_URL || !AWS_S3_ACCESS_KEY_ID || !AWS_S3_SECRET_ACCESS_KEY) +) { + // fail at boot instead of as an opaque 401 storm on first connect + throw new Error( + 'SOFT_MIGRATION=true requires AWS_S3_ENDPOINT_URL, AWS_S3_ACCESS_KEY_ID and AWS_S3_SECRET_ACCESS_KEY', + ); +} +const s3 = SOFT_MIGRATION + ? (() => { + const url = new URL(AWS_S3_ENDPOINT_URL); + if (url.pathname !== '/' && url.pathname !== '') { + // boto3 accepts path-prefixed endpoints but the minio client cannot + // address a base path — dropping it silently would probe the wrong + // keys and "migrate" every doc as empty + throw new Error('AWS_S3_ENDPOINT_URL must not contain a path'); + } + return new S3Client({ + endPoint: url.hostname, + port: + url.port !== '' + ? Number(url.port) + : url.protocol === 'https:' + ? 443 + : 80, + useSSL: url.protocol === 'https:', + accessKey: AWS_S3_ACCESS_KEY_ID, + secretKey: AWS_S3_SECRET_ACCESS_KEY, + ...(AWS_S3_REGION_NAME ? { region: AWS_S3_REGION_NAME } : {}), + }); + })() + : null; +const migrationLog = logger.child({ module: 'soft-migration' }); + // Public keys verifying the RS256 admin tokens Django issues (JWTService). // Lazily fetched on first use; jose caches the keys and refetches on unknown // "kid", so Django can rotate the signing key without a yhub restart. @@ -56,6 +116,273 @@ const backendFetch = async (path, { cookie, origin }) => { return res.json(); }; +// Legacy Django document store: object `{docid}/file`, body = UTF-8 text that +// is the base64 encoding of a raw Yjs update. Returns null when the object +// does not exist — a document that never had content saved, e.g. brand new. +// Throws on any other failure (network, auth, timeout, oversize); corrupt +// base64 decodes leniently to garbage that patchYdoc later rejects. +const fetchLegacyDoc = async (docid) => { + let stream = null; + let cancelTimeout = () => {}; + // minio 8 takes no AbortSignal — race a timer that also destroys the body + // stream once reading, so a stalled transfer cannot hold the ws upgrade + const timeout = new Promise((_, reject) => { + const timer = setTimeout(() => { + const err = new Error( + `s3 fetch timed out after ${S3_FETCH_TIMEOUT_MS}ms`, + ); + err.transient = true; // a slow S3 may recover — cache the failure briefly + stream?.destroy(err); + reject(err); + }, S3_FETCH_TIMEOUT_MS); + cancelTimeout = () => clearTimeout(timer); + }); + try { + let objPromise; + try { + objPromise = s3.getObject(AWS_STORAGE_BUCKET_NAME, `${docid}/file`); + stream = await Promise.race([objPromise, timeout]); + } catch (err) { + if (err?.code === 'NoSuchKey') return null; + // if the timeout won the race, getObject may still resolve later — + // destroy the late-arriving response stream, otherwise its never-read + // socket leaks (minio 8 sets no request timeout and cannot abort) + objPromise?.then((s) => s.destroy(err), () => {}); + throw err; + } + const body = await Promise.race([ + new Promise((resolve, reject) => { + const chunks = []; + let received = 0; + stream.on('data', (chunk) => { + received += chunk.byteLength; + if (received > MAX_LEGACY_B64_BYTES) { + stream.destroy( + new Error(`legacy object exceeds the ${MAX_LEGACY_B64_BYTES}B cap`), + ); + return; + } + chunks.push(chunk); + }); + stream.on('error', reject); + stream.on('end', () => resolve(Buffer.concat(chunks))); + }), + timeout, + ]); + const decoded = Buffer.from(body.toString('utf8'), 'base64'); + if (decoded.byteLength > MAX_CREATE_BYTES) { + throw new Error( + `decoded legacy update (${decoded.byteLength}B) exceeds the ${MAX_CREATE_BYTES}B cap`, + ); + } + // compute-task schema requires an exact Uint8Array (lib0 compares the + // constructor) — re-view the Buffer without copying + return new Uint8Array( + decoded.buffer, + decoded.byteOffset, + decoded.byteLength, + ); + } finally { + cancelTimeout(); + } +}; + +// Quick existence check: does yhub already have content for this room? +// Sequenced cheapest-first: a persisted postgres row (bare SELECT, no blob +// columns; rows are never deleted, so a hit is always safe) — then the valkey +// stream (only ydoc:update:v1 counts: awareness and auth-check messages share +// the stream but carry no content) — then the SELECT again, which closes the +// store-before-trim compaction race (and the worst case of a miss is only a +// redundant, idempotent re-seed). +const ydocExists = async (room) => { + if ((await yhub.persistence.retrieveDoc(room, {})).lastClock !== '0') { + return true; + } + const streams = await yhub.stream.getMessages([{ room, clock: '0' }]); + if ((streams[0]?.messages ?? []).some((m) => m.type === 'ydoc:update:v1')) { + return true; + } + return (await yhub.persistence.retrieveDoc(room, {})).lastClock !== '0'; +}; + +// Per-docid migration verdicts, in-memory (per replica). 'exists' is monotone +// in normal operation — its TTL only bounds staleness after an operator +// manually wipes a room's yhub state (restart yhub after a wipe to drop the +// cache immediately). 'empty' (no S3 object) keeps never-edited docs and +// rechecks off S3; 'failed' breaks the retry-refetch storm a permanently +// corrupt object would otherwise sustain (y-websocket retries denied upgrades +// forever). +const VERDICT_TTL_MS = { exists: 600000, empty: 60000, failed: 300000 }; +// transient failures (network blips, timeouts, S3 restarting) are cached just +// long enough to blunt a retry storm without turning a hiccup into a lockout +const TRANSIENT_TTL_MS = 15000; +const TRANSIENT_CODES = new Set([ + 'ECONNREFUSED', + 'ECONNRESET', + 'ETIMEDOUT', + 'EHOSTUNREACH', + 'ENETUNREACH', + 'ENOTFOUND', + 'EAI_AGAIN', + 'EPIPE', +]); +const isTransient = (err) => + err?.transient === true || + TRANSIENT_CODES.has(err?.code) || + TRANSIENT_CODES.has(err?.cause?.code); +const VERDICT_CACHE_MAX = 50000; +const verdicts = new Map(); // docid -> { verdict, error, expires } +const rememberVerdict = (docid, verdict, error = null, ttl = VERDICT_TTL_MS[verdict]) => { + // delete-then-set keeps Map insertion order ≈ recency, so the FIFO eviction + // drops the stalest entry — and re-setting an existing docid never evicts + // an unrelated one + if (!verdicts.delete(docid) && verdicts.size >= VERDICT_CACHE_MAX) { + verdicts.delete(verdicts.keys().next().value); + } + verdicts.set(docid, { + verdict, + error, + expires: Date.now() + ttl, + }); +}; +const inflightMigrations = new Map(); // docid -> Promise +let activeSeeds = 0; + +const migrate = async (room) => { + if (await ydocExists(room)) return 'exists'; + // collapse cross-replica herds: one seeder per room, the rest wait and + // re-probe. The key sits outside yhub's scanned `:room:*` patterns. + const lockKey = `${REDIS_PREFIX}:softmigrate:${room.org}:${room.docid}:${room.branch}`; + const lockToken = randomUUID(); + const redis = yhub.stream.redis; + const acquired = await redis.set(lockKey, lockToken, { + condition: 'NX', + expiration: { type: 'PX', value: MIGRATE_LOCK_TTL_MS }, + }); + try { + if (acquired == null) { + // another connection or replica is seeding — wait for its lock, then + // re-probe. If the doc is still absent (the holder crashed or its S3 + // fetch failed), fall through and seed ourselves: duplicate seeds use + // byte-identical updates from one lineage and merge as CRDT no-ops. + const deadline = Date.now() + MIGRATE_LOCK_TTL_MS + 5000; + while (Date.now() < deadline && (await redis.exists(lockKey)) === 1) { + await new Promise((resolve) => setTimeout(resolve, 300)); + } + if (await ydocExists(room)) return 'exists'; + } + if (activeSeeds >= MAX_CONCURRENT_SEEDS) { + // fail fast under a herd of distinct cold docs — the client's retry + // backoff spreads the load. Probes above stay uncapped. noCache: + // momentary per-replica backpressure must deny once, not be cached as + // a failure — a slot frees up within seconds + const err = new Error('too many concurrent soft migrations'); + err.noCache = true; + throw err; + } + activeSeeds++; + try { + const start = Date.now(); + const update = await fetchLegacyDoc(room.docid); + if (update == null) { + migrationLog.info( + { event: 'seed.empty', docid: room.docid }, + 'no legacy s3 object; room starts empty', + ); + return 'empty'; + } + // legacy content has no per-user history — attribute it to 'system' + // (the admin-JWT identity), marked so audits can tell migrated content + // apart from other system writes + const result = await yhub.computePool.patchYdoc( + { + update, + currentDoc: EMPTY_YDOC, + userid: 'system', + customAttributions: [{ k: 'migration', v: 's3' }], + }, + { room }, + ); + if (result == null) { + // structurally valid but no effective content — nothing to seed + migrationLog.info( + { event: 'seed.empty', docid: room.docid }, + 'legacy s3 object has no effective content; room starts empty', + ); + return 'empty'; + } + await yhub.stream.addMessage(room, { + type: 'ydoc:update:v1', + contentmap: result.contentmap, + update: result.update, + }); + migrationLog.info( + { + event: 'seed.ok', + docid: room.docid, + bytes: update.byteLength, + durationMs: Date.now() - start, + }, + 'seeded legacy doc from s3', + ); + return 'exists'; + } finally { + activeSeeds--; + } + } finally { + if (acquired != null) { + // compare-and-delete: if this seed outlived the lock TTL, another + // seeder holds a fresh lock — a bare DEL would release it under them + redis + .eval( + "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) end", + { keys: [lockKey], arguments: [lockToken] }, + ) + .catch(() => {}); + } + } +}; + +// Resolves when the room is usable (already known, freshly seeded, or +// legitimately empty); rejects to deny access. Idempotent and safe to +// re-enter — it also runs on rechecks and default-purpose REST calls. +const maybeMigrate = async (room) => { + const cached = verdicts.get(room.docid); + if (cached != null && cached.expires > Date.now()) { + if (cached.verdict === 'failed') throw cached.error; + return; + } + let migration = inflightMigrations.get(room.docid); + if (migration == null) { + migration = migrate(room) + .then( + (verdict) => rememberVerdict(room.docid, verdict), + (err) => { + // logged here (once per attempt) rather than per denied connection: + // cached failures deny without new logs until the verdict expires + migrationLog.error( + { event: 'seed.failed', err, docid: room.docid }, + 'soft migration failed; denying access', + ); + if (err?.noCache !== true) { + // transient failures get a short TTL so a hiccup cannot lock a + // doc out for the full poison-object window + rememberVerdict( + room.docid, + 'failed', + err, + isTransient(err) ? TRANSIENT_TTL_MS : VERDICT_TTL_MS.failed, + ); + } + throw err; + }, + ) + .finally(() => inflightMigrations.delete(room.docid)); + inflightMigrations.set(room.docid, migration); + } + return migration; +}; + const auth = createAuthPlugin({ // uws req is only valid synchronously — read headers AND query before first await. async readAuthInfo(req) { @@ -128,18 +455,31 @@ const auth = createAuthPlugin({ ) { return null; } + let doc; try { - const doc = await backendFetch( - `/api/v1.0/documents/${docid}/`, - authInfo, - ); - if (!doc.abilities?.retrieve) { - return null; - } - return doc.abilities.update ? 'rw' : 'r'; + doc = await backendFetch(`/api/v1.0/documents/${docid}/`, authInfo); } catch { return null; } + if (!doc.abilities?.retrieve) { + return null; + } + if (SOFT_MIGRATION) { + // First access to a room yhub does not know: seed it from the legacy + // Django S3 store before admitting the connection. Awaited inside the + // upgrade handler, so the post-upgrade initial sync (which merges + // postgres and the stream from clock 0) is guaranteed to include the + // seed. Runs only for authorized readers. A missing S3 object is the + // brand-new-document case and allows an empty room; a real S3/compute + // failure denies access — an opaque 401 the client retries with + // backoff. + try { + await maybeMigrate({ org, docid, branch }); + } catch { + return null; // already logged (once per attempt) in maybeMigrate + } + } + return doc.abilities.update ? 'rw' : 'r'; }, }); @@ -281,7 +621,9 @@ const api = [ }), ]; -await createYHub({ +// the instance is referenced by the soft-migration helpers above — safe: auth +// callbacks only fire once the server is up, i.e. after this assignment +const yhub = await createYHub({ redis: { url: REDIS, prefix: REDIS_PREFIX, From d70a57bc09121028f8588cfe17d3cbd6bedf3481 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 6 Aug 2026 09:29:04 +0200 Subject: [PATCH 21/59] =?UTF-8?q?=F0=9F=93=9D(changelog)=20note=20that=20g?= =?UTF-8?q?et-connections=20is=20dropped,=20not=20deferred?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hocuspocus-to-yhub migration entry claimed both the kick and get-connections APIs were deferred pending a yhub equivalent. The kick flow now has its server-side replacement (the reset-connections endpoint, backend wiring pending), and get-connections lost its only consumer when the can-edit mechanism was removed — it is dropped, not awaiting reimplementation. Signed-off-by: Kevin Jahns --- CHANGELOG.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 00a6d38376..73e6b6b25f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,9 +56,11 @@ and this project adheres to - 🐛(y-provider) fix sentry init #2579 - 🐛(helm) show the database error while jobs wait for it to be ready #2578 - ♻️(collaboration) migrate the collaboration server from hocuspocus to yhub: - the dev stack gains dedicated valkey and postgres services for yhub, and - the kick (reset-connections) and get-connections APIs have no yhub - equivalent yet — they are deferred with TODO(yhub) stubs + the dev stack gains dedicated valkey and postgres services for yhub. The + kick flow is deferred with TODO(yhub) stubs (the reset-connections + endpoint added above is its server-side replacement, backend wiring + pending). The get-connections API is dropped for good: its only consumer + was the removed can-edit mechanism, so it is not needed anymore - 🔥(backend) remove the unused `CollaborationService` - 💥(backend) remove the `documents/{id}/can-edit/` endpoint - 💥(y-provider) the published `lasuite/impress-y-provider` image becomes From 00ac7552838117c7fe2542a1162715a98ddd862f Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Thu, 6 Aug 2026 12:12:05 +0200 Subject: [PATCH 22/59] =?UTF-8?q?=F0=9F=94=92=EF=B8=8F(collaboration)=20re?= =?UTF-8?q?ject=20admin=20jwts=20not=20issued=20for=20the=20yhub=20audienc?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit yhub verified Django's RS256 admin JWT without checking "aud", so the y-converter token Django hands to the converter process was replayable here — and admin: true short-circuits getAccessType to "rw" on every document, plus the backend-internal reset-connections purpose and the X-User-Id attribution override. Require aud: "yhub", as y-provider already does for its own audience. Nothing in the backend calls yhub's admin endpoints yet, so no caller is affected. Signed-off-by: Kevin Jahns --- CHANGELOG.md | 9 ++++++--- src/yhub-server/server.js | 11 ++++++++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73e6b6b25f..c807839396 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,12 +21,15 @@ and this project adheres to so the Django backend can create documents without speaking yhub's lib0 wire encoding. Strict create (409 when the document already has content), initial content attributed to the optional `X-User-Id` header; guarded by - standard document write access (admin JWT or user session) + standard document write access (the `aud: "yhub"` admin JWT, or a user + session with update ability) - ✨(collaboration) add an admin reset-connections endpoint on yhub: `POST /collaboration/reset-connections/v1/docs/{id}` re-checks the authorization of the document's connected clients and disconnects (close - code 4401) only those whose access changed. Authenticated with the admin - JWT verified against the backend JWKS; not yet triggered by the backend on + code 4401) only those whose access changed. Authenticated with an admin JWT + verified against the backend JWKS and required to carry `aud: "yhub"`, so + an admin token Django issued for another service (e.g. the `y-converter` + one) cannot be replayed here; not yet triggered by the backend on permission changes (follow-up) - ⬆️(collaboration) upgrade yhub to 0.4.0 and serve all its routes under the `/collaboration/` prefix (`server.apiPrefix`): the websocket moves to diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index 79d2cca023..ac782202da 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -22,6 +22,12 @@ const allowedOrigins = ( ).split(','); const Y_PROVIDER_API_KEY = secret('Y_PROVIDER_API_KEY', 'yprovider-api-key'); const ORG = process.env.YHUB_ORG || 'docs'; +// Requiring this audience stops a valid admin JWT that Django issued for +// another service (today: the y-converter token in converter_services.py, +// which is handed to the converter process) from being replayed against yhub. +// Hardcoded, like y-provider's Y_CONVERTER_AUDIENCE: both ends of a two-party +// contract, so an env var would only add a way to misconfigure it into a 401. +const YHUB_AUDIENCE = 'yhub'; // lowercase only (no /i): Django serializes UUIDs lowercase, while yhub rooms // and S3 keys are case-sensitive strings — accepting case variants would let a // client open a parallel room for the same document (and, with soft migration, @@ -407,6 +413,7 @@ const auth = createAuthPlugin({ // silently dropped as a 401. const { payload } = await jwtVerify(token, JWKS, { algorithms: ['RS256'], + audience: YHUB_AUDIENCE, clockTolerance: 5, }); // admin tokens act as the "system" user (no per-user admin identities yet) @@ -414,7 +421,9 @@ const auth = createAuthPlugin({ ? { userid: 'system', admin: true } : null; } catch { - return null; // bad signature / expired / JWKS unreachable — fail closed + // bad signature / expired / wrong (or missing) audience / JWKS + // unreachable — fail closed + return null; } } if (gcOff) return null; // full-history connections: not for Docs users From 4c60fc5c657449359f8a1f7e5ee05f5629068d9b Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Fri, 7 Aug 2026 11:45:57 +0200 Subject: [PATCH 23/59] =?UTF-8?q?=E2=9C=A8(collaboration)=20replay=20legac?= =?UTF-8?q?y=20s3=20version=20history=20into=20yhub?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add POST /collaboration/migrate/v1/docs/{id}, which replays every S3 version of a document's legacy `{id}/file` object into one gc:false Yjs document and stores it as a single row at clock 0, crediting each version with its own S3 timestamp. Nothing existing is deleted and nothing goes on the stream, so the next compaction merges that row like any other. The clock-0 insert is ON CONFLICT DO NOTHING and migrated ids are kept in a valkey set, so the endpoint is idempotent without a lock. The activity api then reports the same timeline as the backend's /documents/{id}/versions/, instead of the single migration-time change the lazy soft migration leaves behind. That lazy seed now writes no insertAt/deleteAt. Persisted contentmaps are merged rather than de-duplicated, so a seed timestamp would survive next to the real per-version one on the same ids and the activity api would report whichever the unordered row scan put last. A seed is not an editing event and has no honest time to report. Upgrade yhub to 0.5.0, where error codes encode retry semantics (4xx permanent, 5xx and 429 retryable) and auth plugins may throw apiError(503). A temporarily unreachable Django backend, JWKS endpoint or legacy S3 store is now reported as 503 rather than denied like a permission failure, so clients retry instead of giving up. The legacy-store code moves out of server.js into migration.js, with the shared *_FILE secret helper in env.js. Signed-off-by: Kevin Jahns --- CHANGELOG.md | 35 +- src/yhub-server/Dockerfile | 3 +- src/yhub-server/README.md | 154 ++++++-- src/yhub-server/env.js | 10 + src/yhub-server/migration.js | 591 ++++++++++++++++++++++++++++++ src/yhub-server/package-lock.json | 17 +- src/yhub-server/package.json | 3 +- src/yhub-server/server.js | 478 +++++++----------------- 8 files changed, 898 insertions(+), 393 deletions(-) create mode 100644 src/yhub-server/env.js create mode 100644 src/yhub-server/migration.js diff --git a/CHANGELOG.md b/CHANGELOG.md index c807839396..478b05a38d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,10 +11,26 @@ and this project adheres to - ✨(collaboration) soft-migrate legacy S3 documents into yhub on first access (`SOFT_MIGRATION=true`): when yhub does not know a document yet, its legacy snapshot (`{id}/file`, base64 Yjs update) is fetched from the Django S3 - media bucket and seeded server-side (attributed to `system`) before the - connection is admitted. A missing S3 object means a brand-new document and - yields an empty room; any real S3/compute failure fails closed (opaque 401, - the client retries with backoff). Enabled in the dev stack via compose.yml + media bucket and seeded server-side (attributed to `system`, with no + timestamp — a lazy seed is not an editing event, and stamping one would + collide with the real per-version times the migrate endpoint writes) before + the connection is admitted. A missing S3 object means a brand-new document and + yields an empty room; any real S3/compute failure fails closed, as a + permanent `403` for a corrupt or oversized object and a retryable `503` for + timeouts and network errors. Enabled in the dev stack via compose.yml +- ✨(collaboration) add a migrate endpoint on yhub: + `POST /collaboration/migrate/v1/docs/{id}` replays a document's **full** + legacy version history from the versioned S3 media bucket into a + `gc: false` Yjs document, crediting each S3 version with its own S3 + timestamp — so `activity?group=false` reports the same timeline as the + backend's `/documents/{id}/versions/`, instead of the single + migration-time change the + lazy soft migration leaves behind. Purely additive: the result is stored as + one new row at clock `0`, so nothing existing is deleted and the next + compaction merges it like any other row. Idempotent by construction (the + clock-`0` insert is `ON CONFLICT DO NOTHING`, and migrated ids are recorded + in a valkey set), lock-free, admin JWT only, and the intended way to backfill + the corpus before `SOFT_MIGRATION` is turned off - ✨(collaboration) add a create-ydoc endpoint on yhub: `POST /collaboration/create-ydoc/v1/docs/{id}` seeds a document's initial Yjs state from a raw binary update posted as `application/octet-stream`, @@ -31,11 +47,16 @@ and this project adheres to an admin token Django issued for another service (e.g. the `y-converter` one) cannot be replayed here; not yet triggered by the backend on permission changes (follow-up) -- ⬆️(collaboration) upgrade yhub to 0.4.0 and serve all its routes under the +- ⬆️(collaboration) upgrade yhub to 0.5.0 and serve all its routes under the `/collaboration/` prefix (`server.apiPrefix`): the websocket moves to `/collaboration/ws/v1/docs`. All `/collaboration/` routes are meant to be - publicly exposed except `reset-connections`, which stays backend-internal - (admin JWT only) + publicly exposed except `reset-connections` and `migrate`, which stay + backend-internal (admin JWT only). Following 0.5.0's error semantics + (`4xx` permanent, `5xx`/`429` retryable), the auth plugin now reports a + temporarily unreachable Django backend, JWKS endpoint or legacy S3 store as + `503` instead of denying access like a permission failure, so clients retry + instead of giving up. The built-in endpoints can also answer JSON on + `Accept: application/json` - ✨(backend) add a service generating cached RS256 JWT tokens - ✨(backend) publish the JWT public key on a JWKS endpoint - 🔧(dev) generate the JWT signing key when bootstrapping the dev stack diff --git a/src/yhub-server/Dockerfile b/src/yhub-server/Dockerfile index 31394e2d9c..b7ef20248b 100644 --- a/src/yhub-server/Dockerfile +++ b/src/yhub-server/Dockerfile @@ -6,7 +6,8 @@ WORKDIR /app COPY package.json package-lock.json ./ RUN npm ci --omit=dev -COPY server.js ./ +# server.js, migration.js, env.js — glob so a new module cannot be forgotten +COPY *.js ./ EXPOSE 3002 diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index d6e04670f1..a980f437e6 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -4,7 +4,14 @@ This directory contains the La Suite Docs-specific configuration for [yhub](https://www.npmjs.com/package/@y/hub) (`@y/hub`), the collaboration server that synchronizes Yjs documents between editors in real time. -It is not a fork of yhub — it is a thin wrapper (`server.js`) that: +It is not a fork of yhub — it is a thin wrapper: + +- `server.js` — configuration, the auth plugin, and the custom REST endpoints, +- `migration.js` — everything that reads the legacy Django/S3 document store + (both migrations described below), +- `env.js` — the `*_FILE` secret indirection shared by the two. + +`server.js`: - starts a yhub instance (websocket sync on port 3002, backed by Redis/Valkey and PostgreSQL), @@ -29,6 +36,9 @@ It is not a fork of yhub — it is a thin wrapper (`server.js`) that: server-side. Strict create: 409 when the document already has content. Guarded by standard document write access (the admin JWT, or a user session with update ability), +- exposes `POST /collaboration/migrate/v1/{org}/{docid}`, which replays a + document's **full** legacy version history out of the S3 media bucket (see + "Full migration" below) — admin JWT only, like `reset-connections`, - mirrors the environment conventions used elsewhere in this repository (`*_FILE` secret indirection, `COLLABORATION_SERVER_ORIGIN` allowlist, …). @@ -36,8 +46,8 @@ Public exposure: route the whole `/collaboration/` prefix to this server — the websocket and the built-in document APIs (`ydoc`, `rollback`, `prune`, `changeset`, `activity`) are all guarded by the same cookie-based document authorization and are meant to be reachable by browsers. The one exception -is `/collaboration/reset-connections/`, which is backend-internal and should -not be routed through the public ingress. +is `/collaboration/reset-connections/` and `/collaboration/migrate/`, which are +backend-internal and should not be routed through the public ingress. The `Dockerfile` builds the container image used by the `yhub` service in `compose.yml`. @@ -50,11 +60,11 @@ bucket, as UTF-8 text that is the base64 encoding of a raw Yjs update, at key documents into yhub lazily, on first access: 1. After a user's document authorization succeeds, the auth plugin checks - whether yhub already has content for the room — a bare postgres `SELECT` - (persisted rows), then the valkey stream (uncompacted `ydoc:update:v1` - messages), then the `SELECT` again to close the compaction race. Verdicts - are cached in-process (existing docs 10 min, empty docs 60 s, failures - 5 min). + whether yhub already has content for the room — the migrated set written by + the full migration (below), then a bare postgres `SELECT` (persisted rows), + then the valkey stream (uncompacted `ydoc:update:v1` messages), then the + `SELECT` again to close the compaction race. Verdicts are cached in-process + (existing docs 10 min, empty docs 60 s, failures 5 min). 2. If the room is unknown, the legacy object is fetched from S3 (10 s timeout, 10 MiB decoded cap — the same limit as `create-ydoc`), decoded, diffed through yhub's compute pool and appended to the room's stream — @@ -63,8 +73,20 @@ documents into yhub lazily, on first access: initial sync always includes the seeded content. First access to an unmigrated document is therefore slower by one S3 round-trip plus one compute pass. + + **A seed carries no timestamp.** Its contentmap has `insert`/`delete` and + `migration=s3` but deliberately no `insertAt`/`deleteAt`: a lazy seed is not + an editing event, and the only honest timestamps for legacy content are the + S3 version times that the full migration writes. Stamping the seed too would + put a second `insertAt` on the same ids — persisted contentmaps are merged, + not de-duplicated — and `activity` would report whichever the (unordered) + row scan happened to put last. The practical consequence: seeded content + produces **no `activity` entry** and is skipped by `from`/`to`-filtered + `changeset`/`rollback`/`prune` queries until the full migration supplies the + real history. Unfiltered queries, `by=system` and + `withCustomAttributions=migration:s3` still match it. 3. Concurrent first-connections are collapsed: an in-process in-flight map, a - per-room valkey lock (`{prefix}:softmigrate:*`, 30 s TTL), and a cap of 4 + per-room valkey lock (`{prefix}:softmigrate:*`, 30 s TTL), and a cap of 20 concurrent seeds per replica (excess connections fail fast and retry). Guarantees and failure behavior: @@ -72,12 +94,14 @@ Guarantees and failure behavior: - **Missing S3 object is not an error** — that is the brand-new-document case (Django writes no object until the first content save); the room simply starts empty. -- **Everything else fails closed**: network/auth errors, timeouts, oversized - objects and corrupt updates deny the connection (an opaque 401 the client - retries with backoff) and log a `soft-migration` error. A cached failure +- **Everything else fails closed**, and says whether it is worth retrying + (yhub 0.5.0 error semantics): an oversized or corrupt legacy object will fail + the same way forever, so the connection is denied `403`; network errors, + timeouts and momentary seed backpressure answer `503`, which clients retry + with backoff. Either way a `soft-migration` error is logged. A cached failure verdict prevents retry storms from hammering S3 — permanent failures (corrupt/oversized objects) for 5 minutes, transient ones (network errors, - timeouts) for 15 seconds, and per-replica seed backpressure (more than 4 + timeouts) for 15 seconds, and per-replica seed backpressure (more than 20 concurrent seeds) is denied without caching so the client's next retry goes through. - **Seeding is idempotent**: the legacy S3 snapshots are frozen (the @@ -116,13 +140,95 @@ Operational notes: "exists" and suppress the re-seed. - Lazy migration never finishes on its own: documents that are never opened stay in S3 forever, and they are only reachable through this flag now that - the frontend's client-side seeding is gone. A batch backfill (Django - posting each document's **exact** S3 bytes to `create-ydoc` with the admin - JWT, treating 409 as success) is the intended completion path and composes - safely with concurrent first-accesses — as long as content is never - re-converted: independently generated updates for the same document would - duplicate its content on merge, while re-posting the stored bytes is a - no-op. Only after the backfill may `SOFT_MIGRATION` be turned off. + the frontend's client-side seeding is gone. Running `migrate` (below) over + the corpus is the intended completion path. Only after that backfill may + `SOFT_MIGRATION` be turned off. + +## Full migration (`POST /collaboration/migrate/v1/{org}/{docid}`) + +The media bucket is versioned, so `{docid}/file` keeps every snapshot Django +ever wrote — that is the version history the backend exposes at +`/documents/{id}/versions/`. The lazy seed above replays only the newest one, so +a soft-migrated document lands in yhub as a single `system` change stamped with +the migration time and its past is gone. + +`migrate` replays the whole history instead. It lists the object's versions and +applies them, oldest first, to a single `Y.Doc({ gc: false })`; after each one +it credits the ids that version introduced (and the ones it deleted) with +**that version's own S3 timestamp**. `GET +/collaboration/activity/v1/{org}/{docid}?group=false` then reports one entry per +S3 version, at the same timestamps the backend's version listing reports as +`last_modified` — which is what lines the two up. (Pass `group=false`: the +default grouping merges changes by the same author less than a second apart, +which would fold versions saved in quick succession into one entry.) +`gc: false` is what preserves content that later versions deleted — most of +what makes a history worth keeping. + +Since yhub 0.5.0 the built-in endpoints also speak JSON on request, so a +non-JavaScript caller can read that timeline without a lib0 decoder: send +`Accept: application/json` and `activity`/`changeset` answer +`application/json` (binary fields base64-encoded) instead of +`application/x-lib0any`. + +The result lands as **one new row in `yhub_ydoc_v1` at clock `0`**, written +through `yhub.persistence.store`. Nothing is deleted and nothing goes on the +redis stream: the migration is purely additive. Clock `0` is what makes that +safe — + +- `store` is `ON CONFLICT (org, docid, branch, t) DO NOTHING`, so a repeated or + concurrent call is a database no-op; +- `retrieveDoc` derives the room's `lastClock` from the *newest* row, so a `0` + row can never hide stream messages a live editor is writing; +- the history genuinely is the oldest thing in the room. + +The next compact task merges that row into the room's normal state and deletes +it, like any other row — yhub needs no special case for it. + +Called with the admin JWT (`aud: "yhub"`), doc-scoped, `branch=main` only (the +legacy store is branchless). Running it over the whole corpus — any 2xx means +done — is the backfill that finishes the migration. + +Guarantees: + +- **Idempotent, twice over.** The docid is recorded in the valkey set + `{prefix}:migrated:v1` and skipped on later calls; and even without that, the + `t = 0` insert is a no-op. There is no lock: concurrent calls for the same + document all succeed and the database keeps one row. +- **Never destructive.** No existing row, stream message or attribution is + removed, so a document's own yhub history — edits made after it was seeded — + survives untouched alongside the imported one. +- **Nothing usable, nothing touched, nothing remembered.** A document with no + legacy object (`versions: 0`) or with no readable version (`applied: 0`) is + left exactly as it is and is *not* added to the set, so a later run can still + pick it up. Both answer `200 {"migrated": false}`, so a backfill driver can + treat every 2xx as done. +- **A corrupt version is skipped, not fatal** (counted as `skipped`, logged with + its version id). Snapshots are decoded before they are applied, so a bad one + can neither corrupt the accumulating document nor kill a compute worker. + Later versions are full snapshots, so their content still arrives — only that + one timeline entry is lost. +- **More than 500 versions**: only the newest 500 are replayed and the rest fold + into the first replayed version, reported as `dropped`. + +Response (200): `{ migrated, versions, applied, skipped, dropped, bytes, +durationMs }`. + +Caveats: + +- **`?force=true` re-runs a document that is already in the set.** Only safe + while its clock-0 row is still there. Once compaction has folded that row + away, a forced re-run inserts a second contentmap for ids that already carry + one, both `insertAt` values survive the merge, and the activity timestamp for + that content becomes whichever the unordered row scan puts last. To genuinely + redo a document, wipe its yhub state first (rows, stream key, set member). +- **The replay runs on the server's main thread.** yhub's compute pool only + accepts its own fixed task types, so a very long history briefly blocks the + event loop; that is what the 500-version cap bounds. +- **`activity` and `changeset` responses are cached for ~5s** (yhub's + `redis.cacheTtl`). A call made right after a migration can still answer with + the pre-migration timeline; it resolves itself. +- Requires `SOFT_MIGRATION=true` (that is what configures the S3 client); + otherwise it answers `503`. ## ⚠️ License warning (AGPL) @@ -131,10 +237,10 @@ This directory depends on `@y/hub`, which is licensed under the the rest of this repository (MIT), the code in this directory is loaded into the same process as AGPL-licensed code. As a consequence: -- **Any modification to the code in this directory (in particular - `server.js`) must be released under an AGPL-compatible license** if you run - or distribute the resulting server, including making it available to users - over a network (AGPL section 13). +- **Any modification to the code in this directory (in particular `server.js` + and `migration.js`) must be released under an AGPL-compatible license** if + you run or distribute the resulting server, including making it available to + users over a network (AGPL section 13). - See the [LICENSE](./LICENSE) file in this directory for details. **The rest of La Suite Docs is not affected.** The Django backend and the diff --git a/src/yhub-server/env.js b/src/yhub-server/env.js new file mode 100644 index 0000000000..a962bc9a77 --- /dev/null +++ b/src/yhub-server/env.js @@ -0,0 +1,10 @@ +import { readFileSync } from 'node:fs'; + +// Read a config value that may be supplied either directly (`NAME`) or as a +// path to a file holding it (`NAME_FILE`) — the secret-file convention used +// across this repository, mirroring y-provider's env.ts. Shared by server.js +// and migration.js. +export const secret = (name, dflt) => + process.env[`${name}_FILE`] + ? readFileSync(process.env[`${name}_FILE`], 'utf8').trim() + : process.env[name] || dflt; diff --git a/src/yhub-server/migration.js b/src/yhub-server/migration.js new file mode 100644 index 0000000000..5e42fee898 --- /dev/null +++ b/src/yhub-server/migration.js @@ -0,0 +1,591 @@ +// Migration off the legacy Django document store (see README.md). +// +// Documents were historically stored by the Django backend in the S3 media +// bucket, as UTF-8 text that is the base64 encoding of a raw Yjs update, at key +// `{document-uuid}/file`. The bucket is versioned, so every snapshot Django +// ever wrote for a document survives as an object version — that is the +// version history the backend exposes at `/documents/{id}/versions/`. +// +// Two paths bring that content into yhub, and they compose: +// +// maybeMigrate — the lazy seed. On first access to a room yhub does not know, +// fetch the *newest* version and seed the room with it before admitting the +// connection. Attributed to `system`, with no timestamp (see below). +// +// fullMigrate — the backfill. Replay *every* version into one gc:false +// document and store the result as a single row at clock 0, so yhub's +// activity API reports the same timeline as the S3 version listing. +// +// Everything here takes the `yhub` instance explicitly rather than closing over +// it: the endpoint handlers already receive one as `req.yhub`, and the auth +// plugin has the module-level instance by the time it first runs. + +import { randomUUID } from 'node:crypto'; + +import { logger } from '@y/hub'; +import * as Y from '@y/y'; +import { Client as S3Client } from 'minio'; + +import { secret } from './env.js'; + +export const SOFT_MIGRATION = process.env.SOFT_MIGRATION === 'true'; +const AWS_S3_ENDPOINT_URL = process.env.AWS_S3_ENDPOINT_URL; +const AWS_S3_ACCESS_KEY_ID = secret('AWS_S3_ACCESS_KEY_ID'); +const AWS_S3_SECRET_ACCESS_KEY = secret('AWS_S3_SECRET_ACCESS_KEY'); +const AWS_S3_REGION_NAME = process.env.AWS_S3_REGION_NAME; +// Django's default bucket name (impress settings.py) — prod overrides it +const AWS_STORAGE_BUCKET_NAME = + process.env.AWS_STORAGE_BUCKET_NAME || 'impress-media-storage'; +// the same limit create-ydoc applies to a posted update in server.js: one +// legacy snapshot handed to a compute worker, or written to the stream as a +// single message +const MAX_LEGACY_BYTES = 10 * 1024 * 1024; +// base64 inflates 3 bytes to 4 — cap the streamed read at the encoded size of +// MAX_LEGACY_BYTES plus padding slack +const MAX_LEGACY_B64_BYTES = Math.ceil(MAX_LEGACY_BYTES / 3) * 4 + 1024; +const S3_FETCH_TIMEOUT_MS = 10000; +const MIGRATE_LOCK_TTL_MS = 30000; +const MAX_CONCURRENT_SEEDS = 20; +// The full migration replays *every* S3 version of a document, so its budget is +// per-document rather than per-connection — no client is waiting on it. +const S3_LIST_TIMEOUT_MS = 30000; +// A document with more versions than this is migrated from its newest +// MAX_MIGRATE_VERSIONS only: everything older folds into the first replayed +// version, which keeps the run bounded instead of failing it outright. The +// response reports how many were dropped. The replay runs on the main thread, +// so the cap also bounds how long the event loop is blocked. +const MAX_MIGRATE_VERSIONS = 500; +// an empty Yjs update, what patchYdoc diffs the first snapshot against +const EMPTY_YDOC = Y.encodeStateAsUpdate(new Y.Doc()); + +if ( + SOFT_MIGRATION && + (!AWS_S3_ENDPOINT_URL || !AWS_S3_ACCESS_KEY_ID || !AWS_S3_SECRET_ACCESS_KEY) +) { + // fail at boot instead of as an opaque 401 storm on first connect + throw new Error( + 'SOFT_MIGRATION=true requires AWS_S3_ENDPOINT_URL, AWS_S3_ACCESS_KEY_ID and AWS_S3_SECRET_ACCESS_KEY', + ); +} +const s3 = SOFT_MIGRATION + ? (() => { + const url = new URL(AWS_S3_ENDPOINT_URL); + if (url.pathname !== '/' && url.pathname !== '') { + // boto3 accepts path-prefixed endpoints but the minio client cannot + // address a base path — dropping it silently would probe the wrong + // keys and "migrate" every doc as empty + throw new Error('AWS_S3_ENDPOINT_URL must not contain a path'); + } + return new S3Client({ + endPoint: url.hostname, + port: + url.port !== '' + ? Number(url.port) + : url.protocol === 'https:' + ? 443 + : 80, + useSSL: url.protocol === 'https:', + accessKey: AWS_S3_ACCESS_KEY_ID, + secretKey: AWS_S3_SECRET_ACCESS_KEY, + ...(AWS_S3_REGION_NAME ? { region: AWS_S3_REGION_NAME } : {}), + }); + })() + : null; +const migrationLog = logger.child({ module: 'soft-migration' }); + +// Both keys are derived from the prefix yhub itself resolved, so they cannot +// drift from the room keys, and both sit outside its scanned `:room:*` pattern. +// +// One seeder per room: +const migrateLockKey = (yhub, room) => + `${yhub.stream.prefix}:softmigrate:${room.org}:${room.docid}:${room.branch}`; +// Documents whose version history has been replayed into postgres. Membership +// is permanent: a second replay of the same versions would attribute the same +// content twice (see fullMigrate). +const migratedSetKey = (yhub) => `${yhub.stream.prefix}:migrated:v1`; + +// Legacy Django document store: object `{docid}/file`, body = UTF-8 text that +// is the base64 encoding of a raw Yjs update. With `versionId`, reads that +// specific object version instead of the current one. Returns null when the +// object (or version) does not exist — a document that never had content +// saved, e.g. brand new. Throws on any other failure (network, auth, timeout, +// oversize); corrupt base64 decodes leniently to garbage that the callers +// reject. +const fetchLegacyDoc = async (docid, versionId = null) => { + let stream = null; + let cancelTimeout = () => {}; + // minio 8 takes no AbortSignal — race a timer that also destroys the body + // stream once reading, so a stalled transfer cannot hold the ws upgrade + const timeout = new Promise((_, reject) => { + const timer = setTimeout(() => { + const err = new Error( + `s3 fetch timed out after ${S3_FETCH_TIMEOUT_MS}ms`, + ); + err.transient = true; // a slow S3 may recover — cache the failure briefly + stream?.destroy(err); + reject(err); + }, S3_FETCH_TIMEOUT_MS); + cancelTimeout = () => clearTimeout(timer); + }); + try { + let objPromise; + try { + objPromise = s3.getObject( + AWS_STORAGE_BUCKET_NAME, + `${docid}/file`, + // minio stringifies the whole opts object into the query — pass + // undefined, not {}, so the unversioned read stays byte-identical + versionId != null ? { versionId } : undefined, + ); + stream = await Promise.race([objPromise, timeout]); + } catch (err) { + // NoSuchVersion: the version vanished between listing and reading + if (err?.code === 'NoSuchKey' || err?.code === 'NoSuchVersion') { + return null; + } + // if the timeout won the race, getObject may still resolve later — + // destroy the late-arriving response stream, otherwise its never-read + // socket leaks (minio 8 sets no request timeout and cannot abort) + objPromise?.then( + (s) => s.destroy(err), + () => {}, + ); + throw err; + } + const body = await Promise.race([ + new Promise((resolve, reject) => { + const chunks = []; + let received = 0; + stream.on('data', (chunk) => { + received += chunk.byteLength; + if (received > MAX_LEGACY_B64_BYTES) { + stream.destroy( + new Error( + `legacy object exceeds the ${MAX_LEGACY_B64_BYTES}B cap`, + ), + ); + return; + } + chunks.push(chunk); + }); + stream.on('error', reject); + stream.on('end', () => resolve(Buffer.concat(chunks))); + }), + timeout, + ]); + const decoded = Buffer.from(body.toString('utf8'), 'base64'); + if (decoded.byteLength > MAX_LEGACY_BYTES) { + throw new Error( + `decoded legacy update (${decoded.byteLength}B) exceeds the ${MAX_LEGACY_BYTES}B cap`, + ); + } + // compute-task schema requires an exact Uint8Array (lib0 compares the + // constructor) — re-view the Buffer without copying + return new Uint8Array( + decoded.buffer, + decoded.byteOffset, + decoded.byteLength, + ); + } finally { + cancelTimeout(); + } +}; + +// Every version of the legacy object, oldest first. Delete markers are skipped +// (they record a deletion and carry no body), and so are keys that merely share +// the prefix — S3 has no exact-key version listing. +const listLegacyVersions = async (docid) => { + const key = `${docid}/file`; + const found = await new Promise((resolve, reject) => { + const versions = []; + const stream = s3.listObjects(AWS_STORAGE_BUCKET_NAME, key, true, { + IncludeVersion: true, + }); + const timer = setTimeout(() => { + const err = new Error( + `s3 version listing timed out after ${S3_LIST_TIMEOUT_MS}ms`, + ); + err.transient = true; + stream.destroy(err); + }, S3_LIST_TIMEOUT_MS); + stream.on('data', (obj) => { + if (obj.name === key && obj.isDeleteMarker !== true && obj.versionId) { + versions.push({ + versionId: String(obj.versionId), + // the moment S3 accepted the write: what the backend's version + // listing reports as `last_modified`, and what we attribute to + timestamp: obj.lastModified?.getTime() ?? 0, + }); + } + }); + stream.on('error', (err) => { + clearTimeout(timer); + reject(err); + }); + stream.on('end', () => { + clearTimeout(timer); + resolve(versions); + }); + }); + // S3 lists a key's versions newest first; reverse to replay them in write + // order. The sort is a stable safeguard across paginated listings — equal + // timestamps keep S3's own ordering. + found.reverse(); + found.sort((a, b) => a.timestamp - b.timestamp); + const dropped = Math.max(0, found.length - MAX_MIGRATE_VERSIONS); + return { versions: found.slice(dropped), dropped }; +}; + +// Quick existence check: does yhub already have content for this room? +// Sequenced cheapest-first: a persisted postgres row (bare SELECT, no blob +// columns; rows are never deleted, so a hit is always safe) — then the valkey +// stream (only ydoc:update:v1 counts: awareness and auth-check messages share +// the stream but carry no content) — then the SELECT again, which closes the +// store-before-trim compaction race (and the worst case of a miss is only a +// redundant, idempotent re-seed). +const ydocExists = async (yhub, room) => { + if ((await yhub.persistence.retrieveDoc(room, {})).lastClock !== '0') { + return true; + } + const streams = await yhub.stream.getMessages([{ room, clock: '0' }]); + if ((streams[0]?.messages ?? []).some((m) => m.type === 'ydoc:update:v1')) { + return true; + } + return (await yhub.persistence.retrieveDoc(room, {})).lastClock !== '0'; +}; + +// Per-docid migration verdicts, in-memory (per replica). 'exists' is monotone +// in normal operation — its TTL only bounds staleness after an operator +// manually wipes a room's yhub state (restart yhub after a wipe to drop the +// cache immediately). 'empty' (no S3 object) keeps never-edited docs and +// rechecks off S3; 'failed' breaks the retry-refetch storm a permanently +// corrupt object would otherwise sustain (y-websocket retries denied upgrades +// forever). +const VERDICT_TTL_MS = { exists: 600000, empty: 60000, failed: 300000 }; +// transient failures (network blips, timeouts, S3 restarting) are cached just +// long enough to blunt a retry storm without turning a hiccup into a lockout +const TRANSIENT_TTL_MS = 15000; +const TRANSIENT_CODES = new Set([ + 'ECONNREFUSED', + 'ECONNRESET', + 'ETIMEDOUT', + 'EHOSTUNREACH', + 'ENETUNREACH', + 'ENOTFOUND', + 'EAI_AGAIN', + 'EPIPE', +]); +// Will this failure plausibly resolve on its own? Callers use it twice: to pick +// the verdict TTL below, and — in server.js — to decide whether a denied +// connection reports a permanent 403 or a retryable 503. `noCache` is the +// per-replica seed backpressure, transient by construction (a slot frees up +// within seconds). +export const isTransientFailure = (err) => + err?.transient === true || + err?.noCache === true || + TRANSIENT_CODES.has(err?.code) || + TRANSIENT_CODES.has(err?.cause?.code); +const VERDICT_CACHE_MAX = 50000; +const verdicts = new Map(); // docid -> { verdict, error, expires } +const rememberVerdict = ( + docid, + verdict, + error = null, + ttl = VERDICT_TTL_MS[verdict], +) => { + // delete-then-set keeps Map insertion order ≈ recency, so the FIFO eviction + // drops the stalest entry — and re-setting an existing docid never evicts + // an unrelated one + if (!verdicts.delete(docid) && verdicts.size >= VERDICT_CACHE_MAX) { + verdicts.delete(verdicts.keys().next().value); + } + verdicts.set(docid, { + verdict, + error, + expires: Date.now() + ttl, + }); +}; +const inflightMigrations = new Map(); // docid -> Promise +let activeSeeds = 0; + +const migrate = async (yhub, room) => { + // A fully migrated room holds a single row at clock 0, which leaves + // `lastClock` at '0' — so ydocExists cannot see it and would seed on top of a + // complete history. Harmless (the seed's attributions are excluded as already + // known) but a pointless S3 round-trip per document during a backfill. + if (await yhub.stream.redis.sIsMember(migratedSetKey(yhub), room.docid)) { + return 'exists'; + } + if (await ydocExists(yhub, room)) return 'exists'; + // collapse cross-replica herds: one seeder per room, the rest wait and + // re-probe + const lockKey = migrateLockKey(yhub, room); + const lockToken = randomUUID(); + const redis = yhub.stream.redis; + const acquired = await redis.set(lockKey, lockToken, { + condition: 'NX', + expiration: { type: 'PX', value: MIGRATE_LOCK_TTL_MS }, + }); + try { + if (acquired == null) { + // another connection or replica is seeding — wait for its lock, then + // re-probe. If the doc is still absent (the holder crashed or its S3 + // fetch failed), fall through and seed ourselves: duplicate seeds use + // byte-identical updates from one lineage and merge as CRDT no-ops. + const deadline = Date.now() + MIGRATE_LOCK_TTL_MS + 5000; + while (Date.now() < deadline && (await redis.exists(lockKey)) === 1) { + await new Promise((resolve) => setTimeout(resolve, 300)); + } + if (await ydocExists(yhub, room)) return 'exists'; + } + if (activeSeeds >= MAX_CONCURRENT_SEEDS) { + // fail fast under a herd of distinct cold docs — the client's retry + // backoff spreads the load. Probes above stay uncapped. noCache: + // momentary per-replica backpressure must deny once, not be cached as + // a failure — a slot frees up within seconds + const err = new Error('too many concurrent soft migrations'); + err.noCache = true; + throw err; + } + activeSeeds++; + try { + const start = Date.now(); + const update = await fetchLegacyDoc(room.docid); + if (update == null) { + migrationLog.info( + { event: 'seed.empty', docid: room.docid }, + 'no legacy s3 object; room starts empty', + ); + return 'empty'; + } + await yhub.stream.addMessage(room, { + type: 'ydoc:update:v1', + // Deliberately no insertAt/deleteAt. A lazy seed is not an editing + // event: stamping it would put a second, meaningless timestamp on + // content that the full migration attributes to its real S3 version + // time — and persisted contentmaps are merged, not de-duplicated, so + // both would survive on the same ids and the activity API would report + // whichever the row order happened to put last. Content seeded this way + // carries an author but no timestamp, so it produces no activity entry + // until fullMigrate supplies the history. + // + // Reading the ids also validates the update: a corrupt legacy object + // throws here, on this thread, before anything reaches the stream. + contentmap: Y.encodeContentMap( + Y.createContentMapFromContentIds( + Y.createContentIdsFromUpdate(update), + [ + Y.createContentAttribute('insert', 'system'), + Y.createContentAttribute('insert:migration', 's3'), + ], + [ + Y.createContentAttribute('delete', 'system'), + Y.createContentAttribute('delete:migration', 's3'), + ], + ), + ), + update, + }); + migrationLog.info( + { + event: 'seed.ok', + docid: room.docid, + bytes: update.byteLength, + durationMs: Date.now() - start, + }, + 'seeded legacy doc from s3', + ); + return 'exists'; + } finally { + activeSeeds--; + } + } finally { + if (acquired != null) { + // compare-and-delete: if this seed outlived the lock TTL, another + // seeder holds a fresh lock — a bare DEL would release it under them + redis + .eval( + "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) end", + { keys: [lockKey], arguments: [lockToken] }, + ) + .catch(() => {}); + } + } +}; + +// Resolves when the room is usable (already known, freshly seeded, or +// legitimately empty); rejects to deny access. Idempotent and safe to +// re-enter — it also runs on rechecks and default-purpose REST calls. +export const maybeMigrate = async (yhub, room) => { + const cached = verdicts.get(room.docid); + if (cached != null && cached.expires > Date.now()) { + if (cached.verdict === 'failed') throw cached.error; + return; + } + let migration = inflightMigrations.get(room.docid); + if (migration == null) { + migration = migrate(yhub, room) + .then( + (verdict) => rememberVerdict(room.docid, verdict), + (err) => { + // logged here (once per attempt) rather than per denied connection: + // cached failures deny without new logs until the verdict expires + migrationLog.error( + { event: 'seed.failed', err, docid: room.docid }, + 'soft migration failed; denying access', + ); + if (err?.noCache !== true) { + // transient failures get a short TTL so a hiccup cannot lock a + // doc out for the full poison-object window + rememberVerdict( + room.docid, + 'failed', + err, + isTransientFailure(err) + ? TRANSIENT_TTL_MS + : VERDICT_TTL_MS.failed, + ); + } + throw err; + }, + ) + .finally(() => inflightMigrations.delete(room.docid)); + inflightMigrations.set(room.docid, migration); + } + return migration; +}; + +// Add the legacy version history to the room. Returns a `status` the endpoint +// maps to a response: +// 'already' — already replayed for this docid; the room is left untouched +// 'empty' — no legacy object in S3; the room is left untouched +// 'nothing' — versions exist but none is readable; the room is left untouched +// 'ok' — history stored +// +// Additive, never destructive: the versions are replayed into one gc:false +// document, and the result lands as a *single new row* at clock 0 — nothing +// existing is deleted and nothing goes on the stream. Clock 0 is what makes +// that safe. `store` is ON CONFLICT DO NOTHING on (org, docid, branch, t), so a +// concurrent or repeated call is a database no-op; `retrieveDoc` derives +// `lastClock` from the *newest* row, so a 0 row can never hide the stream +// messages a live editor is writing; and the history is genuinely the oldest +// thing in the room. The next compact task merges the row into the room's +// normal state and drops it — yhub needs no special case for any of this. +export const fullMigrate = async (yhub, room, { force = false } = {}) => { + const start = Date.now(); + const redis = yhub.stream.redis; + // Membership is the guard against attributing the same content twice: once + // compaction has folded the clock-0 row into a normal one and deleted it, + // a second replay would insert a second contentmap for ids that already + // carry one, and the two timestamps would both survive the merge. + if (!force && (await redis.sIsMember(migratedSetKey(yhub), room.docid))) { + return { status: 'already' }; + } + const { versions, dropped } = await listLegacyVersions(room.docid); + if (versions.length === 0) { + return { status: 'empty', versions: 0, durationMs: Date.now() - start }; + } + // Replay every snapshot into one gc:false document. gc matters: a collected + // doc would lose the content later versions deleted, which is most of what + // makes a history worth having. + const ydoc = new Y.Doc({ gc: false }); + // ids already attributed, so each version is credited only with what it added + let seen = Y.createContentIds(); + const contentmaps = []; + let bytes = 0; + let skipped = 0; + try { + for (const version of versions) { + const update = await fetchLegacyDoc(room.docid, version.versionId); + if (update == null) continue; // deleted between listing and read + bytes += update.byteLength; + try { + // Decode before applying. applyUpdate throwing part-way through would + // leave the accumulating doc in an undefined state, and this is the + // same lazy structural scan it would fail on. + Y.createContentIdsFromUpdate(update); + } catch (err) { + // Skip an unreadable snapshot rather than failing the document: every + // later version is a full snapshot, so its content still arrives — only + // this timeline entry is lost, and a corrupt version has nothing else + // to give. + skipped++; + migrationLog.warn( + { + event: 'full.version-skipped', + err, + docid: room.docid, + versionId: version.versionId, + }, + 'legacy s3 version is not a valid yjs update; skipping it', + ); + continue; + } + Y.applyUpdate(ydoc, update); + // `true`: inserts include content the doc has already deleted, so this is + // the full structural snapshot rather than what is currently visible + const all = Y.createContentIdsFromDoc(ydoc, true); + const fresh = Y.excludeContentIds(all, seen); + seen = all; + if ( + fresh.inserts.clients.size === 0 && + fresh.deletes.clients.size === 0 + ) { + continue; // this snapshot added nothing new + } + // Legacy snapshots carry no author, so every version is attributed to the + // 'system' identity (as the lazy seed is). The timestamp is what makes an + // entry identifiable: it is the version's S3 `LastModified`, so activity + // entries line up with the backend's version listing by time. + const attrs = (verb) => [ + Y.createContentAttribute(verb, 'system'), + Y.createContentAttribute(`${verb}At`, version.timestamp), + ]; + contentmaps.push( + Y.createContentMapFromContentIds( + fresh, + attrs('insert'), + attrs('delete'), + ), + ); + } + if (contentmaps.length === 0) { + // every version was unreadable or contentless — nothing to store, and + // nothing to remember either, so a later run can still pick it up + return { + status: 'nothing', + versions: versions.length, + applied: 0, + skipped, + dropped, + bytes, + durationMs: Date.now() - start, + }; + } + const nongcDoc = Y.encodeStateAsUpdate(ydoc); + await yhub.persistence.store(room, { + lastClock: '0', + gcDoc: await yhub.computePool.mergeUpdates(true, [nongcDoc], { room }), + nongcDoc, + contentmap: Y.encodeContentMap(Y.mergeContentMaps(contentmaps)), + contentids: Y.encodeContentIds(seen), + }); + } finally { + ydoc.destroy(); + } + await redis.sAdd(migratedSetKey(yhub), room.docid); + const result = { + status: 'ok', + versions: versions.length, + applied: contentmaps.length, + skipped, + dropped, + bytes, + durationMs: Date.now() - start, + }; + migrationLog.info( + { event: 'full.ok', docid: room.docid, ...result }, + 'stored document history from s3 versions', + ); + return result; +}; diff --git a/src/yhub-server/package-lock.json b/src/yhub-server/package-lock.json index 814f8e8a02..b200dc4317 100644 --- a/src/yhub-server/package-lock.json +++ b/src/yhub-server/package-lock.json @@ -6,7 +6,8 @@ "": { "name": "yhub-server", "dependencies": { - "@y/hub": "0.4.0", + "@y/hub": "0.5.0", + "@y/y": "14.0.0-rc.24", "jose": "6.2.8", "minio": "8.0.7" }, @@ -111,15 +112,15 @@ "license": "ISC" }, "node_modules/@y/hub": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@y/hub/-/hub-0.4.0.tgz", - "integrity": "sha512-LejJTSrBt86DI88pMO8rck4tQcHHR1SPLrZrOIfb5Ws/zNd601x5SXX0MOuTwKqoNOweT4esUX0Eh/2ER4sSsA==", + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/@y/hub/-/hub-0.5.0.tgz", + "integrity": "sha512-hMpSC3R+TvV13qPRHsSy4D+vyoSvrJah7ZoGaYtwPnHh92Ce53ufEb678SVZMN2susjKNyHUNrjoFIB7d6DjWg==", "license": "AGPL-3.0 OR PROPRIETARY", "dependencies": { "@y-crdt/yn": "^0.1.4", "@y/protocols": "^1.0.6-rc.1", "@y/y": "^14.0.0-rc.24", - "lib0": "^1.0.0-rc.23", + "lib0": "^1.0.0-rc.25", "minio": "^8.0.6", "pino": "^10.3.1", "postgres": "^3.4.3", @@ -335,9 +336,9 @@ } }, "node_modules/lib0": { - "version": "1.0.0-rc.23", - "resolved": "https://registry.npmjs.org/lib0/-/lib0-1.0.0-rc.23.tgz", - "integrity": "sha512-JPomcbwgKoTIDoXP61DFZV+Yvkw8bCyQhr9QYxps0fmHzsliEw+mhbUP1/nyVmk6ugzapJw1hUpFEMFRA4sRIg==", + "version": "1.0.0-rc.25", + "resolved": "https://registry.npmjs.org/lib0/-/lib0-1.0.0-rc.25.tgz", + "integrity": "sha512-UVxr56D1kVTx8P2Om5cAWXg2Pawk7JiOkCio+GLwlHcMko3FRE5xklOKB+t+fZSLA4ZLcVTZUKEIXPd9cRhEJg==", "license": "MIT", "bin": { "0ecdsa-generate-keypair": "src/bin/0ecdsa-generate-keypair.js", diff --git a/src/yhub-server/package.json b/src/yhub-server/package.json index e0129e4904..485e6e8fd1 100644 --- a/src/yhub-server/package.json +++ b/src/yhub-server/package.json @@ -6,7 +6,8 @@ "start": "node server.js" }, "dependencies": { - "@y/hub": "0.4.0", + "@y/hub": "0.5.0", + "@y/y": "14.0.0-rc.24", "jose": "6.2.8", "minio": "8.0.7" }, diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index ac782202da..472da12f22 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -1,15 +1,21 @@ -import { createHash, randomUUID } from 'node:crypto'; -import { readFileSync } from 'node:fs'; +import { createHash } from 'node:crypto'; -import { createApiEndpoint, createAuthPlugin, createYHub, logger } from '@y/hub'; +import { + apiError, + createApiEndpoint, + createAuthPlugin, + createYHub, +} from '@y/hub'; import { createRemoteJWKSet, jwtVerify } from 'jose'; -import { Client as S3Client } from 'minio'; -// mirror y-provider's env.ts secret-file support -const secret = (name, dflt) => - process.env[`${name}_FILE`] - ? readFileSync(process.env[`${name}_FILE`], 'utf8').trim() - : process.env[name] || dflt; +import { secret } from './env.js'; +// legacy Django/S3 document store — see migration.js and README.md +import { + SOFT_MIGRATION, + fullMigrate, + isTransientFailure, + maybeMigrate, +} from './migration.js'; const PORT = Number(process.env.PORT || 3002); const REDIS = process.env.REDIS; @@ -44,61 +50,6 @@ const EMPTY_YDOC = new Uint8Array([0, 0]); // websocket path. const MAX_CREATE_BYTES = 10 * 1024 * 1024; -// Soft migration (see README.md): legacy documents live in Django's S3 media -// bucket as UTF-8 base64 of a raw Yjs update at key `{docid}/file`. With -// SOFT_MIGRATION=true, the first access to a room yhub does not know yet -// fetches that object, seeds the room with it (attributed to 'system'), and -// only then admits the connection. -const SOFT_MIGRATION = process.env.SOFT_MIGRATION === 'true'; -const AWS_S3_ENDPOINT_URL = process.env.AWS_S3_ENDPOINT_URL; -const AWS_S3_ACCESS_KEY_ID = secret('AWS_S3_ACCESS_KEY_ID'); -const AWS_S3_SECRET_ACCESS_KEY = secret('AWS_S3_SECRET_ACCESS_KEY'); -const AWS_S3_REGION_NAME = process.env.AWS_S3_REGION_NAME; -// Django's default bucket name (impress settings.py) — prod overrides it -const AWS_STORAGE_BUCKET_NAME = - process.env.AWS_STORAGE_BUCKET_NAME || 'impress-media-storage'; -// base64 inflates 3 bytes to 4 — cap the streamed read at the encoded size of -// MAX_CREATE_BYTES (the same effective limit as create-ydoc) plus padding slack -const MAX_LEGACY_B64_BYTES = Math.ceil(MAX_CREATE_BYTES / 3) * 4 + 1024; -const S3_FETCH_TIMEOUT_MS = 10000; -const MIGRATE_LOCK_TTL_MS = 30000; -const MAX_CONCURRENT_SEEDS = 4; - -if ( - SOFT_MIGRATION && - (!AWS_S3_ENDPOINT_URL || !AWS_S3_ACCESS_KEY_ID || !AWS_S3_SECRET_ACCESS_KEY) -) { - // fail at boot instead of as an opaque 401 storm on first connect - throw new Error( - 'SOFT_MIGRATION=true requires AWS_S3_ENDPOINT_URL, AWS_S3_ACCESS_KEY_ID and AWS_S3_SECRET_ACCESS_KEY', - ); -} -const s3 = SOFT_MIGRATION - ? (() => { - const url = new URL(AWS_S3_ENDPOINT_URL); - if (url.pathname !== '/' && url.pathname !== '') { - // boto3 accepts path-prefixed endpoints but the minio client cannot - // address a base path — dropping it silently would probe the wrong - // keys and "migrate" every doc as empty - throw new Error('AWS_S3_ENDPOINT_URL must not contain a path'); - } - return new S3Client({ - endPoint: url.hostname, - port: - url.port !== '' - ? Number(url.port) - : url.protocol === 'https:' - ? 443 - : 80, - useSSL: url.protocol === 'https:', - accessKey: AWS_S3_ACCESS_KEY_ID, - secretKey: AWS_S3_SECRET_ACCESS_KEY, - ...(AWS_S3_REGION_NAME ? { region: AWS_S3_REGION_NAME } : {}), - }); - })() - : null; -const migrationLog = logger.child({ module: 'soft-migration' }); - // Public keys verifying the RS256 admin tokens Django issues (JWTService). // Lazily fetched on first use; jose caches the keys and refetches on unknown // "kid", so Django can rotate the signing key without a yhub restart. @@ -122,273 +73,6 @@ const backendFetch = async (path, { cookie, origin }) => { return res.json(); }; -// Legacy Django document store: object `{docid}/file`, body = UTF-8 text that -// is the base64 encoding of a raw Yjs update. Returns null when the object -// does not exist — a document that never had content saved, e.g. brand new. -// Throws on any other failure (network, auth, timeout, oversize); corrupt -// base64 decodes leniently to garbage that patchYdoc later rejects. -const fetchLegacyDoc = async (docid) => { - let stream = null; - let cancelTimeout = () => {}; - // minio 8 takes no AbortSignal — race a timer that also destroys the body - // stream once reading, so a stalled transfer cannot hold the ws upgrade - const timeout = new Promise((_, reject) => { - const timer = setTimeout(() => { - const err = new Error( - `s3 fetch timed out after ${S3_FETCH_TIMEOUT_MS}ms`, - ); - err.transient = true; // a slow S3 may recover — cache the failure briefly - stream?.destroy(err); - reject(err); - }, S3_FETCH_TIMEOUT_MS); - cancelTimeout = () => clearTimeout(timer); - }); - try { - let objPromise; - try { - objPromise = s3.getObject(AWS_STORAGE_BUCKET_NAME, `${docid}/file`); - stream = await Promise.race([objPromise, timeout]); - } catch (err) { - if (err?.code === 'NoSuchKey') return null; - // if the timeout won the race, getObject may still resolve later — - // destroy the late-arriving response stream, otherwise its never-read - // socket leaks (minio 8 sets no request timeout and cannot abort) - objPromise?.then((s) => s.destroy(err), () => {}); - throw err; - } - const body = await Promise.race([ - new Promise((resolve, reject) => { - const chunks = []; - let received = 0; - stream.on('data', (chunk) => { - received += chunk.byteLength; - if (received > MAX_LEGACY_B64_BYTES) { - stream.destroy( - new Error(`legacy object exceeds the ${MAX_LEGACY_B64_BYTES}B cap`), - ); - return; - } - chunks.push(chunk); - }); - stream.on('error', reject); - stream.on('end', () => resolve(Buffer.concat(chunks))); - }), - timeout, - ]); - const decoded = Buffer.from(body.toString('utf8'), 'base64'); - if (decoded.byteLength > MAX_CREATE_BYTES) { - throw new Error( - `decoded legacy update (${decoded.byteLength}B) exceeds the ${MAX_CREATE_BYTES}B cap`, - ); - } - // compute-task schema requires an exact Uint8Array (lib0 compares the - // constructor) — re-view the Buffer without copying - return new Uint8Array( - decoded.buffer, - decoded.byteOffset, - decoded.byteLength, - ); - } finally { - cancelTimeout(); - } -}; - -// Quick existence check: does yhub already have content for this room? -// Sequenced cheapest-first: a persisted postgres row (bare SELECT, no blob -// columns; rows are never deleted, so a hit is always safe) — then the valkey -// stream (only ydoc:update:v1 counts: awareness and auth-check messages share -// the stream but carry no content) — then the SELECT again, which closes the -// store-before-trim compaction race (and the worst case of a miss is only a -// redundant, idempotent re-seed). -const ydocExists = async (room) => { - if ((await yhub.persistence.retrieveDoc(room, {})).lastClock !== '0') { - return true; - } - const streams = await yhub.stream.getMessages([{ room, clock: '0' }]); - if ((streams[0]?.messages ?? []).some((m) => m.type === 'ydoc:update:v1')) { - return true; - } - return (await yhub.persistence.retrieveDoc(room, {})).lastClock !== '0'; -}; - -// Per-docid migration verdicts, in-memory (per replica). 'exists' is monotone -// in normal operation — its TTL only bounds staleness after an operator -// manually wipes a room's yhub state (restart yhub after a wipe to drop the -// cache immediately). 'empty' (no S3 object) keeps never-edited docs and -// rechecks off S3; 'failed' breaks the retry-refetch storm a permanently -// corrupt object would otherwise sustain (y-websocket retries denied upgrades -// forever). -const VERDICT_TTL_MS = { exists: 600000, empty: 60000, failed: 300000 }; -// transient failures (network blips, timeouts, S3 restarting) are cached just -// long enough to blunt a retry storm without turning a hiccup into a lockout -const TRANSIENT_TTL_MS = 15000; -const TRANSIENT_CODES = new Set([ - 'ECONNREFUSED', - 'ECONNRESET', - 'ETIMEDOUT', - 'EHOSTUNREACH', - 'ENETUNREACH', - 'ENOTFOUND', - 'EAI_AGAIN', - 'EPIPE', -]); -const isTransient = (err) => - err?.transient === true || - TRANSIENT_CODES.has(err?.code) || - TRANSIENT_CODES.has(err?.cause?.code); -const VERDICT_CACHE_MAX = 50000; -const verdicts = new Map(); // docid -> { verdict, error, expires } -const rememberVerdict = (docid, verdict, error = null, ttl = VERDICT_TTL_MS[verdict]) => { - // delete-then-set keeps Map insertion order ≈ recency, so the FIFO eviction - // drops the stalest entry — and re-setting an existing docid never evicts - // an unrelated one - if (!verdicts.delete(docid) && verdicts.size >= VERDICT_CACHE_MAX) { - verdicts.delete(verdicts.keys().next().value); - } - verdicts.set(docid, { - verdict, - error, - expires: Date.now() + ttl, - }); -}; -const inflightMigrations = new Map(); // docid -> Promise -let activeSeeds = 0; - -const migrate = async (room) => { - if (await ydocExists(room)) return 'exists'; - // collapse cross-replica herds: one seeder per room, the rest wait and - // re-probe. The key sits outside yhub's scanned `:room:*` patterns. - const lockKey = `${REDIS_PREFIX}:softmigrate:${room.org}:${room.docid}:${room.branch}`; - const lockToken = randomUUID(); - const redis = yhub.stream.redis; - const acquired = await redis.set(lockKey, lockToken, { - condition: 'NX', - expiration: { type: 'PX', value: MIGRATE_LOCK_TTL_MS }, - }); - try { - if (acquired == null) { - // another connection or replica is seeding — wait for its lock, then - // re-probe. If the doc is still absent (the holder crashed or its S3 - // fetch failed), fall through and seed ourselves: duplicate seeds use - // byte-identical updates from one lineage and merge as CRDT no-ops. - const deadline = Date.now() + MIGRATE_LOCK_TTL_MS + 5000; - while (Date.now() < deadline && (await redis.exists(lockKey)) === 1) { - await new Promise((resolve) => setTimeout(resolve, 300)); - } - if (await ydocExists(room)) return 'exists'; - } - if (activeSeeds >= MAX_CONCURRENT_SEEDS) { - // fail fast under a herd of distinct cold docs — the client's retry - // backoff spreads the load. Probes above stay uncapped. noCache: - // momentary per-replica backpressure must deny once, not be cached as - // a failure — a slot frees up within seconds - const err = new Error('too many concurrent soft migrations'); - err.noCache = true; - throw err; - } - activeSeeds++; - try { - const start = Date.now(); - const update = await fetchLegacyDoc(room.docid); - if (update == null) { - migrationLog.info( - { event: 'seed.empty', docid: room.docid }, - 'no legacy s3 object; room starts empty', - ); - return 'empty'; - } - // legacy content has no per-user history — attribute it to 'system' - // (the admin-JWT identity), marked so audits can tell migrated content - // apart from other system writes - const result = await yhub.computePool.patchYdoc( - { - update, - currentDoc: EMPTY_YDOC, - userid: 'system', - customAttributions: [{ k: 'migration', v: 's3' }], - }, - { room }, - ); - if (result == null) { - // structurally valid but no effective content — nothing to seed - migrationLog.info( - { event: 'seed.empty', docid: room.docid }, - 'legacy s3 object has no effective content; room starts empty', - ); - return 'empty'; - } - await yhub.stream.addMessage(room, { - type: 'ydoc:update:v1', - contentmap: result.contentmap, - update: result.update, - }); - migrationLog.info( - { - event: 'seed.ok', - docid: room.docid, - bytes: update.byteLength, - durationMs: Date.now() - start, - }, - 'seeded legacy doc from s3', - ); - return 'exists'; - } finally { - activeSeeds--; - } - } finally { - if (acquired != null) { - // compare-and-delete: if this seed outlived the lock TTL, another - // seeder holds a fresh lock — a bare DEL would release it under them - redis - .eval( - "if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) end", - { keys: [lockKey], arguments: [lockToken] }, - ) - .catch(() => {}); - } - } -}; - -// Resolves when the room is usable (already known, freshly seeded, or -// legitimately empty); rejects to deny access. Idempotent and safe to -// re-enter — it also runs on rechecks and default-purpose REST calls. -const maybeMigrate = async (room) => { - const cached = verdicts.get(room.docid); - if (cached != null && cached.expires > Date.now()) { - if (cached.verdict === 'failed') throw cached.error; - return; - } - let migration = inflightMigrations.get(room.docid); - if (migration == null) { - migration = migrate(room) - .then( - (verdict) => rememberVerdict(room.docid, verdict), - (err) => { - // logged here (once per attempt) rather than per denied connection: - // cached failures deny without new logs until the verdict expires - migrationLog.error( - { event: 'seed.failed', err, docid: room.docid }, - 'soft migration failed; denying access', - ); - if (err?.noCache !== true) { - // transient failures get a short TTL so a hiccup cannot lock a - // doc out for the full poison-object window - rememberVerdict( - room.docid, - 'failed', - err, - isTransient(err) ? TRANSIENT_TTL_MS : VERDICT_TTL_MS.failed, - ); - } - throw err; - }, - ) - .finally(() => inflightMigrations.delete(room.docid)); - inflightMigrations.set(room.docid, migration); - } - return migration; -}; - const auth = createAuthPlugin({ // uws req is only valid synchronously — read headers AND query before first await. async readAuthInfo(req) { @@ -420,9 +104,20 @@ const auth = createAuthPlugin({ return payload.admin === true ? { userid: 'system', admin: true } : null; - } catch { - // bad signature / expired / wrong (or missing) audience / JWKS - // unreachable — fail closed + } catch (err) { + // jose tags every token-validation failure with an `ERR_J…` code (bad + // signature, expired, wrong or missing audience) — those are permanent, + // fail closed. A JWKS fetch that times out or never connects has no + // such code (or ERR_JWKS_TIMEOUT): the token may be perfectly valid and + // we simply cannot check it, so report it as retryable instead of + // accusing the caller of forging it. + if ( + err?.code === 'ERR_JWKS_TIMEOUT' || + typeof err?.code !== 'string' || + !err.code.startsWith('ERR_J') + ) { + throw apiError(503, 'Token verification keys are unavailable'); + } return null; } } @@ -437,11 +132,14 @@ const auth = createAuthPlugin({ return { userid: String(user.id), cookie, origin }; // MUST be string (yhub server.js:667) } catch (err) { // Only a genuine "not signed in" falls back to the anonymous identity. - // On backend failure (5xx/network) fail closed: a signed-in editor - // authorized under an anon userid would be invisible to the targeted - // reset-connections recheck (users: []) for the connection's - // whole lifetime. - if (err?.status !== 401 && err?.status !== 403) return null; + // On backend failure (5xx/network) still refuse to admit the connection — + // a signed-in editor authorized under an anon userid would be invisible + // to the targeted reset-connections recheck (users: []) for the + // connection's whole lifetime — but report it as retryable rather than as + // an authentication failure the client should give up on. + if (err?.status !== 401 && err?.status !== 403) { + throw apiError(503, 'Authentication backend is unavailable'); + } // anonymous (public docs): stable per-session id — random ids would mint a new // permanent attribution identity per reconnect const anon = createHash('sha256') @@ -467,8 +165,14 @@ const auth = createAuthPlugin({ let doc; try { doc = await backendFetch(`/api/v1.0/documents/${docid}/`, authInfo); - } catch { - return null; + } catch (err) { + // the backend answered "no": a real, permanent denial (403 Forbidden) + if (err?.status === 401 || err?.status === 403 || err?.status === 404) { + return null; + } + // it did not answer at all — say so, so the caller retries instead of + // reading a 5xx or a network blip as a permission decision + throw apiError(503, 'Document authorization backend is unavailable'); } if (!doc.abilities?.retrieve) { return null; @@ -480,12 +184,21 @@ const auth = createAuthPlugin({ // postgres and the stream from clock 0) is guaranteed to include the // seed. Runs only for authorized readers. A missing S3 object is the // brand-new-document case and allows an empty room; a real S3/compute - // failure denies access — an opaque 401 the client retries with - // backoff. + // failure denies access, either way already logged (once per attempt) + // inside maybeMigrate. try { - await maybeMigrate({ org, docid, branch }); - } catch { - return null; // already logged (once per attempt) in maybeMigrate + // `yhub` is declared at the bottom of this file — safe: auth callbacks + // only fire once the server is up, i.e. after that assignment + await maybeMigrate(yhub, { org, docid, branch }); + } catch (err) { + // A corrupt or oversized legacy object will fail the same way forever, + // so that denial is permanent (403). An S3 timeout, a network blip or + // momentary seed backpressure will not — 503 tells the caller to come + // back rather than to treat the document as unreadable. + if (isTransientFailure(err)) { + throw apiError(503, 'Legacy document store is unavailable'); + } + return null; } } return doc.abilities.update ? 'rw' : 'r'; @@ -527,6 +240,66 @@ const api = [ }, }, }), + // POST /collaboration/migrate/v1/{org}/{docid} — replay a document's full + // legacy version history from the S3 media bucket into yhub (see README.md). + // Backend-internal, like reset-connections: gated to the admin token via the + // 'migrate' access purpose, since it writes history and reads the legacy + // store. + createApiEndpoint('migrate', { + accessPurpose: 'migrate', + post: { + handler: async (req) => { + if (req.org !== ORG) { + return jsonResponse(400, { error: 'Unknown org' }); + } + if (!UUID4.test(req.docid)) { + return jsonResponse(400, { error: 'Room name is invalid' }); + } + if (req.branch !== 'main') { + // the legacy store is branchless: `{docid}/file` is the main branch + return jsonResponse(400, { error: 'Unknown branch' }); + } + if (!SOFT_MIGRATION) { + // the flag is what configures the S3 client (migration.js) + return jsonResponse(503, { error: 'Legacy store is not configured' }); + } + // ?force=true replays a document that is already in the migrated set. + // Only safe while its clock-0 row is still there: once compaction has + // folded that row away, a second replay attributes the same content a + // second time and the activity timestamps become ambiguous. + const { status, ...stats } = await fullMigrate(req.yhub, req.room, { + force: req.query.force === 'true', + }); + if (status === 'already') { + return jsonResponse(200, { + message: 'Already migrated', + migrated: false, + }); + } + if (status === 'empty') { + // brand-new documents never had a legacy object; nothing to replay + // and nothing wrong — a backfill driver treats this as done + return jsonResponse(200, { + message: 'No legacy document in s3', + migrated: false, + ...stats, + }); + } + if (status === 'nothing') { + return jsonResponse(200, { + message: 'No usable content in the legacy versions', + migrated: false, + ...stats, + }); + } + return jsonResponse(200, { + message: 'Migration completed', + migrated: true, + ...stats, + }); + }, + }, + }), // POST /collaboration/create-ydoc/v1/{org}/{docid} — create a document's // initial Yjs state from a RAW binary update (`Y.encodeStateAsUpdate` / // pycrdt `get_update()` output) posted as application/octet-stream. Unlike @@ -561,8 +334,9 @@ const api = [ body.byteLength, ); if (update.byteLength > MAX_CREATE_BYTES) { - // 413 is missing from yhub's status-line map, so the reason phrase - // is empty ("HTTP/1.1 413 ") — legal, and callers switch on the code + // 413 is missing from yhub's status-line map (503 was added in + // 0.5.0, 413 was not), so the reason phrase is empty + // ("HTTP/1.1 413 ") — legal, and callers switch on the code return jsonResponse(413, { error: 'Update too large' }); } // <= 3 bytes is yhub's "no effective content" convention (an empty @@ -630,8 +404,8 @@ const api = [ }), ]; -// the instance is referenced by the soft-migration helpers above — safe: auth -// callbacks only fire once the server is up, i.e. after this assignment +// referenced by getAccessType above — safe: auth callbacks only fire once the +// server is up, i.e. after this assignment const yhub = await createYHub({ redis: { url: REDIS, @@ -647,6 +421,6 @@ const yhub = await createYHub({ server: { port: PORT, auth, api, apiPrefix: 'collaboration' }, worker: { taskConcurrency: 5 }, // TODO(yhub): worker.events.docUpdate could push snapshots to Django and replace the - // client useSaveDoc PATCH flow — blocked upstream: the payload is a DocTable without - // room/org/docid (yhub src/index.js:90); needs an upstream change first. + // client useSaveDoc PATCH flow. No longer blocked upstream — yhub 0.5.0 adds `room` + // to the event payload, which was the missing piece. }); From 3869fea484c1e2440e7aea8302dc9d5c36135953 Mon Sep 17 00:00:00 2001 From: Kevin Jahns Date: Mon, 10 Aug 2026 20:30:57 +0200 Subject: [PATCH 24/59] =?UTF-8?q?=E2=9C=85(collaboration)=20test=20the=20l?= =?UTF-8?q?egacy=20migrations=20against=20a=20real=20yhub?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cover both paths off the legacy Django store end to end: the lazy seed on first access, and the migrate endpoint replaying every S3 version. The tests need no database — the admin JWT short-circuits document authorization, so a fixture is an S3 object on a random uuid — and read the timeline through yhub 0.5.0's `Accept: application/json`, which spares python a lib0 decoder. CI grows a valkey service and starts a collaboration server alongside the backend test job; the tests skip themselves when nothing answers on the new COLLABORATION_API_URL setting, so `make test` without the dev stack still passes. Writing them turned up three things worth fixing in the server. Backend reads now seed too. getAccessType short-circuited on the admin token before reaching the legacy store, so a server-side read of an unmigrated document answered with an empty one, and a create-ydoc against it would have written a second lineage beside the content the first user access was about to seed in. Seeding no longer decides access; the backend's answer alone does. A legacy object that cannot be migrated — it does not decode, or it exceeds the size we load — opens as a new document instead of denying, since no retry can fix it and refusing would leave the document unopenable by anyone. The cause is logged once per attempt with the bucket, key and stack, and every later access logs that it admitted a caller without migrating. That made the failure classifier dangerous, so it is inverted. It was an allowlist of retryable errors — eight socket errnos — which left every way S3 can refuse (AccessDenied on a rotated key, NoSuchBucket, a region redirect) counting as "this object is unusable". Denying, that was survivable; opening empty, one misscoped credential would fork every document touched during the window. Now only a failure raised while interpreting bytes we already hold is permanent, marked at the throw site, and everything else answers a retryable 503. Guessing wrong that way costs a retry; the other way costs the document. The admin seed is also fenced to the org and to main, like the user path above it. The legacy store is branchless — {docid}/file is main — and the bookkeeping is per document, so seeding ?branch=draft would have written main's content into an orphan room and left the real one permanently empty. Signed-off-by: Kevin Jahns --- .github/workflows/impress.yml | 66 +++ CHANGELOG.md | 18 +- env.d/development/common | 2 + .../tests/test_integration_yhub_migration.py | 391 ++++++++++++++++++ src/backend/impress/settings.py | 7 + src/yhub-server/README.md | 43 +- src/yhub-server/migration.js | 92 +++-- src/yhub-server/server.js | 92 +++-- 8 files changed, 630 insertions(+), 81 deletions(-) create mode 100644 src/backend/core/tests/test_integration_yhub_migration.py diff --git a/.github/workflows/impress.yml b/.github/workflows/impress.yml index b0c6fbc319..272f523eb4 100644 --- a/.github/workflows/impress.yml +++ b/.github/workflows/impress.yml @@ -128,6 +128,13 @@ jobs: # needed because the postgres container does not provide a healthcheck options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 + # message stream for the collaboration server (see the yhub steps below) + valkey: + image: valkey/valkey:alpine + ports: + - 6379:6379 + options: --health-cmd "valkey-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5 + env: DJANGO_CONFIGURATION: Test DJANGO_SETTINGS_MODULE: impress.settings @@ -142,6 +149,12 @@ jobs: AWS_S3_ENDPOINT_URL: http://localhost:9000 AWS_S3_ACCESS_KEY_ID: impress AWS_S3_SECRET_ACCESS_KEY: password + # Collaboration server. The integration tests reach it over this url and + # skip themselves when nothing answers; yhub reads the JWKS back from the + # django server started alongside it, so both sides must share + # JWT_PRIVATE_KEY_FILE. + COLLABORATION_API_URL: http://localhost:3002/collaboration + JWT_PRIVATE_KEY_FILE: ${{ github.workspace }}/data/jwt/private.pem steps: - name: Checkout repository @@ -203,8 +216,61 @@ jobs: sudo apt-get install -y gettext pandoc shared-mime-info sudo wget https://raw.githubusercontent.com/suitenumerique/django-lasuite/refs/heads/main/assets/conf/mime.types -O /etc/mime.types + # --- collaboration server ------------------------------------------- + # The yhub integration tests drive a real collaboration server: it reads + # legacy documents out of MinIO and reads this backend's JWKS back to + # verify the admin token the tests mint, so the two must share the + # signing key. Tests skip themselves when nothing answers on + # COLLABORATION_API_URL. + - name: Generate the JWT signing key + working-directory: . + run: bin/generate-jwt-private-key.sh + + - name: Create the collaboration server database + run: | + PGPASSWORD=pass psql -h localhost -U dinum -d impress \ + -c 'CREATE DATABASE yhub' + PGPASSWORD=pass psql -h localhost -U dinum -d yhub \ + -f ../../docker/files/yhub/initdb/01-yhub.sql + - name: Generate a MO file from strings extracted from the project run: uv run python manage.py compilemessages + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: "22.x" + + - name: Install the collaboration server + working-directory: src/yhub-server + run: npm ci --omit=dev + + - name: Start the backend for the collaboration server to authenticate against + env: + DJANGO_ALLOWED_HOSTS: "*" + run: | + nohup uv run python manage.py runserver 0.0.0.0:8000 --noreload \ + > /tmp/backend.log 2>&1 & + dockerize -wait http://localhost:8000/api/v1.0/jwks -timeout 60s + + - name: Start the collaboration server + working-directory: src/yhub-server + env: + PORT: 3002 + REDIS: redis://localhost:6379 + POSTGRES: postgres://dinum:pass@localhost:5432/yhub + REDIS_PREFIX: yhub + COLLABORATION_BACKEND_BASE_URL: http://localhost:8000 + COLLABORATION_SERVER_ORIGIN: http://localhost:3000 + AWS_STORAGE_BUCKET_NAME: impress-media-storage + SOFT_MIGRATION: "true" + run: | + nohup node server.js > /tmp/yhub.log 2>&1 & + dockerize -wait tcp://localhost:3002 -timeout 30s + - name: Run tests run: uv run pytest -n 2 + + - name: Collaboration server logs + if: failure() + run: cat /tmp/yhub.log /tmp/backend.log diff --git a/CHANGELOG.md b/CHANGELOG.md index 478b05a38d..a792d50e6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ and this project adheres to ### Added +- ✅(collaboration) add integration tests covering both legacy migrations, + running against a real yhub: CI now starts the collaboration server and a + valkey alongside the backend test job. They skip themselves when nothing + answers on `COLLABORATION_API_URL`, so a `make test` without the dev stack + still passes - ✨(collaboration) soft-migrate legacy S3 documents into yhub on first access (`SOFT_MIGRATION=true`): when yhub does not know a document yet, its legacy snapshot (`{id}/file`, base64 Yjs update) is fetched from the Django S3 @@ -15,9 +20,16 @@ and this project adheres to timestamp — a lazy seed is not an editing event, and stamping one would collide with the real per-version times the migrate endpoint writes) before the connection is admitted. A missing S3 object means a brand-new document and - yields an empty room; any real S3/compute failure fails closed, as a - permanent `403` for a corrupt or oversized object and a retryable `503` for - timeouts and network errors. Enabled in the dev stack via compose.yml + yields an empty room. Seeding never decides access: a legacy object that + cannot be migrated (undecodable or oversized) opens as a new document, logged + per access, since no retry could fix it and refusing would make the document + permanently unopenable. Every other failure — an unreachable store, but also + any refusal from S3 such as `AccessDenied` on a rotated key or a wrong bucket + name — answers a retryable `503`, so an empty document is never started on + top of content that exists. + Backend reads carrying the admin JWT are seeded too, so a server-side read of + an unmigrated document never answers with an empty one. Enabled in the dev + stack via compose.yml - ✨(collaboration) add a migrate endpoint on yhub: `POST /collaboration/migrate/v1/docs/{id}` replays a document's **full** legacy version history from the versioned S3 media bucket into a diff --git a/env.d/development/common b/env.d/development/common index 7a294177d3..8b406be286 100644 --- a/env.d/development/common +++ b/env.d/development/common @@ -81,6 +81,8 @@ COLLABORATION_BACKEND_BASE_URL=http://app-dev:8000 COLLABORATION_SERVER_ORIGIN=http://localhost:3000 COLLABORATION_WS_URL=ws://localhost:3002/collaboration/ws/v1/docs COLLABORATION_WS_INACTIVITY_TIMEOUT=15 # Seconds +# server-to-server, reached with an admin JWT (aud: yhub) +COLLABORATION_API_URL=http://yhub:3002/collaboration DJANGO_SERVER_TO_SERVER_API_TOKENS=server-api-token Y_PROVIDER_API_BASE_URL=http://y-provider-development-converter:4444/api/ diff --git a/src/backend/core/tests/test_integration_yhub_migration.py b/src/backend/core/tests/test_integration_yhub_migration.py new file mode 100644 index 0000000000..36f06c053f --- /dev/null +++ b/src/backend/core/tests/test_integration_yhub_migration.py @@ -0,0 +1,391 @@ +""" +Integration tests for the migration of legacy documents into the collaboration +server (yhub). + +These talk to a *running* yhub, and through it to MinIO. They need no database: +the admin JWT short-circuits yhub's document authorization, so nothing here +creates a ``Document`` row — the fixtures are S3 objects and yhub rooms keyed on +a random uuid. + +Two paths are covered, in the order a real corpus goes through them: + +``soft migration`` + yhub does not know the room, so the first read seeds it from the *newest* + S3 version. The seed carries an author but deliberately no timestamp, so it + contributes no activity entry. + +``full migration`` + ``POST /migrate`` replays *every* S3 version, crediting each with its own S3 + ``LastModified``, so the activity api reports the same timeline as the + backend's ``/documents/{id}/versions/``. +""" + +import base64 +import itertools +import uuid + +from django.conf import settings +from django.core.files.base import ContentFile +from django.core.files.storage import default_storage + +import pytest +import requests + +from core.services.jwt_services import JWTService + +# yhub verifies that its own name is the audience of the token (server.js) +YHUB_AUDIENCE = "yhub" +# the org yhub is configured with; documents live under /docs/{docid} +YHUB_ORG = "docs" +TIMEOUT = 10 +# see _activity: distinct values defeat yhub's few-second response cache +_cache_buster = itertools.count(1000) + + +def _yhub_url(): + """Base url of the collaboration api, or None when it is not configured.""" + return (settings.COLLABORATION_API_URL or "").rstrip("/") or None + + +def _yhub_reachable(): + """Is a collaboration server actually listening? These tests need one.""" + url = _yhub_url() + if url is None: + return False + try: + # any route answers *something*; we only care that the port is served + requests.get(f"{url}/activity/v1/{YHUB_ORG}/nope", timeout=2) + except requests.RequestException: + return False + return True + + +pytestmark = pytest.mark.skipif( + not _yhub_reachable(), + reason=( + "needs a running collaboration server (COLLABORATION_API_URL); " + "start the dev stack with `make run`" + ), +) + + +@pytest.fixture(name="admin_headers") +def admin_headers_fixture(settings): # pylint: disable=redefined-outer-name + """ + Authorization for the backend-to-yhub calls. + + The token is signed with the same key the running yhub validates against + (it fetches the JWKS from this backend), so this only works when both sides + share ``JWT_PRIVATE_KEY`` — which they do in the dev stack and in CI. + """ + if not settings.JWT_PRIVATE_KEY: + pytest.skip("JWT_PRIVATE_KEY is not configured") + token = JWTService().get_admin_token({"aud": YHUB_AUDIENCE}) + return {"Authorization": f"Bearer {token}"} + + +def _write_legacy_versions(docid, updates): + """ + Write `updates` as successive versions of the legacy object `{docid}/file`. + + Mirrors what the Django backend used to do on every content save: the body + is the base64 encoding of a raw Yjs update, and the bucket is versioned, so + each write leaves the previous one behind as an object version. + + Returns the versions oldest first, as (version_id, last_modified). + """ + key = f"{docid}/file" + for update in updates: + default_storage.save(key, ContentFile(base64.b64encode(update))) + + client = default_storage.connection.meta.client + response = client.list_object_versions( + Bucket=default_storage.bucket_name, Prefix=key + ) + versions = [v for v in response.get("Versions", []) if v["Key"] == key] + versions.sort(key=lambda v: v["LastModified"]) + return [(v["VersionId"], v["LastModified"]) for v in versions] + + +def _activity(docid, admin_headers, **params): + """ + The document's activity, one entry per change, oldest first. + + ``Accept: application/json`` opts out of yhub's lib0-any encoding (0.5.0), + which is what lets a python caller read the timeline without a decoder. + ``group=false`` keeps one entry per version: the default grouping merges + changes by the same author less than a second apart. + + ``groupMaxGap`` is a cache buster, not a parameter we care about: yhub + caches activity responses for a few seconds keyed on the full query, and a + test reads the timeline immediately after changing it. With ``group=false`` + the value is never read (``groupDistance = group ? groupMaxGap : 1``), so a + unique one buys a fresh computation without touching the result. + """ + response = requests.get( + f"{_yhub_url()}/activity/v1/{YHUB_ORG}/{docid}", + params={ + "group": "false", + "groupMaxGap": next(_cache_buster), + **params, + }, + headers={**admin_headers, "Accept": "application/json"}, + timeout=TIMEOUT, + ) + assert response.status_code == 200, response.text + assert response.headers["content-type"].startswith("application/json") + return response.json()["activity"] + + +def _read_ydoc(docid, admin_headers): + """Read the room, which is also what triggers the lazy soft migration.""" + return requests.get( + f"{_yhub_url()}/ydoc/v1/{YHUB_ORG}/{docid}", + headers=admin_headers, + timeout=TIMEOUT, + ) + + +def _ydoc_bytes(docid, admin_headers): + """ + The room's Yjs state as raw bytes. + + The response is an envelope around the document, so its size says nothing + on its own; ``Accept: application/json`` renders the field as base64, which + is comparable. + """ + response = requests.get( + f"{_yhub_url()}/ydoc/v1/{YHUB_ORG}/{docid}", + headers={**admin_headers, "Accept": "application/json"}, + timeout=TIMEOUT, + ) + assert response.status_code == 200, response.text + return base64.b64decode(response.json()["doc"]) + + +def _migrate(docid, admin_headers, **params): + """Replay the document's full legacy version history into yhub.""" + return requests.post( + f"{_yhub_url()}/migrate/v1/{YHUB_ORG}/{docid}", + params=params, + headers={**admin_headers, "Accept": "application/json"}, + timeout=60, + ) + + +# Three successive snapshots of one Yjs document, as the legacy store held them: +# each is a full `Y.encodeStateAsUpdate` of the same doc after another insert, +# so they share a lineage and every later one is a superset of the last. +# Generated with @y/y; hardcoded so the tests need no javascript. +LEGACY_SNAPSHOTS = [ + base64.b64decode(b64) + for b64 in ( + "AQG8yr6Vi7ATAAQBDmRvY3VtZW50LXN0b3JlCkFMUEhBLW9uZSAA", + "AQG8yr6Vi7ATAAQBDmRvY3VtZW50LXN0b3JlFEFMUEhBLW9uZSBCUkFWTy10d28gAA==", + "AQG8yr6Vi7ATAAQBDmRvY3VtZW50LXN0b3JlIkFMUEhBLW9uZSBCUkFWTy10d28gQ0hBUkxJRS10aHJlZSAA", + ) +] + + +def test_integration_yhub_soft_migration_seeds_without_a_timestamp(admin_headers): + """ + The first read of an unknown room seeds it from the newest legacy version. + + The seed is a migration artifact, not an editing event: it has no honest + time to report, so it writes no `insertAt` and therefore shows up in no + activity entry. Anything else would put a second, meaningless timestamp on + content the full migration is about to date properly. + """ + docid = str(uuid.uuid4()) + _write_legacy_versions(docid, LEGACY_SNAPSHOTS) + + response = _read_ydoc(docid, admin_headers) + + assert response.status_code == 200, response.text + # seeded, so the room is no longer empty (an empty update is 2 bytes) + assert len(response.content) > 3 + assert _activity(docid, admin_headers) == [] + + +def test_integration_yhub_soft_migration_admits_when_it_cannot_migrate(admin_headers): + """ + A legacy object that cannot be decoded must not lock its document. + + Nobody can repair such an object from the outside, so refusing access would + make the document permanently unopenable. It opens as a new one instead — + the legacy bytes stay in S3, and the server logs that it admitted a caller + without migrating. + """ + docid = str(uuid.uuid4()) + _write_legacy_versions(docid, [b"@@not-a-valid-ydoc@@"]) + + response = _read_ydoc(docid, admin_headers) + + assert response.status_code == 200, response.text + # and what it opens is exactly what a document that never existed opens as + never_existed = str(uuid.uuid4()) + assert _ydoc_bytes(docid, admin_headers) == _ydoc_bytes( + never_existed, admin_headers + ) + assert _activity(docid, admin_headers) == [] + + +def test_integration_yhub_soft_migration_ignores_a_non_main_branch(admin_headers): + """ + Seeding a branch other than main must not consume the document's one seed. + + The legacy store is branchless — ``{docid}/file`` *is* main — and the admin + token is the only identity that can name another branch. Seeding one would + write main's content into an orphan room and, because the "already seeded" + bookkeeping is per document, leave the real room empty. + """ + docid = str(uuid.uuid4()) + _write_legacy_versions(docid, LEGACY_SNAPSHOTS) + + on_a_branch = requests.get( + f"{_yhub_url()}/ydoc/v1/{YHUB_ORG}/{docid}", + params={"branch": "draft"}, + headers=admin_headers, + timeout=TIMEOUT, + ) + assert on_a_branch.status_code == 200, on_a_branch.text + + # main is untouched by that, so it still seeds on its own first read + assert _read_ydoc(docid, admin_headers).status_code == 200 + assert _ydoc_bytes(docid, admin_headers) != _ydoc_bytes( + str(uuid.uuid4()), admin_headers + ) + + +def test_integration_yhub_full_migration_reports_every_s3_version(admin_headers): + """ + The migrate endpoint replays every legacy version, dated by that version. + + This is the property the whole feature exists for: activity and the + backend's version listing describe the same timeline. + """ + docid = str(uuid.uuid4()) + versions = _write_legacy_versions(docid, LEGACY_SNAPSHOTS) + + response = _migrate(docid, admin_headers) + + assert response.status_code == 200, response.text + body = response.json() + assert body["migrated"] is True + assert body["versions"] == len(versions) + assert body["applied"] == len(versions) + assert body["skipped"] == 0 + assert body["dropped"] == 0 + # the decoded size of every snapshot it read, not the base64 on the wire + assert body["bytes"] == sum(len(update) for update in LEGACY_SNAPSHOTS) + + activity = _activity(docid, admin_headers) + + assert len(activity) == len(versions) + for entry, (_, last_modified) in zip(activity, versions, strict=True): + # yhub stores timestamps in milliseconds (lib0 `getUnixTime` is + # `Date.now`), which is what the S3 write time converts to + assert entry["from"] == pytest.approx(last_modified.timestamp() * 1000, abs=1) + assert entry["from"] == entry["to"] + # legacy snapshots carry no author of their own + assert entry["by"] == "system" + + +def test_integration_yhub_full_migration_after_a_soft_migration(admin_headers): + """ + The two migrations compose: seeding first does not duplicate the history. + + The seed writes the legacy bytes unchanged, so its content ids are the ones + the replay regenerates — the replay covers them and, carrying no timestamp + of its own, the seed adds no entry beside them. + """ + docid = str(uuid.uuid4()) + versions = _write_legacy_versions(docid, LEGACY_SNAPSHOTS) + + assert _read_ydoc(docid, admin_headers).status_code == 200 + assert _activity(docid, admin_headers) == [] + + assert _migrate(docid, admin_headers).status_code == 200 + + activity = _activity(docid, admin_headers) + assert len(activity) == len(versions) + assert [entry["from"] for entry in activity] == sorted( + entry["from"] for entry in activity + ) + + +def test_integration_yhub_full_migration_is_idempotent(admin_headers): + """ + A document is migrated once, ever. + + Replaying a second time would attribute the same content twice, so the + docid is remembered in a valkey set and later calls decline. `?force=true` + is the escape hatch, and the clock-0 row it writes conflicts with the first + one, so even that leaves the timeline alone. + """ + docid = str(uuid.uuid4()) + versions = _write_legacy_versions(docid, LEGACY_SNAPSHOTS) + + assert _migrate(docid, admin_headers).json()["migrated"] is True + + again = _migrate(docid, admin_headers) + assert again.status_code == 200 + assert again.json() == {"message": "Already migrated", "migrated": False} + + forced = _migrate(docid, admin_headers, force="true") + assert forced.json()["migrated"] is True + + assert len(_activity(docid, admin_headers)) == len(versions) + + +def test_integration_yhub_migration_without_a_legacy_document(admin_headers): + """ + A document that never had a legacy object is nothing to migrate. + + It answers 2xx all the same, so a backfill driver walking the corpus can + treat every success as "done" without special-casing new documents. + """ + response = _migrate(str(uuid.uuid4()), admin_headers) + + assert response.status_code == 200 + body = response.json() + assert body["migrated"] is False + assert body["message"] == "No legacy document in s3" + assert body["versions"] == 0 + + +def test_integration_yhub_migration_skips_an_unreadable_version(admin_headers): + """ + One corrupt snapshot must not cost the document its whole history. + + Every version is a full snapshot, so the content of an unreadable one + arrives with the next readable version anyway — only its timeline entry is + lost, and the migration reports how many it dropped that way. + """ + docid = str(uuid.uuid4()) + _write_legacy_versions( + docid, + [LEGACY_SNAPSHOTS[0], b"@@not-a-valid-ydoc@@", LEGACY_SNAPSHOTS[2]], + ) + + body = _migrate(docid, admin_headers).json() + + assert body["migrated"] is True + assert body["versions"] == 3 + assert body["skipped"] == 1 + assert body["applied"] == 2 + assert len(_activity(docid, admin_headers)) == 2 + + +def test_integration_yhub_migration_rejects_a_token_for_another_audience(): + """ + An admin token minted for another service must not be replayable here. + + Django's JWTService signs for whoever asks, so the audience is the only + thing separating the converter's token from yhub's. + """ + token = JWTService().get_admin_token({"aud": "y-converter"}) + + response = _migrate(str(uuid.uuid4()), {"Authorization": f"Bearer {token}"}) + + assert response.status_code == 401 diff --git a/src/backend/impress/settings.py b/src/backend/impress/settings.py index 4c4823da31..62a3e1f188 100755 --- a/src/backend/impress/settings.py +++ b/src/backend/impress/settings.py @@ -520,6 +520,13 @@ class Base(Configuration): environ_name="COLLABORATION_WS_INACTIVITY_TIMEOUT", environ_prefix=None, ) + # Base url of the collaboration server's REST api, including its route + # prefix (e.g. "http://yhub:3002/collaboration"). Server-to-server only: + # used with an admin JWT to migrate legacy documents and, later, to kick + # connections when permissions change. + COLLABORATION_API_URL = values.Value( + None, environ_name="COLLABORATION_API_URL", environ_prefix=None + ) # JWT # RSA private key (PEM) used to sign the tokens issued by diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index a980f437e6..4be057c335 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -59,7 +59,9 @@ bucket, as UTF-8 text that is the base64 encoding of a raw Yjs update, at key `{document-uuid}/file`. With `SOFT_MIGRATION=true`, this server migrates those documents into yhub lazily, on first access: -1. After a user's document authorization succeeds, the auth plugin checks +1. After a caller's document authorization succeeds — a user's, or the + backend's own admin JWT, so a server-side read never sees an *empty* + document where legacy content exists — the auth plugin checks whether yhub already has content for the room — the migrated set written by the full migration (below), then a bare postgres `SELECT` (persisted rows), then the valkey stream (uncompacted `ydoc:update:v1` messages), then the @@ -94,16 +96,35 @@ Guarantees and failure behavior: - **Missing S3 object is not an error** — that is the brand-new-document case (Django writes no object until the first content save); the room simply starts empty. -- **Everything else fails closed**, and says whether it is worth retrying - (yhub 0.5.0 error semantics): an oversized or corrupt legacy object will fail - the same way forever, so the connection is denied `403`; network errors, - timeouts and momentary seed backpressure answer `503`, which clients retry - with backoff. Either way a `soft-migration` error is logged. A cached failure - verdict prevents retry storms from hammering S3 — permanent failures - (corrupt/oversized objects) for 5 minutes, transient ones (network errors, - timeouts) for 15 seconds, and per-replica seed backpressure (more than 20 - concurrent seeds) is denied without caching so the client's next retry goes - through. +- **Seeding never decides access** — the backend's answer does. What a failure + changes is only what the room contains, and the two kinds are treated + differently (yhub 0.5.0 error semantics): + - **The legacy object cannot be migrated** — it does not decode, or it + exceeds the size we will load. Retrying cannot change that, and nobody can + repair the object from the outside, so refusing would make the document + permanently unopenable. It opens as a *new* document instead. The cause is + logged once per attempt (`seed.failed`, with the bucket, key and stack) and + every subsequent access logs a `seed.skipped` warning, because the caller + is now editing beside legacy content that stayed behind in S3. + - **Everything else** — network error, timeout, seed backpressure, and every + way S3 can refuse (`AccessDenied` on a rotated key, `NoSuchBucket` on a + misconfigured name, a region redirect). The same request later may well + succeed, so it answers `503` and clients retry with backoff. + + The split is deliberately asymmetric: only a failure raised while + *interpreting bytes we already hold* counts as permanent, and it is marked as + such at the throw site. Everything else is retryable by default. An allowlist + of retryable errors would have to enumerate every way the store can say no, + and each case it missed would be read as "this document has no content" and + open the room empty over content that is alive in S3 — one misscoped + credential would fork the corpus. Guessing wrong this way costs a retry; + guessing wrong the other way costs the document. + + A cached failure verdict prevents retry storms from hammering S3 — permanent + failures (corrupt/oversized objects) for 5 minutes, transient ones (network + errors, timeouts) for 15 seconds, and per-replica seed backpressure (more + than 20 concurrent seeds) is not cached at all, so the client's next retry + goes through. - **Seeding is idempotent**: the legacy S3 snapshots are frozen (the frontend no longer PATCHes content snapshots to Django) and share one Yjs lineage with everything in yhub, so duplicate or concurrent seeds merge as diff --git a/src/yhub-server/migration.js b/src/yhub-server/migration.js index 5e42fee898..70401a5c0b 100644 --- a/src/yhub-server/migration.js +++ b/src/yhub-server/migration.js @@ -91,7 +91,9 @@ const s3 = SOFT_MIGRATION }); })() : null; -const migrationLog = logger.child({ module: 'soft-migration' }); +// exported so the auth path can report, under the same module name, that it +// admitted a caller to a document it could not migrate +export const migrationLog = logger.child({ module: 'soft-migration' }); // Both keys are derived from the prefix yhub itself resolved, so they cannot // drift from the room keys, and both sit outside its scanned `:room:*` pattern. @@ -118,10 +120,10 @@ const fetchLegacyDoc = async (docid, versionId = null) => { // stream once reading, so a stalled transfer cannot hold the ws upgrade const timeout = new Promise((_, reject) => { const timer = setTimeout(() => { + // unmarked, so it counts as retryable: a slow S3 may recover const err = new Error( `s3 fetch timed out after ${S3_FETCH_TIMEOUT_MS}ms`, ); - err.transient = true; // a slow S3 may recover — cache the failure briefly stream?.destroy(err); reject(err); }, S3_FETCH_TIMEOUT_MS); @@ -159,11 +161,11 @@ const fetchLegacyDoc = async (docid, versionId = null) => { stream.on('data', (chunk) => { received += chunk.byteLength; if (received > MAX_LEGACY_B64_BYTES) { - stream.destroy( - new Error( - `legacy object exceeds the ${MAX_LEGACY_B64_BYTES}B cap`, - ), + const err = new Error( + `legacy object exceeds the ${MAX_LEGACY_B64_BYTES}B cap`, ); + err.permanent = true; // the object will be this big next time too + stream.destroy(err); return; } chunks.push(chunk); @@ -175,9 +177,11 @@ const fetchLegacyDoc = async (docid, versionId = null) => { ]); const decoded = Buffer.from(body.toString('utf8'), 'base64'); if (decoded.byteLength > MAX_LEGACY_BYTES) { - throw new Error( + const err = new Error( `decoded legacy update (${decoded.byteLength}B) exceeds the ${MAX_LEGACY_BYTES}B cap`, ); + err.permanent = true; // the object will be this big next time too + throw err; } // compute-task schema requires an exact Uint8Array (lib0 compares the // constructor) — re-view the Buffer without copying @@ -205,7 +209,6 @@ const listLegacyVersions = async (docid) => { const err = new Error( `s3 version listing timed out after ${S3_LIST_TIMEOUT_MS}ms`, ); - err.transient = true; stream.destroy(err); }, S3_LIST_TIMEOUT_MS); stream.on('data', (obj) => { @@ -265,26 +268,17 @@ const VERDICT_TTL_MS = { exists: 600000, empty: 60000, failed: 300000 }; // transient failures (network blips, timeouts, S3 restarting) are cached just // long enough to blunt a retry storm without turning a hiccup into a lockout const TRANSIENT_TTL_MS = 15000; -const TRANSIENT_CODES = new Set([ - 'ECONNREFUSED', - 'ECONNRESET', - 'ETIMEDOUT', - 'EHOSTUNREACH', - 'ENETUNREACH', - 'ENOTFOUND', - 'EAI_AGAIN', - 'EPIPE', -]); -// Will this failure plausibly resolve on its own? Callers use it twice: to pick -// the verdict TTL below, and — in server.js — to decide whether a denied -// connection reports a permanent 403 or a retryable 503. `noCache` is the -// per-replica seed backpressure, transient by construction (a slot frees up -// within seconds). -export const isTransientFailure = (err) => - err?.transient === true || - err?.noCache === true || - TRANSIENT_CODES.has(err?.code) || - TRANSIENT_CODES.has(err?.cause?.code); +// Is this legacy object beyond saving, as opposed to merely out of reach right +// now? Only a failure raised while *interpreting* bytes we already hold +// qualifies: the object does not decode, or it is larger than we will load. +// Those are marked at the throw site, and nothing else counts — an allowlist +// of retryable errors would have to enumerate every way S3 can say no +// (AccessDenied on a rotated key, NoSuchBucket on a misconfigured name, a +// region redirect), and each one it missed would be read as "this document has +// no content" and open the room empty over content that is alive in S3. +// Guessing wrong in this direction costs a retry; guessing wrong in the other +// costs the document. +export const isPermanentFailure = (err) => err?.permanent === true; const VERDICT_CACHE_MAX = 50000; const verdicts = new Map(); // docid -> { verdict, error, expires } const rememberVerdict = ( @@ -358,6 +352,16 @@ const migrate = async (yhub, room) => { ); return 'empty'; } + // Decode before writing anything: a legacy object that is not a valid + // Yjs update fails here, on this thread, and is the one failure we know + // no retry can fix — so it is marked as such. + let contentids; + try { + contentids = Y.createContentIdsFromUpdate(update); + } catch (err) { + err.permanent = true; + throw err; + } await yhub.stream.addMessage(room, { type: 'ydoc:update:v1', // Deliberately no insertAt/deleteAt. A lazy seed is not an editing @@ -368,12 +372,9 @@ const migrate = async (yhub, room) => { // whichever the row order happened to put last. Content seeded this way // carries an author but no timestamp, so it produces no activity entry // until fullMigrate supplies the history. - // - // Reading the ids also validates the update: a corrupt legacy object - // throws here, on this thread, before anything reaches the stream. contentmap: Y.encodeContentMap( Y.createContentMapFromContentIds( - Y.createContentIdsFromUpdate(update), + contentids, [ Y.createContentAttribute('insert', 'system'), Y.createContentAttribute('insert:migration', 's3'), @@ -428,22 +429,31 @@ export const maybeMigrate = async (yhub, room) => { .then( (verdict) => rememberVerdict(room.docid, verdict), (err) => { - // logged here (once per attempt) rather than per denied connection: - // cached failures deny without new logs until the verdict expires + // The one place the *cause* is recorded, once per attempt rather + // than per access: a cached verdict re-raises this error without + // logging again until it expires. + const permanent = isPermanentFailure(err); migrationLog.error( - { event: 'seed.failed', err, docid: room.docid }, - 'soft migration failed; denying access', + { + event: 'seed.failed', + err, + docid: room.docid, + permanent, + bucket: AWS_STORAGE_BUCKET_NAME, + key: `${room.docid}/file`, + }, + permanent + ? 'soft migration is not possible for this legacy object' + : 'soft migration failed; the caller is asked to retry', ); if (err?.noCache !== true) { - // transient failures get a short TTL so a hiccup cannot lock a - // doc out for the full poison-object window + // a retryable failure is remembered only briefly, so a hiccup + // cannot lock a document out for the full poison-object window rememberVerdict( room.docid, 'failed', err, - isTransientFailure(err) - ? TRANSIENT_TTL_MS - : VERDICT_TTL_MS.failed, + permanent ? VERDICT_TTL_MS.failed : TRANSIENT_TTL_MS, ); } throw err; diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index 472da12f22..029c9cb1c3 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -13,8 +13,9 @@ import { secret } from './env.js'; import { SOFT_MIGRATION, fullMigrate, - isTransientFailure, + isPermanentFailure, maybeMigrate, + migrationLog, } from './migration.js'; const PORT = Number(process.env.PORT || 3002); @@ -73,6 +74,39 @@ const backendFetch = async (path, { cookie, origin }) => { return res.json(); }; +// First access to a room yhub does not know: seed it from the legacy Django S3 +// store before admitting the caller. Awaited inside the upgrade handler, so the +// post-upgrade initial sync (which merges postgres and the stream from clock 0) +// is guaranteed to include the seed. +// +// Seeding never decides whether the caller may read the document — that is the +// backend's answer alone. There are two ways this ends other than a seed: +// +// the legacy object cannot be migrated (it does not decode, or it is bigger +// than we will load) — retrying will not change that, so the room opens as +// a new document. Refusing instead would lock a document nobody can repair +// from the outside. Logged per access, because the caller is now editing +// alongside legacy content that stayed behind in S3. +// the legacy store could not be reached (timeout, network, backpressure) — +// the same request later may well succeed, so it answers 503 rather than +// silently starting an empty document on top of content that exists. +const seedFromLegacyStore = async (room) => { + try { + // `yhub` is declared at the bottom of this file — safe: auth callbacks only + // fire once the server is up, i.e. after that assignment + await maybeMigrate(yhub, room); + } catch (err) { + if (!isPermanentFailure(err)) { + throw apiError(503, 'Legacy document store is unavailable'); + } + // why it failed was logged once, at the attempt, inside maybeMigrate + migrationLog.warn( + { event: 'seed.skipped', docid: room.docid, err: err?.message }, + 'admitting caller to a document that could not be migrated; it opens as new', + ); + } +}; + const auth = createAuthPlugin({ // uws req is only valid synchronously — read headers AND query before first await. async readAuthInfo(req) { @@ -150,10 +184,35 @@ const auth = createAuthPlugin({ } }, async getAccessType(authInfo, { org, docid, branch }, purpose) { - if (authInfo.admin === true) return 'rw'; // Django's admin token: full access + if (authInfo.admin === true) { + // Django's admin token: full access. It still goes through the legacy + // seed, on the same terms as a user (default purpose only, so a + // `migrate` call is not seeded out from under fullMigrate). Without it a + // backend read of an unmigrated document would answer with an *empty* + // doc, and a create-ydoc against one would write a second lineage next + // to the legacy content the first user access is about to seed in. + // Access itself is never in question here — the token already granted it. + // The same org/branch fence the user path applies below. The admin token + // is the only identity that can name an arbitrary org or branch, and the + // legacy store is branchless — `{docid}/file` *is* main — so seeding any + // other room would write main's content into an orphan room, and the + // per-docid verdict cache would then report that docid as done and leave + // the real room empty. + if ( + SOFT_MIGRATION && + purpose == null && + org === ORG && + branch === 'main' && + UUID4.test(docid) + ) { + await seedFromLegacyStore({ org, docid, branch }); + } + return 'rw'; + } // Regular users only get access for the default purpose — custom-endpoint - // purposes (reset-connections) are backend-internal. Loose != on purpose: - // ws upgrades and rechecks pass undefined, built-in rest endpoints null. + // purposes (reset-connections, migrate) are backend-internal. Loose != on + // purpose: ws upgrades and rechecks pass undefined, built-in rest + // endpoints null. if ( org !== ORG || branch !== 'main' || @@ -177,29 +236,10 @@ const auth = createAuthPlugin({ if (!doc.abilities?.retrieve) { return null; } + // the backend has already decided the caller may read this document; the + // seed only decides what is in it if (SOFT_MIGRATION) { - // First access to a room yhub does not know: seed it from the legacy - // Django S3 store before admitting the connection. Awaited inside the - // upgrade handler, so the post-upgrade initial sync (which merges - // postgres and the stream from clock 0) is guaranteed to include the - // seed. Runs only for authorized readers. A missing S3 object is the - // brand-new-document case and allows an empty room; a real S3/compute - // failure denies access, either way already logged (once per attempt) - // inside maybeMigrate. - try { - // `yhub` is declared at the bottom of this file — safe: auth callbacks - // only fire once the server is up, i.e. after that assignment - await maybeMigrate(yhub, { org, docid, branch }); - } catch (err) { - // A corrupt or oversized legacy object will fail the same way forever, - // so that denial is permanent (403). An S3 timeout, a network blip or - // momentary seed backpressure will not — 503 tells the caller to come - // back rather than to treat the document as unreadable. - if (isTransientFailure(err)) { - throw apiError(503, 'Legacy document store is unavailable'); - } - return null; - } + await seedFromLegacyStore({ org, docid, branch }); } return doc.abilities.update ? 'rw' : 'r'; }, From ee2d1e49d93934ff13d12590b1e4403896d1d4f7 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Wed, 12 Aug 2026 09:22:58 +0200 Subject: [PATCH 25/59] =?UTF-8?q?=E2=AC=86=EF=B8=8F(yhub)=20upgrade=20yhub?= =?UTF-8?q?=20to=20version=200.6.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/yhub-server/package-lock.json | 8 ++++---- src/yhub-server/package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/yhub-server/package-lock.json b/src/yhub-server/package-lock.json index b200dc4317..fe96e9a648 100644 --- a/src/yhub-server/package-lock.json +++ b/src/yhub-server/package-lock.json @@ -6,7 +6,7 @@ "": { "name": "yhub-server", "dependencies": { - "@y/hub": "0.5.0", + "@y/hub": "0.6.0", "@y/y": "14.0.0-rc.24", "jose": "6.2.8", "minio": "8.0.7" @@ -112,9 +112,9 @@ "license": "ISC" }, "node_modules/@y/hub": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@y/hub/-/hub-0.5.0.tgz", - "integrity": "sha512-hMpSC3R+TvV13qPRHsSy4D+vyoSvrJah7ZoGaYtwPnHh92Ce53ufEb678SVZMN2susjKNyHUNrjoFIB7d6DjWg==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@y/hub/-/hub-0.6.0.tgz", + "integrity": "sha512-EI22ikgpeh3FgWo045hSpHPt9SZU6q7KyXvvw2czuP9rvnsE8kus1traWLVH6n2Tmd8JCo/0n1bTzc13Ise1yg==", "license": "AGPL-3.0 OR PROPRIETARY", "dependencies": { "@y-crdt/yn": "^0.1.4", diff --git a/src/yhub-server/package.json b/src/yhub-server/package.json index 485e6e8fd1..17beae20fc 100644 --- a/src/yhub-server/package.json +++ b/src/yhub-server/package.json @@ -6,7 +6,7 @@ "start": "node server.js" }, "dependencies": { - "@y/hub": "0.5.0", + "@y/hub": "0.6.0", "@y/y": "14.0.0-rc.24", "jose": "6.2.8", "minio": "8.0.7" From 49e416075b73305c1ab3da6804d8dabfccd86262 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Wed, 12 Aug 2026 09:47:13 +0200 Subject: [PATCH 26/59] =?UTF-8?q?=E2=99=BB=EF=B8=8F(yhub)=20maintain=20dat?= =?UTF-8?q?abase=20schema=20using=20npm=20run=20init-db?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The yhub database schema have new update and will probably be modified in the future. We don't want to maintain this sql schema in the Docs repo, we want to reuse what is directly made in the yhub project. For this we reuse the existing bin/init-db.js script --- .github/workflows/impress.yml | 15 ++++++++------- CHANGELOG.md | 18 ++++++++++++++++++ Makefile | 15 +++++++++++++++ compose.yml | 6 ++++-- docker/files/yhub/initdb/01-yhub.sql | 14 -------------- src/yhub-server/README.md | 19 +++++++++++++++++++ src/yhub-server/package.json | 3 ++- 7 files changed, 66 insertions(+), 24 deletions(-) delete mode 100644 docker/files/yhub/initdb/01-yhub.sql diff --git a/.github/workflows/impress.yml b/.github/workflows/impress.yml index 272f523eb4..f9c13f2a08 100644 --- a/.github/workflows/impress.yml +++ b/.github/workflows/impress.yml @@ -226,13 +226,6 @@ jobs: working-directory: . run: bin/generate-jwt-private-key.sh - - name: Create the collaboration server database - run: | - PGPASSWORD=pass psql -h localhost -U dinum -d impress \ - -c 'CREATE DATABASE yhub' - PGPASSWORD=pass psql -h localhost -U dinum -d yhub \ - -f ../../docker/files/yhub/initdb/01-yhub.sql - - name: Generate a MO file from strings extracted from the project run: uv run python manage.py compilemessages @@ -245,6 +238,14 @@ jobs: working-directory: src/yhub-server run: npm ci --omit=dev + # yhub ships its own DDL and creates the database as well, so this needs + # the dependencies installed above — hence its place after them + - name: Create the collaboration server database + working-directory: src/yhub-server + env: + POSTGRES: postgres://dinum:pass@localhost:5432/yhub + run: npm run init-db + - name: Start the backend for the collaboration server to authenticate against env: DJANGO_ALLOWED_HOSTS: "*" diff --git a/CHANGELOG.md b/CHANGELOG.md index a792d50e6d..66421ef6a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,24 @@ and this project adheres to ### Added +- ⬆️(collaboration) upgrade yhub to 0.6.0, which needs a schema change: a + `yhub_ydoc_tombstones_v1` table (it adds document deletion) and four + `*_is_reference` markers on `yhub_ydoc_v1`. Neither is optional — every + document read joins the tombstone table, so without them yhub answers + `relation "yhub_ydoc_tombstones_v1" does not exist`. Existing rows read as + "may be a reference", exactly as before the markers existed, so there is no + backfill and no downtime beyond applying the DDL +- 🔧(collaboration) let yhub own its schema: `npm run init-db` in + `src/yhub-server` runs the DDL script yhub ships (`bin/init-db.js`), which + creates the database when missing and every table the installed version + needs. It replaces the copy of the schema we kept in + `docker/files/yhub/initdb/`, which only replayed on a fresh postgres volume — + so an upgrade that added a table silently skipped an existing database, and + the copy had to be updated by hand on every upgrade. `make migrate-yhub` runs + it against the dev stack, the counterpart of `make migrate` for the Django + database, and is part of `make bootstrap`; CI runs the same script instead of + applying the SQL by hand. It is the only thing that runs DDL — the server and + the worker never do — and it is idempotent, so re-running it is always safe - ✅(collaboration) add integration tests covering both legacy migrations, running against a real yhub: CI now starts the collaboration server and a valkey alongside the backend test job. They skip themselves when nothing diff --git a/Makefile b/Makefile index 6f0a033b2a..82729249a6 100644 --- a/Makefile +++ b/Makefile @@ -101,6 +101,7 @@ pre-bootstrap: \ post-bootstrap: \ migrate \ + migrate-yhub \ demo \ back-i18n-compile \ mails-install \ @@ -332,6 +333,20 @@ migrate: ## run django migrations for the impress project. @$(MANAGE) migrate .PHONY: migrate +# Runs the DDL script yhub ships (`bin/init-db.js`, wrapped as `npm run +# init-db`): it creates the yhub database when missing, then every table the +# installed @y/hub version needs. yhub never runs DDL from the server or the +# worker, so this is what applies a schema change after an upgrade. +# Both stores are started because the script also creates the valkey worker +# stream and connects to it whenever REDIS is set. Re-running is safe and +# expected; on an existing stream it logs a harmless `BUSYGROUP` error and +# still exits 0, since the server creates that stream at startup anyway. +migrate-yhub: ## create or upgrade the collaboration server (yhub) schema. + @echo "$(BOLD)Running yhub migrations$(RESET)" + @$(COMPOSE) up -d yhub-postgres yhub-valkey + @$(COMPOSE_RUN) --no-deps yhub npm run init-db +.PHONY: migrate-yhub + superuser: ## Create an admin superuser with password "admin" @echo "$(BOLD)Creating a Django superuser$(RESET)" @$(MANAGE) createsuperuser --email admin@example.com --password admin diff --git a/compose.yml b/compose.yml index 02bf221bc3..e4d307f780 100644 --- a/compose.yml +++ b/compose.yml @@ -221,14 +221,16 @@ services: POSTGRES_DB: yhub volumes: - yhub-pgdata:/var/lib/postgresql/data - # NOTE: initdb.d only runs on a FRESH volume; schema changes need `podman volume rm` - - ./docker/files/yhub/initdb:/docker-entrypoint-initdb.d:ro healthcheck: test: ["CMD-SHELL", "pg_isready -U yhub"] interval: 1s timeout: 2s retries: 60 # no published port (Django's postgres already publishes) + # the schema is not seeded here: initdb.d would only replay on a fresh + # volume, so an upgrade that adds a table would silently skip an existing + # one. `make migrate-yhub` runs yhub's own DDL script instead, the same way + # `make migrate` runs Django's migrations. yhub: user: ${DOCKER_USER:-1000} diff --git a/docker/files/yhub/initdb/01-yhub.sql b/docker/files/yhub/initdb/01-yhub.sql deleted file mode 100644 index 0a28fb3c06..0000000000 --- a/docker/files/yhub/initdb/01-yhub.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Column-for-column from yhub bin/init-db.js (unquoted identifiers so --- case-folding matches yhub's persistence.js queries). -CREATE TABLE IF NOT EXISTS yhub_ydoc_v1 ( - org text, - docid text, - branch text, - t text, - created INT8, - gcDoc bytea, - nongcDoc bytea, - contentmap bytea, - contentids bytea, - PRIMARY KEY (org,docid,branch,t) -); diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index 4be057c335..7c6e0544e9 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -52,6 +52,25 @@ backend-internal and should not be routed through the public ingress. The `Dockerfile` builds the container image used by the `yhub` service in `compose.yml`. +## Database schema (`npm run init-db`) + +yhub never runs DDL from the server or the worker, so the schema is created by +the script it ships (`node_modules/@y/hub/bin/init-db.js`), wrapped here as +`npm run init-db`. It reads `POSTGRES` from the environment, creates the +database when it does not exist, then every table and index the **installed** +yhub version needs. It is idempotent, so re-running it is always safe. + +Run it whenever `@y/hub` is upgraded — releases that add a table or a column +say so in their changelog, and the server fails on every document read until +the DDL is applied (`relation "yhub_ydoc_tombstones_v1" does not exist`, for +instance). Nothing in this repository copies the schema, so an upgrade is +`package.json` plus this script and nothing else. + +From the repository root, `make migrate-yhub` runs it against the dev stack — +the counterpart of `make migrate` for the Django database. `make bootstrap` +already includes it, so a fresh checkout needs nothing extra; an upgrade is +`make migrate-yhub` and restart the service. + ## Soft migration (`SOFT_MIGRATION=true`) Documents were historically stored by the Django backend in the S3 media diff --git a/src/yhub-server/package.json b/src/yhub-server/package.json index 17beae20fc..998924faff 100644 --- a/src/yhub-server/package.json +++ b/src/yhub-server/package.json @@ -3,7 +3,8 @@ "private": true, "type": "module", "scripts": { - "start": "node server.js" + "start": "node server.js", + "init-db": "node node_modules/@y/hub/bin/init-db.js" }, "dependencies": { "@y/hub": "0.6.0", From e06bfa9a8561a6d0f9521939f5af377d0a932500 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Wed, 12 Aug 2026 16:33:55 +0200 Subject: [PATCH 27/59] =?UTF-8?q?=F0=9F=94=A7(collaboration)=20adapt=20doc?= =?UTF-8?q?ker=20stack=20for=20development=20purpose?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The yhub image was build only for a production usage. In development we want to have a hot reload when a file is modified. For this the Dockerfile is modified, the nodemon package install in dev environment and used to watch modification against the source files. --- CHANGELOG.md | 4 + compose.yml | 10 +- src/yhub-server/Dockerfile | 29 ++- src/yhub-server/README.md | 31 ++- src/yhub-server/package-lock.json | 377 ++++++++++++++++++++++++++++++ src/yhub-server/package.json | 4 + 6 files changed, 449 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66421ef6a6..32880fbffa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -121,6 +121,10 @@ and this project adheres to converter-only and no longer serves `/collaboration/ws/`; deployments using the existing helm values lose collaboration until the helm chart routes collaboration to yhub (follow-up) +- 🔧(collaboration) split the yhub image into a development and a production + stage, like the other services: the dev stack now bind-mounts + `src/yhub-server` and runs the server through nodemon, so editing a source + file restarts it instead of needing `make build-yhub` ## [v5.4.1] - 2026-07-09 diff --git a/compose.yml b/compose.yml index e4d307f780..c9f0e9a2d6 100644 --- a/compose.yml +++ b/compose.yml @@ -237,8 +237,8 @@ services: build: context: ./src/yhub-server dockerfile: Dockerfile - target: yhub - image: impress:yhub + target: yhub-development + image: impress:yhub-development environment: HOME: /tmp # same reason as node-based services above (unmapped uid) PORT: 3002 @@ -263,6 +263,12 @@ services: # starting before minio would cache 401s for the first accessed docs minio: condition: service_healthy + volumes: + # editing a source file restarts the server (nodemon), no rebuild + - ./src/yhub-server:/app + # node_modules is installed in the image, not in the source tree: keep + # the bind mount above from hiding it + - /app/node_modules kc_postgresql: image: postgres:14.3 diff --git a/src/yhub-server/Dockerfile b/src/yhub-server/Dockerfile index b7ef20248b..4d31804940 100644 --- a/src/yhub-server/Dockerfile +++ b/src/yhub-server/Dockerfile @@ -1,12 +1,37 @@ # trixie for glibc >= 2.38 — uws prebuilt binaries reject bookworm's 2.36 -FROM node:22-trixie AS yhub +FROM node:22-trixie AS base WORKDIR /app COPY package.json package-lock.json ./ + + +# ---- Development image ---- +FROM base AS yhub-development + +# dev dependencies included: nodemon, plus this is where one-off scripts run +# (`make migrate-yhub` runs `npm run init-db` in it) +RUN npm ci + +# server.js, migration.js, env.js — glob so a new module cannot be forgotten. +# compose bind-mounts the sources over /app on top of this copy, so an edit on +# the host is seen immediately; the copy keeps the image usable on its own. +COPY *.js ./ + +EXPOSE 3002 + +# `npm run dev` restarts the server on every source change, no rebuild needed. +# nodemon rather than node's own --watch: the latter watches inodes, so it goes +# deaf as soon as a file is replaced by a rename — which is what `git checkout` +# and most editors do when saving. +CMD ["npm", "run", "dev"] + + +# ---- Production image ---- +FROM base AS yhub + RUN npm ci --omit=dev -# server.js, migration.js, env.js — glob so a new module cannot be forgotten COPY *.js ./ EXPOSE 3002 diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index 7c6e0544e9..c87929235d 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -49,8 +49,35 @@ authorization and are meant to be reachable by browsers. The one exception is `/collaboration/reset-connections/` and `/collaboration/migrate/`, which are backend-internal and should not be routed through the public ingress. -The `Dockerfile` builds the container image used by the `yhub` service in -`compose.yml`. +## Container image + +The `Dockerfile` has two final stages, like the other services of this +repository: + +- `yhub-development` — what the `yhub` service of `compose.yml` builds. It + installs the dev dependencies and starts the server through `npm run dev` + (nodemon), and compose bind-mounts `src/yhub-server` over `/app`: **editing + `server.js`, `migration.js` or `env.js` restarts the server, no rebuild**. + Watch it happen with `docker compose logs -f yhub`. A syntax error stops at + `app crashed - waiting for file changes` and the next save starts the server + again, +- `yhub` — the production image: production dependencies only, `node + server.js`, sources baked in. + +nodemon rather than node's own `--watch`: the latter watches inodes, so it +stops seeing a file as soon as it is replaced by a rename — which is what `git +checkout` and most editors do when saving. The one-second `--delay` debounces +partial writes, so a branch switch restarts the server once, after the files +have settled. + +Only source edits are picked up live. A dependency change (`package.json`) is a +rebuild, and `node_modules` lives in an anonymous volume that survives a plain +recreate, so it needs renewing: + +``` +make build-yhub +docker compose up -d --force-recreate --renew-anon-volumes yhub +``` ## Database schema (`npm run init-db`) diff --git a/src/yhub-server/package-lock.json b/src/yhub-server/package-lock.json index fe96e9a648..d12a212b59 100644 --- a/src/yhub-server/package-lock.json +++ b/src/yhub-server/package-lock.json @@ -11,6 +11,9 @@ "jose": "6.2.8", "minio": "8.0.7" }, + "devDependencies": { + "nodemon": "3.1.14" + }, "engines": { "node": ">=22" } @@ -176,6 +179,20 @@ "url": "https://github.com/sponsors/dmonad" } }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/anynum": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", @@ -203,6 +220,29 @@ "node": ">=8.0.0" } }, + "node_modules/balanced-match": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", + "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", + "dev": true, + "license": "MIT", + "engines": { + "node": "18 || 20 || >=22" + } + }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/block-stream2": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/block-stream2/-/block-stream2-2.1.0.tgz", @@ -212,6 +252,32 @@ "readable-stream": "^3.4.0" } }, + "node_modules/brace-expansion": { + "version": "5.0.9", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", + "integrity": "sha512-ScQ4IuvIEF1TMlP7Zt+vjJ//9zlPb2SDcxWxM3bk8s6t6GGdJ7KO1dCcTidOPJKePW30LE/2cT7wCyPho9/Wxg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^4.0.2" + }, + "engines": { + "node": "20 || >=22" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/browser-or-node": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/browser-or-node/-/browser-or-node-2.1.1.tgz", @@ -227,6 +293,31 @@ "node": ">=8.0.0" } }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, "node_modules/cluster-key-slot": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", @@ -236,6 +327,24 @@ "node": ">=0.10.0" } }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, "node_modules/decode-uri-component": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", @@ -290,6 +399,19 @@ "fxparser": "src/cli/cli.js" } }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/filter-obj": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/filter-obj/-/filter-obj-1.1.0.tgz", @@ -299,6 +421,51 @@ "node": ">=0.10.0" } }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/ignore-by-default": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", + "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==", + "dev": true, + "license": "ISC" + }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -314,6 +481,52 @@ "node": ">= 10" } }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, "node_modules/is-unsafe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.0.tgz", @@ -380,6 +593,22 @@ "node": ">= 0.6" } }, + "node_modules/minimatch": { + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.8" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/minio": { "version": "8.0.7", "resolved": "https://registry.npmjs.org/minio/-/minio-8.0.7.tgz", @@ -404,6 +633,52 @@ "node": "^16 || ^18 || >=20" } }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nodemon": { + "version": "3.1.14", + "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.14.tgz", + "integrity": "sha512-jakjZi93UtB3jHMWsXL68FXSAosbLfY0In5gtKq3niLSkrWznrVBzXFNOEMJUfc9+Ke7SHWoAZsiMkNP3vq6Jw==", + "dev": true, + "license": "MIT", + "dependencies": { + "chokidar": "^3.5.2", + "debug": "^4", + "ignore-by-default": "^1.0.1", + "minimatch": "^10.2.1", + "pstree.remy": "^1.1.8", + "semver": "^7.5.3", + "simple-update-notifier": "^2.0.0", + "supports-color": "^5.5.0", + "touch": "^3.1.0", + "undefsafe": "^2.0.5" + }, + "bin": { + "nodemon": "bin/nodemon.js" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/nodemon" + } + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/on-exit-leak-free": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", @@ -428,6 +703,19 @@ "node": ">=14.0.0" } }, + "node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/pino": { "version": "10.3.1", "resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz", @@ -494,6 +782,13 @@ ], "license": "MIT" }, + "node_modules/pstree.remy": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", + "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", + "dev": true, + "license": "MIT" + }, "node_modules/query-string": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/query-string/-/query-string-7.1.3.tgz", @@ -532,6 +827,19 @@ "node": ">= 6" } }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, "node_modules/real-require": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", @@ -595,6 +903,32 @@ "node": ">=11.0.0" } }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/simple-update-notifier": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz", + "integrity": "sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/sonic-boom": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.1.tgz", @@ -670,6 +1004,19 @@ "anynum": "^1.0.1" } }, + "node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/thread-stream": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.2.0.tgz", @@ -697,6 +1044,36 @@ "readable-stream": "3" } }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/touch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.1.tgz", + "integrity": "sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==", + "dev": true, + "license": "ISC", + "bin": { + "nodetouch": "bin/nodetouch.js" + } + }, + "node_modules/undefsafe": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz", + "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", + "dev": true, + "license": "MIT" + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/src/yhub-server/package.json b/src/yhub-server/package.json index 998924faff..a4d33c6815 100644 --- a/src/yhub-server/package.json +++ b/src/yhub-server/package.json @@ -4,6 +4,7 @@ "type": "module", "scripts": { "start": "node server.js", + "dev": "nodemon --delay 1 server.js", "init-db": "node node_modules/@y/hub/bin/init-db.js" }, "dependencies": { @@ -12,6 +13,9 @@ "jose": "6.2.8", "minio": "8.0.7" }, + "devDependencies": { + "nodemon": "3.1.14" + }, "engines": { "node": ">=22" } From d9294ef51eec35f6381004178284fd1fdee5f867 Mon Sep 17 00:00:00 2001 From: Anthony LC Date: Tue, 4 Aug 2026 18:24:00 +0200 Subject: [PATCH 28/59] =?UTF-8?q?=F0=9F=9B=82(y-provider)=20verify=20jwt?= =?UTF-8?q?=20token=20instead=20of=20the=20shared=20api=20key?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /api/convert route no longer accepts the Y_PROVIDER_API_KEY shared secret. It now verifies the admin JWT signed by Django against the JWKS published on its /api/v1.0/jwks endpoint. --- src/frontend/servers/y-provider/__tests__/convert.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/frontend/servers/y-provider/__tests__/convert.test.ts b/src/frontend/servers/y-provider/__tests__/convert.test.ts index bbbec7bf6f..88ed4174de 100644 --- a/src/frontend/servers/y-provider/__tests__/convert.test.ts +++ b/src/frontend/servers/y-provider/__tests__/convert.test.ts @@ -17,6 +17,8 @@ vi.mock('../src/env', async (importOriginal) => { }; }); +import { mockJwksEndpoint, signAdminToken } from './testUtils/adminJwt'; + import { docsBlockNoteSchema } from '@/blockSpecs'; import { initApp } from '@/servers'; From 9d9c706559121205cb2dc4fc5c33c9b147fa3bc0 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Wed, 5 Aug 2026 15:13:54 +0200 Subject: [PATCH 29/59] =?UTF-8?q?=E2=9C=A8(backend)=20add=20a=20service=20?= =?UTF-8?q?to=20call=20the=20yhub=20REST=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The backend application will have to call the yhub REST API for some operations. We want to use a dedicated service to do that. This first commit introduces the shape of this service, it only does the configuration for now, calling actions will be implemented later. --- CHANGELOG.md | 1 + documentation/env.md | 3 + env.d/development/common | 1 + src/backend/core/services/yhub_services.py | 165 ++++++++++++++++++ src/backend/core/tests/test_api_jwks.py | 2 +- .../tests/test_services_converter_services.py | 2 +- .../core/tests/test_services_jwt_services.py | 2 +- .../core/tests/test_services_yhub_services.py | 134 ++++++++++++++ .../tests/utils/{jwt.py => jwt_helper.py} | 0 src/backend/impress/settings.py | 13 ++ 10 files changed, 320 insertions(+), 3 deletions(-) create mode 100644 src/backend/core/services/yhub_services.py create mode 100644 src/backend/core/tests/test_services_yhub_services.py rename src/backend/core/tests/utils/{jwt.py => jwt_helper.py} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32880fbffa..5304d2e554 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,7 @@ and this project adheres to `503` instead of denying access like a permission failure, so clients retry instead of giving up. The built-in endpoints can also answer JSON on `Accept: application/json` +- ✨(backend) add a service to call the yhub REST API - ✨(backend) add a service generating cached RS256 JWT tokens - ✨(backend) publish the JWT public key on a JWKS endpoint - 🔧(dev) generate the JWT signing key when bootstrapping the dev stack diff --git a/documentation/env.md b/documentation/env.md index d5ef078c99..c02648160c 100644 --- a/documentation/env.md +++ b/documentation/env.md @@ -141,6 +141,9 @@ These are the environment variables you can set for the `impress-backend` contai | USER_ONBOARDING_DOCUMENTS | A list of documents IDs for which a read-only access will be created for new s | [] | | USER_ONBOARDING_SANDBOX_DOCUMENT | ID of a template sandbox document that will be duplicated for new users | | | USER_RECONCILIATION_FORM_URL | URL of a third-party form for user reconciliation requests | | +| YHUB_API_BASE_URL | Base url of the yhub collaboration server REST API | | +| YHUB_API_TIMEOUT | Timeout (in seconds) of the requests to the yhub API | 30 | +| YHUB_ORG | yhub organization the documents live in. Must match the YHUB_ORG of the yhub server | docs | | Y_PROVIDER_API_BASE_URL | Y Provider url | | | Y_PROVIDER_API_KEY | Y provider API key | | diff --git a/env.d/development/common b/env.d/development/common index 8b406be286..4b09afab7b 100644 --- a/env.d/development/common +++ b/env.d/development/common @@ -83,6 +83,7 @@ COLLABORATION_WS_URL=ws://localhost:3002/collaboration/ws/v1/docs COLLABORATION_WS_INACTIVITY_TIMEOUT=15 # Seconds # server-to-server, reached with an admin JWT (aud: yhub) COLLABORATION_API_URL=http://yhub:3002/collaboration +YHUB_API_BASE_URL=http://yhub:3002 DJANGO_SERVER_TO_SERVER_API_TOKENS=server-api-token Y_PROVIDER_API_BASE_URL=http://y-provider-development-converter:4444/api/ diff --git a/src/backend/core/services/yhub_services.py b/src/backend/core/services/yhub_services.py new file mode 100644 index 0000000000..9cf6a66a47 --- /dev/null +++ b/src/backend/core/services/yhub_services.py @@ -0,0 +1,165 @@ +""" +yhub API services. + +yhub is the collaboration server holding the live Yjs state of the documents +(see `src/yhub-server`). Beside the websocket used by the editors, it exposes a +REST API letting a backend read and act on a document out of band. + +Every route is mounted under the `apiPrefix` yhub is configured with, and a +room is addressed as `/{prefix}/{endpoint}/{version}/{org}/{docid}`, where `org` +is the yhub organization Docs runs under and `docid` the document id. The +built-in endpoints are `ydoc` (get the state of a document, patch it with a Yjs +update), `rollback`, `prune`, `changeset` and `activity`, all at `v1`. yhub also +accepts a `branch` query parameter, but our auth plugin only ever grants access +to the `main` branch, so this service never sends it. + +This service only owns the transport for now, the endpoints are added as we +need them. +""" + +import logging + +from django.conf import settings + +import requests + +from core.services.jwt_services import JWTService + +logger = logging.getLogger(__name__) + + +class YHubError(Exception): + """Base exception for yhub related errors.""" + + +class ConfigurationError(YHubError): + """Raised when the yhub service is not properly configured.""" + + +class ServiceUnavailableError(YHubError): + """Raised when the yhub service cannot be reached.""" + + +class APIError(YHubError): + """Raised when the yhub API answers with an error status.""" + + def __init__(self, message, status_code=None): + super().__init__(message) + self.status_code = status_code + + +class YHubService: + """ + Client for the REST API of the yhub collaboration server. + + It owns the transport: where yhub lives, how a request is authenticated and + how a failure is reported. The endpoints themselves are added as we need + them, on top of `build_url` and `request`. + + A call serving the request of an authenticated user should be made by a + service built with that user, the token then names them as its subject. + """ + + # Segment every yhub route is mounted under. yhub defaults it to "api", we + # serve it under "collaboration" and configure its `apiPrefix` to match. It + # is a single path segment, yhub rejects anything else at startup. + api_prefix = "collaboration" + + # Version of the endpoints we call, the one all the built-ins are at. + api_version = "v1" + + def __init__(self, user=None): + """Bind the service to the user a call is made on behalf of, if any.""" + self.user = user + + @property + def base_url(self): + """Return the base url of the yhub API, without its trailing slash.""" + base_url = settings.YHUB_API_BASE_URL + if not base_url: + raise ConfigurationError( + "The YHUB_API_BASE_URL setting is required to reach the yhub API." + ) + return base_url.rstrip("/") + + @property + def org(self): + """Return the yhub organization the documents live in.""" + return settings.YHUB_ORG + + @property + def timeout(self): + """Return the timeout of the requests to the yhub API, in seconds.""" + return settings.YHUB_API_TIMEOUT + + @property + def claims(self): + """ + Build the claims naming who a request to the yhub API is made for. + + The "sub" claim is only there when the call is made on behalf of an + authenticated user, so that yhub attributes what it changes to them + rather than to the backend itself. A call made outside of a request, + from a Celery task for instance, has no subject to name. + """ + if self.user is None or not self.user.is_authenticated: + return {} + + return {"sub": str(self.user.pk)} + + @property + def auth_header(self): + """ + Build the authentication header of a request to the yhub API. + + The token always grants admin, a server-to-server call acts on a + document without going through the abilities of a user. The subject it + may carry is who the call is for, it never restricts what it can do. + """ + return f"Bearer {JWTService().get_admin_token(self.claims)}" + + def build_url(self, endpoint, document_id): + """Build the url of a document scoped endpoint of the yhub API.""" + return ( + f"{self.base_url}/{self.api_prefix}/{endpoint}/{self.api_version}" + f"/{self.org}/{document_id}" + ) + + def request(self, method, url, params=None, data=None): + """ + Send an authenticated request to the yhub API. + + Return the raw response, it is up to the caller to decode its body: the + endpoints do not all answer with the same payload. + """ + try: + response = requests.request( + method, + url, + params=params, + data=data, + headers={ + "Authorization": self.auth_header, + "Content-Type": "application/octet-stream", + }, + timeout=self.timeout, + ) + except requests.RequestException as err: + logger.exception("yhub service error: url=%s", url) + raise ServiceUnavailableError( + f"Failed to connect to the yhub service at {url}" + ) from err + + if not response.ok: + logger.error( + "yhub API error: url=%s, status=%d, response=%s", + url, + response.status_code, + response.text[:200] if response.text else "empty", + ) + raise APIError( + f"The yhub API answered {response.status_code} on {url}", + status_code=response.status_code, + ) + + return response diff --git a/src/backend/core/tests/test_api_jwks.py b/src/backend/core/tests/test_api_jwks.py index 348d55a6c1..1618247dcf 100644 --- a/src/backend/core/tests/test_api_jwks.py +++ b/src/backend/core/tests/test_api_jwks.py @@ -9,7 +9,7 @@ from rest_framework.test import APIClient from core.services.jwt_services import JWTService -from core.tests.utils.jwt import generate_key_pair +from core.tests.utils.jwt_helper import generate_key_pair from core.tests.utils.urls import reload_urls pytestmark = pytest.mark.django_db diff --git a/src/backend/core/tests/test_services_converter_services.py b/src/backend/core/tests/test_services_converter_services.py index 0b6f5183bf..949088599f 100644 --- a/src/backend/core/tests/test_services_converter_services.py +++ b/src/backend/core/tests/test_services_converter_services.py @@ -13,7 +13,7 @@ ValidationError, YdocConverter, ) -from core.tests.utils.jwt import generate_key_pair +from core.tests.utils.jwt_helper import generate_key_pair # Generating an RSA key is expensive, do it once for the whole module PRIVATE_KEY, PUBLIC_KEY = generate_key_pair() diff --git a/src/backend/core/tests/test_services_jwt_services.py b/src/backend/core/tests/test_services_jwt_services.py index f2bc8f2b2a..eaf0f2989c 100644 --- a/src/backend/core/tests/test_services_jwt_services.py +++ b/src/backend/core/tests/test_services_jwt_services.py @@ -17,7 +17,7 @@ JWTService, TokenGenerationError, ) -from core.tests.utils.jwt import generate_key_pair +from core.tests.utils.jwt_helper import generate_key_pair # Generating RSA keys is expensive, do it once for the whole module PRIVATE_KEY, PUBLIC_KEY = generate_key_pair() diff --git a/src/backend/core/tests/test_services_yhub_services.py b/src/backend/core/tests/test_services_yhub_services.py new file mode 100644 index 0000000000..1b92569ce0 --- /dev/null +++ b/src/backend/core/tests/test_services_yhub_services.py @@ -0,0 +1,134 @@ +"""Test yhub services.""" + +from unittest.mock import patch + +from django.contrib.auth.models import AnonymousUser + +import jwt +import pytest +import requests + +from core.factories import UserFactory +from core.services.yhub_services import ( + APIError, + ConfigurationError, + ServiceUnavailableError, + YHubService, +) +from core.tests.utils.jwt_helper import generate_key_pair + +# Generating an RSA key is expensive, do it once for the whole module +PRIVATE_KEY, PUBLIC_KEY = generate_key_pair() + + +@pytest.fixture(autouse=True) +def yhub_settings(settings): + """Setup valid settings for the yhub service and the JWT service it signs with.""" + settings.YHUB_API_BASE_URL = "http://yhub:3002" + settings.YHUB_ORG = "docs" + settings.YHUB_API_TIMEOUT = 30 + settings.JWT_PRIVATE_KEY = PRIVATE_KEY + settings.JWT_TOKEN_LIFETIME = 3600 + + +def test_base_url_required(settings): + """Should raise ConfigurationError when the base url is not configured.""" + settings.YHUB_API_BASE_URL = None + service = YHubService() + + with pytest.raises(ConfigurationError, match="YHUB_API_BASE_URL"): + _ = service.base_url + + +def test_base_url_strips_trailing_slash(settings): + """The trailing slash of the base url should not leak into the urls we build.""" + settings.YHUB_API_BASE_URL = "http://yhub:3002/" + + assert YHubService().base_url == "http://yhub:3002" + + +def test_build_url(): + """A document scoped url should be mounted under the api prefix of yhub.""" + url = YHubService().build_url("ydoc", "8c1c8c4d-4b02-4b0f-a0e9-e00cbd1a9a2f") + + assert url == ( + "http://yhub:3002/collaboration/ydoc/v1/docs/" + "8c1c8c4d-4b02-4b0f-a0e9-e00cbd1a9a2f" + ) + + +def test_auth_header(): + """The auth header should carry an admin JWT signed with the configured key.""" + scheme, token = YHubService().auth_header.split(" ") + + assert scheme == "Bearer" + payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + assert payload["admin"] is True + assert "sub" not in payload + + +def test_auth_header_with_user(): + """The token should name the user a call is made on behalf of as its subject.""" + user = UserFactory.build() + + _scheme, token = YHubService(user=user).auth_header.split(" ") + + payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + assert payload["sub"] == str(user.pk) + # naming a subject should not restrict what the call can do + assert payload["admin"] is True + + +def test_auth_header_with_anonymous_user(): + """An anonymous user is no subject, the token should not name one.""" + _scheme, token = YHubService(user=AnonymousUser()).auth_header.split(" ") + + payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + assert "sub" not in payload + assert payload["admin"] is True + + +@patch("requests.request") +def test_request(mock_request): + """Should send an authenticated request to the yhub API.""" + mock_request.return_value.ok = True + service = YHubService() + + response = service.request( + "get", service.build_url("ydoc", "doc-id"), params={"gc": "false"} + ) + + assert response is mock_request.return_value + args, kwargs = mock_request.call_args + assert args == ("get", "http://yhub:3002/collaboration/ydoc/v1/docs/doc-id") + assert kwargs["params"] == {"gc": "false"} + assert kwargs["timeout"] == 30 + assert kwargs["headers"]["Authorization"].startswith("Bearer ") + + +@patch("requests.request") +def test_request_service_unavailable(mock_request): + """Should raise ServiceUnavailableError when yhub cannot be reached.""" + mock_request.side_effect = requests.RequestException("Connection error") + + with pytest.raises( + ServiceUnavailableError, match="Failed to connect to the yhub service" + ): + YHubService().request( + "get", "http://yhub:3002/collaboration/ydoc/v1/docs/doc-id" + ) + + +@patch("requests.request") +def test_request_error_status(mock_request): + """Should raise APIError, carrying the status, when yhub answers an error.""" + mock_request.return_value.ok = False + mock_request.return_value.status_code = 403 + mock_request.return_value.text = "Forbidden" + + with pytest.raises(APIError, match="The yhub API answered 403") as excinfo: + YHubService().request( + "get", "http://yhub:3002/collaboration/ydoc/v1/docs/doc-id" + ) + + assert excinfo.value.status_code == 403 diff --git a/src/backend/core/tests/utils/jwt.py b/src/backend/core/tests/utils/jwt_helper.py similarity index 100% rename from src/backend/core/tests/utils/jwt.py rename to src/backend/core/tests/utils/jwt_helper.py diff --git a/src/backend/impress/settings.py b/src/backend/impress/settings.py index 62a3e1f188..9b79c48ebb 100755 --- a/src/backend/impress/settings.py +++ b/src/backend/impress/settings.py @@ -528,6 +528,19 @@ class Base(Configuration): None, environ_name="COLLABORATION_API_URL", environ_prefix=None ) + # yhub collaboration server, as reached by core.services.yhub_services + YHUB_API_BASE_URL = values.Value( + None, environ_name="YHUB_API_BASE_URL", environ_prefix=None + ) + # The yhub organization our documents live in. It must match the YHUB_ORG + # of the yhub server, which rejects the rooms of any other organization. + YHUB_ORG = values.Value("docs", environ_name="YHUB_ORG", environ_prefix=None) + YHUB_API_TIMEOUT = values.IntegerValue( + default=30, + environ_name="YHUB_API_TIMEOUT", + environ_prefix=None, + ) + # JWT # RSA private key (PEM) used to sign the tokens issued by # core.services.jwt_services.JWTService. Prefer the JWT_PRIVATE_KEY_FILE From 3b9318de7027e3ef60d87ac322e014f0c0885f22 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Wed, 5 Aug 2026 15:47:33 +0200 Subject: [PATCH 30/59] =?UTF-8?q?=E2=99=BB=EF=B8=8F(backend)=20audience=20?= =?UTF-8?q?is=20an=20enum=20to=20be=20used=20by=20the=20JWTService?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To ease the use of the audience with the JWTService, we choose to create an enum holding all the possible values and then use them in the Yhub and Y-converter services. --- .../core/services/converter_services.py | 9 +-- src/backend/core/services/jwt_services.py | 13 +++- src/backend/core/services/yhub_services.py | 7 ++- .../tests/test_services_converter_services.py | 5 +- .../core/tests/test_services_jwt_services.py | 59 ++++++++++++++----- .../core/tests/test_services_yhub_services.py | 16 ++++- 6 files changed, 76 insertions(+), 33 deletions(-) diff --git a/src/backend/core/services/converter_services.py b/src/backend/core/services/converter_services.py index ce527dcb71..a5edd41953 100644 --- a/src/backend/core/services/converter_services.py +++ b/src/backend/core/services/converter_services.py @@ -9,15 +9,10 @@ import requests from core.services import mime_types -from core.services.jwt_services import JWTService +from core.services.jwt_services import Audiences, JWTService logger = logging.getLogger(__name__) -# Audience of the admin token y-provider expects. Scoping the token to it -# prevents an admin JWT issued for another backend service from being -# replayed against y-provider. -Y_CONVERTER_AUDIENCE = "y-converter" - class ConversionError(Exception): """Base exception for conversion-related errors.""" @@ -115,7 +110,7 @@ class YdocConverter: @property def auth_header(self): """Build microservice authentication header.""" - token = JWTService().get_admin_token({"aud": Y_CONVERTER_AUDIENCE}) + token = JWTService().get_admin_token(audience=Audiences.Y_CONVERTER) return f"Bearer {token}" def _request(self, url, data, content_type, accept): diff --git a/src/backend/core/services/jwt_services.py b/src/backend/core/services/jwt_services.py index 92fa152881..7ac666fbaf 100644 --- a/src/backend/core/services/jwt_services.py +++ b/src/backend/core/services/jwt_services.py @@ -5,6 +5,7 @@ import json import logging from datetime import timedelta +from enum import StrEnum from django.conf import settings from django.core.cache import cache @@ -19,6 +20,13 @@ CACHE_KEY_PREFIX = "jwt_token" +class Audiences(StrEnum): + """Enum of the audiences we can use.""" + + Y_CONVERTER = "y-converter" + YHUB = "yhub" + + class JWTError(Exception): """Base exception for JWT related errors.""" @@ -172,12 +180,11 @@ def get_token(self, claims): return token - def get_admin_token(self, claims=None): + def get_admin_token(self, audience: Audiences, claims=None): """ Return a token with the `admin: true` claim. Extra claims can be injected alongside it. They cannot turn the "admin" claim off: a token issued by this method always grants admin. """ - - return self.get_token({**(claims or {}), "admin": True}) + return self.get_token({**(claims or {}), "admin": True, "aud": audience}) diff --git a/src/backend/core/services/yhub_services.py b/src/backend/core/services/yhub_services.py index 9cf6a66a47..84b1032780 100644 --- a/src/backend/core/services/yhub_services.py +++ b/src/backend/core/services/yhub_services.py @@ -23,7 +23,7 @@ import requests -from core.services.jwt_services import JWTService +from core.services.jwt_services import Audiences, JWTService logger = logging.getLogger(__name__) @@ -116,7 +116,10 @@ def auth_header(self): document without going through the abilities of a user. The subject it may carry is who the call is for, it never restricts what it can do. """ - return f"Bearer {JWTService().get_admin_token(self.claims)}" + token = JWTService().get_admin_token( + audience=Audiences.YHUB, claims=self.claims + ) + return f"Bearer {token}" def build_url(self, endpoint, document_id): """Build the url of a document scoped endpoint of the yhub API.""" diff --git a/src/backend/core/tests/test_services_converter_services.py b/src/backend/core/tests/test_services_converter_services.py index 949088599f..f19a120f63 100644 --- a/src/backend/core/tests/test_services_converter_services.py +++ b/src/backend/core/tests/test_services_converter_services.py @@ -13,6 +13,7 @@ ValidationError, YdocConverter, ) +from core.services.jwt_services import Audiences from core.tests.utils.jwt_helper import generate_key_pair # Generating an RSA key is expensive, do it once for the whole module @@ -34,10 +35,10 @@ def test_auth_header(): assert scheme == "Bearer" payload = jwt.decode( - token, PUBLIC_KEY, algorithms=["RS256"], audience="y-converter" + token, PUBLIC_KEY, algorithms=["RS256"], audience=Audiences.Y_CONVERTER ) assert payload["admin"] is True - assert payload["aud"] == "y-converter" + assert payload["aud"] == Audiences.Y_CONVERTER def test_convert_empty_text(): diff --git a/src/backend/core/tests/test_services_jwt_services.py b/src/backend/core/tests/test_services_jwt_services.py index eaf0f2989c..c8ffdfb678 100644 --- a/src/backend/core/tests/test_services_jwt_services.py +++ b/src/backend/core/tests/test_services_jwt_services.py @@ -13,6 +13,7 @@ from freezegun import freeze_time from core.services.jwt_services import ( + Audiences, ConfigurationError, JWTService, TokenGenerationError, @@ -132,30 +133,43 @@ def test_get_token_caches_each_set_of_claims_separately(): @pytest.mark.usefixtures("jwt_settings") def test_get_admin_token_carries_the_admin_claim(): """The admin token is a regular token carrying the "admin" claim.""" - token = JWTService().get_admin_token() + token = JWTService().get_admin_token(audience=Audiences.YHUB) - payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + payload = jwt.decode( + token, PUBLIC_KEY, algorithms=["RS256"], audience=Audiences.YHUB + ) assert payload["admin"] is True + assert payload["aud"] == Audiences.YHUB @pytest.mark.usefixtures("jwt_settings") def test_get_admin_token_embeds_the_extra_claims(): """Extra claims are carried alongside the "admin" one.""" - token = JWTService().get_admin_token({"sub": "user-id", "scope": "read"}) + token = JWTService().get_admin_token( + audience=Audiences.YHUB, claims={"sub": "user-id", "scope": "read"} + ) - payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + payload = jwt.decode( + token, PUBLIC_KEY, algorithms=["RS256"], audience=Audiences.YHUB + ) assert payload["admin"] is True assert payload["sub"] == "user-id" assert payload["scope"] == "read" + assert payload["aud"] == Audiences.YHUB @pytest.mark.usefixtures("jwt_settings") def test_get_admin_token_extra_claims_cannot_turn_admin_off(): """🔒 A token issued by get_admin_token always grants admin.""" - token = JWTService().get_admin_token({"admin": False}) + token = JWTService().get_admin_token( + audience=Audiences.YHUB, claims={"admin": False} + ) - payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + payload = jwt.decode( + token, PUBLIC_KEY, algorithms=["RS256"], audience=Audiences.YHUB + ) assert payload["admin"] is True + assert payload["aud"] == Audiences.YHUB @pytest.mark.usefixtures("jwt_settings") @@ -163,13 +177,24 @@ def test_get_admin_token_caches_each_set_of_extra_claims_separately(): """Two callers passing different extra claims get their own token.""" service = JWTService() - first_token = service.get_admin_token({"sub": "user-id"}) - second_token = service.get_admin_token({"sub": "other-user-id"}) + first_token = service.get_admin_token( + audience=Audiences.YHUB, claims={"sub": "user-id"} + ) + second_token = service.get_admin_token( + audience=Audiences.YHUB, claims={"sub": "other-user-id"} + ) assert first_token != second_token - assert jwt.decode(first_token, PUBLIC_KEY, algorithms=["RS256"])["sub"] == "user-id" assert ( - jwt.decode(second_token, PUBLIC_KEY, algorithms=["RS256"])["sub"] + jwt.decode( + first_token, PUBLIC_KEY, algorithms=["RS256"], audience=Audiences.YHUB + )["sub"] + == "user-id" + ) + assert ( + jwt.decode( + second_token, PUBLIC_KEY, algorithms=["RS256"], audience=Audiences.YHUB + )["sub"] == "other-user-id" ) @@ -179,7 +204,7 @@ def test_get_admin_token_does_not_mutate_the_given_claims(): """The caller's dictionary is left untouched.""" claims = {"sub": "user-id"} - JWTService().get_admin_token(claims) + JWTService().get_admin_token(audience=Audiences.YHUB, claims=claims) assert claims == {"sub": "user-id"} @@ -189,10 +214,10 @@ def test_get_admin_token_reuses_the_cached_token(): """The admin token is cached, like any other token.""" service = JWTService() - token = service.get_admin_token() + token = service.get_admin_token(audience=Audiences.YHUB) with mock.patch("core.services.jwt_services.jwt.encode") as mock_encode: - assert service.get_admin_token() == token + assert service.get_admin_token(audience=Audiences.YHUB) == token mock_encode.assert_not_called() @@ -205,7 +230,7 @@ def test_get_admin_token_is_not_served_to_a_non_admin_caller(): """ service = JWTService() - admin_token = service.get_admin_token() + admin_token = service.get_admin_token(audience=Audiences.YHUB) tokens = [ service.get_token({"admin": False}), service.get_token({"sub": "user-id"}), @@ -225,8 +250,10 @@ def test_get_admin_token_expires_like_any_other_token(jwt_settings): now = datetime(2026, 8, 4, 10, 0, 0, tzinfo=timezone.utc) with freeze_time(now): - token = JWTService().get_admin_token() - payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + token = JWTService().get_admin_token(audience=Audiences.YHUB) + payload = jwt.decode( + token, PUBLIC_KEY, algorithms=["RS256"], audience=Audiences.YHUB + ) assert payload["exp"] == now.timestamp() + 120 diff --git a/src/backend/core/tests/test_services_yhub_services.py b/src/backend/core/tests/test_services_yhub_services.py index 1b92569ce0..f57da04243 100644 --- a/src/backend/core/tests/test_services_yhub_services.py +++ b/src/backend/core/tests/test_services_yhub_services.py @@ -9,6 +9,7 @@ import requests from core.factories import UserFactory +from core.services.jwt_services import Audiences from core.services.yhub_services import ( APIError, ConfigurationError, @@ -62,8 +63,11 @@ def test_auth_header(): scheme, token = YHubService().auth_header.split(" ") assert scheme == "Bearer" - payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + payload = jwt.decode( + token, PUBLIC_KEY, algorithms=["RS256"], audience=Audiences.YHUB + ) assert payload["admin"] is True + assert payload["aud"] == Audiences.YHUB assert "sub" not in payload @@ -73,19 +77,25 @@ def test_auth_header_with_user(): _scheme, token = YHubService(user=user).auth_header.split(" ") - payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + payload = jwt.decode( + token, PUBLIC_KEY, algorithms=["RS256"], audience=Audiences.YHUB + ) assert payload["sub"] == str(user.pk) # naming a subject should not restrict what the call can do assert payload["admin"] is True + assert payload["aud"] == Audiences.YHUB def test_auth_header_with_anonymous_user(): """An anonymous user is no subject, the token should not name one.""" _scheme, token = YHubService(user=AnonymousUser()).auth_header.split(" ") - payload = jwt.decode(token, PUBLIC_KEY, algorithms=["RS256"]) + payload = jwt.decode( + token, PUBLIC_KEY, algorithms=["RS256"], audience=Audiences.YHUB + ) assert "sub" not in payload assert payload["admin"] is True + assert payload["aud"] == Audiences.YHUB @patch("requests.request") From 4d17b3264fdd1cafe04f14eebffea20b937c06eb Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Wed, 5 Aug 2026 16:53:14 +0200 Subject: [PATCH 31/59] =?UTF-8?q?=E2=9C=A8(backend)=20implement=20reset-co?= =?UTF-8?q?nnections=20and=20create-ydoc=20in=20YHubService?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reset-connections and create-ydoc are the first action we want to implement in the YHubService. They will be used in next commits. --- src/backend/core/services/yhub_services.py | 70 ++++++++++++-- .../core/tests/test_services_yhub_services.py | 96 +++++++++++++++++-- 2 files changed, 152 insertions(+), 14 deletions(-) diff --git a/src/backend/core/services/yhub_services.py b/src/backend/core/services/yhub_services.py index 84b1032780..ebab7cad37 100644 --- a/src/backend/core/services/yhub_services.py +++ b/src/backend/core/services/yhub_services.py @@ -92,6 +92,19 @@ def timeout(self): """Return the timeout of the requests to the yhub API, in seconds.""" return settings.YHUB_API_TIMEOUT + @property + def user_id(self): + """ + Return the id of the user a call is made on behalf of, if any. + + It is the very id yhub knows a user by: the auth plugin resolves the + cookies of a websocket client to the same one. + """ + if self.user is None or not self.user.is_authenticated: + return None + + return str(self.user.pk) + @property def claims(self): """ @@ -102,10 +115,10 @@ def claims(self): rather than to the backend itself. A call made outside of a request, from a Celery task for instance, has no subject to name. """ - if self.user is None or not self.user.is_authenticated: + if self.user_id is None: return {} - return {"sub": str(self.user.pk)} + return {"sub": self.user_id} @property def auth_header(self): @@ -121,14 +134,24 @@ def auth_header(self): ) return f"Bearer {token}" - def build_url(self, endpoint, document_id): + def build_url(self, endpoint, document): """Build the url of a document scoped endpoint of the yhub API.""" return ( f"{self.base_url}/{self.api_prefix}/{endpoint}/{self.api_version}" - f"/{self.org}/{document_id}" + f"/{self.org}/{document.id}" ) - def request(self, method, url, params=None, data=None): + @staticmethod + def build_user_header(user_id): + """ + Name a user to yhub, or nobody when there is no user to name. + + yhub only reads this header from a call authenticated as admin, and + what it does with it depends on the endpoint it is sent to. + """ + return {"X-User-Id": str(user_id)} if user_id else {} + + def request(self, method, url, data=None, headers=None): """ Send an authenticated request to the yhub API. @@ -139,11 +162,11 @@ def request(self, method, url, params=None, data=None): response = requests.request( method, url, - params=params, data=data, headers={ "Authorization": self.auth_header, "Content-Type": "application/octet-stream", + **(headers or {}), }, timeout=self.timeout, ) @@ -166,3 +189,38 @@ def request(self, method, url, params=None, data=None): ) return response + + def create_ydoc(self, document, update): + """ + Seed the initial Yjs state of a document. + + The body is the raw binary update, what pycrdt's `get_update()` + returns, and not the lib0 encoding the built-in `ydoc` endpoint speaks. + The content is attributed to the user the service is bound to, yhub + only takes our word for it because the token grants admin. + + It is a strict create: yhub answers 409 when the document already has + content, 413 over 10MB and 400 on an update it cannot apply, all + reported as an `APIError` carrying the status. + """ + return self.request( + "post", + self.build_url("create-ydoc", document), + data=update, + headers=self.build_user_header(self.user_id), + ) + + def reset_connections(self, document, user_id=None): + """ + Re-check the access of the clients connected to a document. + + yhub re-runs the authorization of the matching connections and closes + only the ones that lost their access, the others are left alone. + Naming a user restricts the re-check to their own connections, which is + what the change of a single access needs. + """ + return self.request( + "post", + self.build_url("reset-connections", document), + headers=self.build_user_header(user_id), + ) diff --git a/src/backend/core/tests/test_services_yhub_services.py b/src/backend/core/tests/test_services_yhub_services.py index f57da04243..e1b966a57b 100644 --- a/src/backend/core/tests/test_services_yhub_services.py +++ b/src/backend/core/tests/test_services_yhub_services.py @@ -1,6 +1,7 @@ """Test yhub services.""" from unittest.mock import patch +from uuid import uuid4 from django.contrib.auth.models import AnonymousUser @@ -8,6 +9,7 @@ import pytest import requests +from core import models from core.factories import UserFactory from core.services.jwt_services import Audiences from core.services.yhub_services import ( @@ -21,6 +23,9 @@ # Generating an RSA key is expensive, do it once for the whole module PRIVATE_KEY, PUBLIC_KEY = generate_key_pair() +# the service only ever reads the id of the document, no need to save one +DOCUMENT = models.Document(id=uuid4()) + @pytest.fixture(autouse=True) def yhub_settings(settings): @@ -50,12 +55,9 @@ def test_base_url_strips_trailing_slash(settings): def test_build_url(): """A document scoped url should be mounted under the api prefix of yhub.""" - url = YHubService().build_url("ydoc", "8c1c8c4d-4b02-4b0f-a0e9-e00cbd1a9a2f") + url = YHubService().build_url("ydoc", DOCUMENT) - assert url == ( - "http://yhub:3002/collaboration/ydoc/v1/docs/" - "8c1c8c4d-4b02-4b0f-a0e9-e00cbd1a9a2f" - ) + assert url == f"http://yhub:3002/collaboration/ydoc/v1/docs/{DOCUMENT.id!s}" def test_auth_header(): @@ -105,15 +107,19 @@ def test_request(mock_request): service = YHubService() response = service.request( - "get", service.build_url("ydoc", "doc-id"), params={"gc": "false"} + "post", service.build_url("ydoc", DOCUMENT), data=b"body" ) assert response is mock_request.return_value args, kwargs = mock_request.call_args - assert args == ("get", "http://yhub:3002/collaboration/ydoc/v1/docs/doc-id") - assert kwargs["params"] == {"gc": "false"} + assert args == ( + "post", + f"http://yhub:3002/collaboration/ydoc/v1/docs/{DOCUMENT.id!s}", + ) + assert kwargs["data"] == b"body" assert kwargs["timeout"] == 30 assert kwargs["headers"]["Authorization"].startswith("Bearer ") + assert kwargs["headers"]["Content-Type"] == "application/octet-stream" @patch("requests.request") @@ -142,3 +148,77 @@ def test_request_error_status(mock_request): ) assert excinfo.value.status_code == 403 + + +@patch("requests.request") +def test_create_ydoc(mock_request): + """Should post the raw update, unencoded, to the create-ydoc endpoint.""" + mock_request.return_value.ok = True + update = b"\x01\x02\x03\x04" + + response = YHubService().create_ydoc(DOCUMENT, update) + + assert response is mock_request.return_value + args, kwargs = mock_request.call_args + assert args == ( + "post", + f"http://yhub:3002/collaboration/create-ydoc/v1/docs/{DOCUMENT.id!s}", + ) + assert kwargs["data"] == update + # nobody to attribute the content to + assert "X-User-Id" not in kwargs["headers"] + + +@patch("requests.request") +def test_create_ydoc_attributes_the_content_to_the_user(mock_request): + """The content should be attributed to the user the service is bound to.""" + mock_request.return_value.ok = True + user = UserFactory.build() + + YHubService(user=user).create_ydoc(DOCUMENT, b"\x01\x02\x03\x04") + + _args, kwargs = mock_request.call_args + assert kwargs["headers"]["X-User-Id"] == str(user.pk) + + +@patch("requests.request") +def test_create_ydoc_already_exists(mock_request): + """The strict create of yhub should surface as an APIError carrying the 409.""" + mock_request.return_value.ok = False + mock_request.return_value.status_code = 409 + mock_request.return_value.text = "Document already exists" + + with pytest.raises(APIError) as excinfo: + YHubService().create_ydoc(DOCUMENT, b"\x01\x02\x03\x04") + + assert excinfo.value.status_code == 409 + + +@patch("requests.request") +def test_reset_connections(mock_request): + """Should ask yhub to re-check every connection of the document.""" + mock_request.return_value.ok = True + + response = YHubService().reset_connections(DOCUMENT) + + assert response is mock_request.return_value + args, kwargs = mock_request.call_args + assert args == ( + "post", + f"http://yhub:3002/collaboration/reset-connections/v1/docs/{DOCUMENT.id!s}", + ) + # no user named: every connection of the document is re-checked + assert "X-User-Id" not in kwargs["headers"] + + +@patch("requests.request") +def test_reset_connections_of_a_single_user(mock_request): + """Naming a user should restrict the re-check to their own connections.""" + mock_request.return_value.ok = True + user = UserFactory.build() + + # the user whose access changed, not the one making the call + YHubService(user=UserFactory.build()).reset_connections(DOCUMENT, user.pk) + + _args, kwargs = mock_request.call_args + assert kwargs["headers"]["X-User-Id"] == str(user.pk) From b3ad1d1a57896a820c093f6ef2384ee864151529 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Wed, 5 Aug 2026 16:59:11 +0200 Subject: [PATCH 32/59] =?UTF-8?q?=E2=8F=AA=EF=B8=8F(backend)=20reintroduce?= =?UTF-8?q?=20the=20reset=20connection=20mechanism?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When an access change or is deleted or a link configuration changes, we call the yhub server to reset connections and remove them if needed. The YHubService is used for this. --- CHANGELOG.md | 2 + src/backend/core/api/viewsets.py | 25 ++- src/backend/core/tasks/access.py | 42 +++++ .../documents/test_api_document_accesses.py | 149 +++++++++++------- .../test_api_documents_link_configuration.py | 85 ++++++---- ...ternal_api_documents_link_configuration.py | 8 +- src/backend/core/tests/test_tasks_access.py | 83 ++++++++++ 7 files changed, 310 insertions(+), 84 deletions(-) create mode 100644 src/backend/core/tasks/access.py create mode 100644 src/backend/core/tests/test_tasks_access.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 5304d2e554..8d40c041e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,8 @@ and this project adheres to `503` instead of denying access like a permission failure, so clients retry instead of giving up. The built-in endpoints can also answer JSON on `Accept: application/json` +- ✨(backend) reset the yhub connections of a document and its descendants + when an access or the link configuration changes - ✨(backend) add a service to call the yhub REST API - ✨(backend) add a service generating cached RS256 JWT tokens - ✨(backend) publish the JWT public key on a JWKS endpoint diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index e97be4e157..1918ab6045 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -71,6 +71,7 @@ get_document_indexer, get_visited_document_ids_of, ) +from core.tasks.access import reset_service_connections_in_cascade from core.tasks.mail import send_ask_for_access_mail from core.utils.analytics import PosthogEventName, posthog_capture from core.utils.paths import filter_descendants @@ -1752,6 +1753,9 @@ def link_configuration(self, request, *args, **kwargs): serializer.save() + # Notify collaboration server about the link updated + reset_service_connections_in_cascade.delay(str(document.id)) + return drf.response.Response(serializer.data, status=drf.status.HTTP_200_OK) @drf.decorators.action(detail=True, methods=["post", "delete"], url_path="favorite") @@ -2746,12 +2750,28 @@ def perform_create(self, serializer): or settings.LANGUAGE_CODE, ) + def perform_update(self, serializer): + """Update an access to the document and notify the collaboration server.""" + access = serializer.save() + + access_user_id = None + if access.user: + access_user_id = str(access.user.id) + + # Notify collaboration server about the access change + reset_service_connections_in_cascade.delay( + str(access.document.id), access_user_id + ) + def perform_destroy(self, instance): - """Delete an access to the document.""" + """Delete an access to the document and notify the collaboration server.""" # Snapshot the identifiers before deletion as Django resets the primary key # on the instance once it is deleted. access_id = str(instance.id) document_id = str(instance.document_id) + # an access is granted either to a user or to a team, only a user has + # connections of their own to reset + user_id = str(instance.user.id) if instance.user else None instance.delete() @@ -2761,6 +2781,9 @@ def perform_destroy(self, instance): {"access_id": access_id, "document_id": document_id}, ) + # Notify collaboration server about the access removed + reset_service_connections_in_cascade.delay(document_id, user_id) + class InvitationViewset( drf.mixins.CreateModelMixin, diff --git a/src/backend/core/tasks/access.py b/src/backend/core/tasks/access.py new file mode 100644 index 0000000000..821fdb809c --- /dev/null +++ b/src/backend/core/tasks/access.py @@ -0,0 +1,42 @@ +"""Tasks dedicated to document's accesses.""" + +from logging import getLogger + +from core import models +from core.services.yhub_services import YHubError, YHubService + +from impress.celery_app import app + +logger = getLogger(__name__) + + +@app.task +def reset_service_connections_in_cascade(document_id, user_id=None): + """ + Reset the connections of a document and all its descendants on the + collaboration server. + + A document inherits the accesses of its ancestors, so a change on one of + them can revoke the access to the whole subtree: yhub re-checks every + connection of each document and disconnects the ones that lost their + access. The endpoint is document scoped, hence the walk down the tree. + + A document failing is logged and does not stop the ones after it, its + clients keep the rights they connected with until they reconnect. + """ + try: + document = models.Document.objects.get(pk=document_id) + except models.Document.DoesNotExist: + logger.error("Document %s does not exists anymore", document_id) + return + + documents = models.Document.objects.filter( + path__startswith=document.path, depth__gte=document.depth + ).order_by("path") + + service = YHubService() + for doc in documents: + try: + service.reset_connections(doc, user_id) + except YHubError: + logger.exception("impossible to reset connections for document %s", doc.id) diff --git a/src/backend/core/tests/documents/test_api_document_accesses.py b/src/backend/core/tests/documents/test_api_document_accesses.py index 96e3a98def..83c4b392f2 100644 --- a/src/backend/core/tests/documents/test_api_document_accesses.py +++ b/src/backend/core/tests/documents/test_api_document_accesses.py @@ -4,6 +4,7 @@ # pylint: disable=too-many-lines import random +from contextlib import contextmanager from unittest import mock from uuid import uuid4 @@ -18,6 +19,25 @@ pytestmark = pytest.mark.django_db +@pytest.fixture(name="mock_reset_connections") +def mock_reset_connections_fixture(): + """ + Provide a context manager that patches the ``reset_service_connections_in_cascade`` + Celery task and asserts its ``delay`` method is called exactly once for the given + document and user when leaving the context. + """ + + @contextmanager + def _mock_reset_connections(document_id, user_id=None): + with mock.patch( + "core.api.viewsets.reset_service_connections_in_cascade.delay" + ) as mock_delay: + yield mock_delay + mock_delay.assert_called_once_with(str(document_id), user_id) + + return _mock_reset_connections + + def test_api_document_accesses_list_anonymous(): """Anonymous users should not be allowed to list document accesses.""" document = factories.DocumentFactory() @@ -734,6 +754,7 @@ def test_api_document_accesses_update_administrator_except_owner( create_for, via, mock_user_teams, + mock_reset_connections, ): """ A user who is a direct administrator in a document should be allowed to update a user @@ -772,12 +793,13 @@ def test_api_document_accesses_update_administrator_except_owner( for field, value in new_values.items(): new_data = {**old_values, field: value} - response = client.put( - f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", - data=new_data, - format="json", - ) - assert response.status_code == 200 + with mock_reset_connections(document.id, str(access.user_id)): + response = client.put( + f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", + data=new_data, + format="json", + ) + assert response.status_code == 200 access.refresh_from_db() updated_values = serializers.DocumentAccessSerializer(instance=access).data @@ -842,6 +864,7 @@ def test_api_document_accesses_update_administrator_from_owner(via, mock_user_te def test_api_document_accesses_update_administrator_to_owner( via, mock_user_teams, + mock_reset_connections, ): """ A user who is an administrator in a document, should not be allowed to update @@ -889,12 +912,13 @@ def test_api_document_accesses_update_administrator_to_owner( assert response.status_code == 403 else: - response = client.put( - f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", - data=new_data, - format="json", - ) - assert response.status_code == 200 + with mock_reset_connections(document.id, str(access.user_id)): + response = client.put( + f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", + data=new_data, + format="json", + ) + assert response.status_code == 200 access.refresh_from_db() updated_values = serializers.DocumentAccessSerializer(instance=access).data @@ -907,6 +931,7 @@ def test_api_document_accesses_update_owner( create_for, via, mock_user_teams, + mock_reset_connections, ): """ A user who is an owner in a document should be allowed to update @@ -943,13 +968,14 @@ def test_api_document_accesses_update_owner( for field, value in new_values.items(): new_data = {**old_values, field: value} - response = client.put( - f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", - data=new_data, - format="json", - ) + with mock_reset_connections(document.id, str(access.user_id)): + response = client.put( + f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", + data=new_data, + format="json", + ) - assert response.status_code == 200 + assert response.status_code == 200 access.refresh_from_db() updated_values = serializers.DocumentAccessSerializer(instance=access).data @@ -968,6 +994,7 @@ def test_api_document_accesses_update_owner( def test_api_document_accesses_update_owner_self_root( via, mock_user_teams, + mock_reset_connections, ): """ A user who is owner of a document should be allowed to update @@ -1006,27 +1033,30 @@ def test_api_document_accesses_update_owner_self_root( # Add another owner and it should now work factories.UserDocumentAccessFactory(document=document, role="owner") - response = client.put( - f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", - data={ - **old_values, - "role": new_role, - "user_id": old_values.get("user", {}).get("id") - if old_values.get("user") is not None - else None, - }, - format="json", - ) + user_id = str(access.user_id) if via == USER else None + with mock_reset_connections(document.id, user_id): + response = client.put( + f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", + data={ + **old_values, + "role": new_role, + "user_id": old_values.get("user", {}).get("id") + if old_values.get("user") is not None + else None, + }, + format="json", + ) - assert response.status_code == 200 - access.refresh_from_db() - assert access.role == new_role + assert response.status_code == 200 + access.refresh_from_db() + assert access.role == new_role @pytest.mark.parametrize("via", VIA) def test_api_document_accesses_update_owner_self_child( via, mock_user_teams, + mock_reset_connections, ): """ A user who is owner of a document should be allowed to update @@ -1054,11 +1084,13 @@ def test_api_document_accesses_update_owner_self_child( old_values = serializers.DocumentAccessSerializer(instance=access).data new_role = random.choice(["administrator", "editor", "reader"]) - response = client.put( - f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", - data={**old_values, "role": new_role}, - format="json", - ) + user_id = str(access.user_id) if via == USER else None + with mock_reset_connections(document.id, user_id): + response = client.put( + f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", + data={**old_values, "role": new_role}, + format="json", + ) assert response.status_code == 200 access.refresh_from_db() @@ -1138,6 +1170,7 @@ def test_api_document_accesses_delete_reader_or_editor(via, role, mock_user_team def test_api_document_accesses_delete_administrators_except_owners( via, mock_user_teams, + mock_reset_connections, ): """ Users who are administrators in a document should be allowed to delete an access @@ -1166,13 +1199,14 @@ def test_api_document_accesses_delete_administrators_except_owners( assert models.DocumentAccess.objects.count() == 2 assert models.DocumentAccess.objects.filter(user=access.user).exists() - with mock.patch("core.api.viewsets.posthog_capture") as mock_capture: - response = client.delete( - f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", - ) + with mock_reset_connections(document.id, str(access.user_id)): + with mock.patch("core.api.viewsets.posthog_capture") as mock_capture: + response = client.delete( + f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", + ) - assert response.status_code == 204 - assert models.DocumentAccess.objects.count() == 1 + assert response.status_code == 204 + assert models.DocumentAccess.objects.count() == 1 # The access deletion should be tracked in PostHog mock_capture.assert_called_once_with( @@ -1221,6 +1255,7 @@ def test_api_document_accesses_delete_administrator_on_owners(via, mock_user_tea def test_api_document_accesses_delete_owners( via, mock_user_teams, + mock_reset_connections, ): """ Users should be able to delete the document access of another user @@ -1245,10 +1280,11 @@ def test_api_document_accesses_delete_owners( assert models.DocumentAccess.objects.count() == 2 assert models.DocumentAccess.objects.filter(user=access.user).exists() - with mock.patch("core.api.viewsets.posthog_capture") as mock_capture: - response = client.delete( - f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", - ) + with mock_reset_connections(document.id, str(access.user_id)): + with mock.patch("core.api.viewsets.posthog_capture") as mock_capture: + response = client.delete( + f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", + ) assert response.status_code == 204 assert models.DocumentAccess.objects.count() == 1 @@ -1291,7 +1327,9 @@ def test_api_document_accesses_delete_owners_last_owner_root(via, mock_user_team assert models.DocumentAccess.objects.count() == 2 -def test_api_document_accesses_delete_owners_last_owner_child_user(): +def test_api_document_accesses_delete_owners_last_owner_child_user( + mock_reset_connections, +): """ It should be possible to delete the last owner access from a document that is not a root. """ @@ -1307,9 +1345,10 @@ def test_api_document_accesses_delete_owners_last_owner_child_user(): ) assert models.DocumentAccess.objects.count() == 2 - response = client.delete( - f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", - ) + with mock_reset_connections(document.id, str(access.user_id)): + response = client.delete( + f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", + ) assert response.status_code == 204 assert models.DocumentAccess.objects.count() == 1 @@ -1320,6 +1359,7 @@ def test_api_document_accesses_delete_owners_last_owner_child_user(): ) def test_api_document_accesses_delete_owners_last_owner_child_team( mock_user_teams, + mock_reset_connections, ): """ It should be possible to delete the last owner access from a document that @@ -1338,9 +1378,10 @@ def test_api_document_accesses_delete_owners_last_owner_child_team( ) assert models.DocumentAccess.objects.count() == 2 - response = client.delete( - f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", - ) + with mock_reset_connections(document.id, str(access.user_id)): + response = client.delete( + f"/api/v1.0/documents/{document.id!s}/accesses/{access.id!s}/", + ) assert response.status_code == 204 assert models.DocumentAccess.objects.count() == 1 diff --git a/src/backend/core/tests/documents/test_api_documents_link_configuration.py b/src/backend/core/tests/documents/test_api_documents_link_configuration.py index 7252b1fc05..9a3a3c8fdf 100644 --- a/src/backend/core/tests/documents/test_api_documents_link_configuration.py +++ b/src/backend/core/tests/documents/test_api_documents_link_configuration.py @@ -1,5 +1,8 @@ """Tests for link configuration of documents on API endpoint""" +from contextlib import contextmanager +from unittest import mock + import pytest from rest_framework.test import APIClient @@ -10,6 +13,25 @@ pytestmark = pytest.mark.django_db +@pytest.fixture(name="mock_reset_connections") +def mock_reset_connections_fixture(): + """ + Provide a context manager that patches the ``reset_service_connections_in_cascade`` + Celery task and asserts its ``delay`` method is called exactly once for the given + document when leaving the context. + """ + + @contextmanager + def _mock_reset_connections(document_id): + with mock.patch( + "core.api.viewsets.reset_service_connections_in_cascade.delay" + ) as mock_delay: + yield mock_delay + mock_delay.assert_called_once_with(str(document_id)) + + return _mock_reset_connections + + @pytest.mark.parametrize("role", models.LinkRoleChoices.values) @pytest.mark.parametrize("reach", models.LinkReachChoices.values) def test_api_documents_link_configuration_update_anonymous(reach, role): @@ -119,6 +141,7 @@ def test_api_documents_link_configuration_update_authenticated_related_success( via, role, mock_user_teams, + mock_reset_connections, # pylint: disable=redefined-outer-name ): """ A user who is administrator or owner of a document should be allowed to update @@ -148,17 +171,18 @@ def test_api_documents_link_configuration_update_authenticated_related_success( ) ).data - response = client.put( - f"/api/v1.0/documents/{document.id!s}/link-configuration/", - new_document_values, - format="json", - ) - assert response.status_code == 200 + with mock_reset_connections(document.id): + response = client.put( + f"/api/v1.0/documents/{document.id!s}/link-configuration/", + new_document_values, + format="json", + ) + assert response.status_code == 200 - document = models.Document.objects.get(pk=document.pk) - document_values = serializers.LinkDocumentSerializer(instance=document).data - for key, value in document_values.items(): - assert value == new_document_values[key] + document = models.Document.objects.get(pk=document.pk) + document_values = serializers.LinkDocumentSerializer(instance=document).data + for key, value in document_values.items(): + assert value == new_document_values[key] def test_api_documents_link_configuration_update_role_restricted_forbidden(): @@ -230,7 +254,9 @@ def test_api_documents_link_configuration_update_link_reach_required(): assert "This field is required" in response.json()["link_reach"][0] -def test_api_documents_link_configuration_update_restricted_without_role_success(): +def test_api_documents_link_configuration_update_restricted_without_role_success( + mock_reset_connections, # pylint: disable=redefined-outer-name +): """ Test that setting link_reach to restricted without specifying link_role succeeds. """ @@ -252,15 +278,16 @@ def test_api_documents_link_configuration_update_restricted_without_role_success "link_reach": models.LinkReachChoices.RESTRICTED, } - response = client.put( - f"/api/v1.0/documents/{document.id!s}/link-configuration/", - new_data, - format="json", - ) + with mock_reset_connections(document.id): + response = client.put( + f"/api/v1.0/documents/{document.id!s}/link-configuration/", + new_data, + format="json", + ) - assert response.status_code == 200 - document.refresh_from_db() - assert document.link_reach == models.LinkReachChoices.RESTRICTED + assert response.status_code == 200 + document.refresh_from_db() + assert document.link_reach == models.LinkReachChoices.RESTRICTED @pytest.mark.parametrize( @@ -270,6 +297,7 @@ def test_api_documents_link_configuration_update_restricted_without_role_success def test_api_documents_link_configuration_update_non_restricted_with_valid_role_success( reach, role, + mock_reset_connections, # pylint: disable=redefined-outer-name ): """ Test that setting non-restricted link_reach with valid link_role succeeds. @@ -292,16 +320,17 @@ def test_api_documents_link_configuration_update_non_restricted_with_valid_role_ "link_role": role, } - response = client.put( - f"/api/v1.0/documents/{document.id!s}/link-configuration/", - new_data, - format="json", - ) + with mock_reset_connections(document.id): + response = client.put( + f"/api/v1.0/documents/{document.id!s}/link-configuration/", + new_data, + format="json", + ) - assert response.status_code == 200 - document.refresh_from_db() - assert document.link_reach == reach - assert document.link_role == role + assert response.status_code == 200 + document.refresh_from_db() + assert document.link_reach == reach + assert document.link_role == role def test_api_documents_link_configuration_update_with_ancestor_constraints(): diff --git a/src/backend/core/tests/external_api/test_external_api_documents_link_configuration.py b/src/backend/core/tests/external_api/test_external_api_documents_link_configuration.py index 38f7f732a4..7c1e6a3086 100644 --- a/src/backend/core/tests/external_api/test_external_api_documents_link_configuration.py +++ b/src/backend/core/tests/external_api/test_external_api_documents_link_configuration.py @@ -6,6 +6,8 @@ """ +from unittest.mock import patch + from django.test import override_settings import pytest @@ -59,8 +61,9 @@ def test_external_api_documents_link_configuration_not_allowed( }, }, ) +@patch("core.api.viewsets.reset_service_connections_in_cascade.delay") def test_external_api_documents_link_configuration_can_be_allowed( - user_token, resource_server_backend, user_specific_sub + mock_reset, user_token, resource_server_backend, user_specific_sub ): """ Connected users SHOULD be allowed to update the link configuration of a document @@ -98,3 +101,6 @@ def test_external_api_documents_link_configuration_can_be_allowed( document.refresh_from_db() assert document.link_reach == models.LinkReachChoices.PUBLIC assert document.link_role == models.LinkRoleChoices.EDITOR + + # the collaboration server should be notified through the Celery task + mock_reset.assert_called_once_with(str(document.id)) diff --git a/src/backend/core/tests/test_tasks_access.py b/src/backend/core/tests/test_tasks_access.py new file mode 100644 index 0000000000..05a7add723 --- /dev/null +++ b/src/backend/core/tests/test_tasks_access.py @@ -0,0 +1,83 @@ +""" +Tests for the `reset_service_connections_in_cascade` Celery task in the +core.tasks.access module. +""" + +from unittest import mock + +import pytest + +from core import factories +from core.services.yhub_services import ServiceUnavailableError +from core.tasks.access import reset_service_connections_in_cascade + +pytestmark = pytest.mark.django_db + + +@mock.patch("core.tasks.access.YHubService") +def test_reset_service_connections_resets_the_document(mock_service): + """The task should reset the connections of the document it is given.""" + document = factories.DocumentFactory() + + reset_service_connections_in_cascade(str(document.id)) + + mock_service.return_value.reset_connections.assert_called_once_with(document, None) + + +@mock.patch("core.tasks.access.YHubService") +def test_reset_service_connections_forwards_the_user_id(mock_service): + """The user whose access changed should be forwarded to the service.""" + document = factories.DocumentFactory() + + reset_service_connections_in_cascade(str(document.id), "user-id") + + mock_service.return_value.reset_connections.assert_called_once_with( + document, "user-id" + ) + + +@mock.patch("core.tasks.access.YHubService") +def test_reset_service_connections_in_cascade(mock_service): + """ + A document inherits the accesses of its ancestors, so the whole subtree + should be reset, the document itself included and its ancestors left out. + """ + parent = factories.DocumentFactory() + document = factories.DocumentFactory(parent=parent) + child = factories.DocumentFactory(parent=document) + grand_child = factories.DocumentFactory(parent=child) + factories.DocumentFactory() # a document of another tree + + reset_service_connections_in_cascade(str(document.id)) + + assert mock_service.return_value.reset_connections.call_args_list == [ + mock.call(document, None), + mock.call(child, None), + mock.call(grand_child, None), + ] + + +@mock.patch("core.tasks.access.YHubService") +def test_reset_service_connections_unknown_document(mock_service): + """A document deleted in the meantime should not reach the service.""" + reset_service_connections_in_cascade("d43ea3c5-b8ee-4a4a-9c60-2ad7a1d9e6cf") + + mock_service.return_value.reset_connections.assert_not_called() + + +@mock.patch("core.tasks.access.YHubService") +def test_reset_service_connections_keeps_going_on_failure(mock_service): + """A document failing should not deprive the ones after it of their reset.""" + document = factories.DocumentFactory() + child = factories.DocumentFactory(parent=document) + mock_service.return_value.reset_connections.side_effect = [ + ServiceUnavailableError("yhub is down"), + None, + ] + + reset_service_connections_in_cascade(str(document.id)) + + assert mock_service.return_value.reset_connections.call_args_list == [ + mock.call(document, None), + mock.call(child, None), + ] From e9bf22f36d56679a3ebab20a61dff407a06945a4 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Thu, 6 Aug 2026 09:35:06 +0200 Subject: [PATCH 33/59] =?UTF-8?q?=E2=9C=A8(backend)=20call=20YHubService?= =?UTF-8?q?=20to=20seed=20initial=20document=20content?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a new Docs is created and a file is sent, as before we convert it first and we need to use the raw content to seed it by calling the create-ydoc api in the YHub service. --- CHANGELOG.md | 1 + src/backend/core/api/serializers.py | 55 ++++---- src/backend/core/api/viewsets.py | 121 +++++++++++------- .../core/services/converter_services.py | 11 +- .../test_api_documents_children_create.py | 12 +- .../test_api_documents_create_for_owner.py | 108 ++++++++++++---- .../test_api_documents_create_with_file.py | 97 ++++++++++---- .../test_external_api_documents.py | 13 +- .../tests/test_services_converter_services.py | 4 +- 9 files changed, 296 insertions(+), 126 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d40c041e2..ba74f1ce41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,7 @@ and this project adheres to `503` instead of denying access like a permission failure, so clients retry instead of giving up. The built-in endpoints can also answer JSON on `Accept: application/json` +- ✨(backend) call YHubService to seed initial document content - ✨(backend) reset the yhub connections of a document and its descendants when an access or the link configuration changes - ✨(backend) add a service to call the yhub REST API diff --git a/src/backend/core/api/serializers.py b/src/backend/core/api/serializers.py index d6b2b422c2..5bf57ebd86 100644 --- a/src/backend/core/api/serializers.py +++ b/src/backend/core/api/serializers.py @@ -7,6 +7,7 @@ from os.path import splitext from django.conf import settings +from django.db import transaction from django.db.models import Q from django.utils.functional import lazy from django.utils.text import slugify @@ -23,6 +24,7 @@ ConversionError, Converter, ) +from core.services.yhub_services import YHubError, YHubService from core.utils.analytics import PosthogEventName, posthog_capture from core.utils.treebeard import create_tree_node_with_retry @@ -482,12 +484,37 @@ def create(self, validated_data): {"content": ["Could not convert content"]} ) from err - document = create_tree_node_with_retry( - lambda: models.Document.add_root( - title=validated_data["title"], - creator=user, + with transaction.atomic(): + document = create_tree_node_with_retry( + lambda: models.Document.add_root( + title=validated_data["title"], + creator=user, + ) ) - ) + + if user: + # Associate the document with the pre-existing user + models.DocumentAccess.objects.create( + document=document, + role=models.RoleChoices.OWNER, + user=user, + ) + else: + # The user doesn't exist in our database: we need to invite him/her + models.Invitation.objects.create( + document=document, + email=email, + role=models.RoleChoices.OWNER, + ) + + # the accesses exist by now, so the owner has access to the very + # first version of the document the collaboration server saves + try: + YHubService(user=user).create_ydoc(document, document_content) + except YHubError as err: + raise serializers.ValidationError( + {"content": ["Could not save the document content"]} + ) from err posthog_capture(PosthogEventName.DOC_CREATED, user, {}, document=document) posthog_capture( @@ -500,24 +527,6 @@ def create(self, validated_data): document=document, ) - if user: - # Associate the document with the pre-existing user - models.DocumentAccess.objects.create( - document=document, - role=models.RoleChoices.OWNER, - user=user, - ) - else: - # The user doesn't exist in our database: we need to invite him/her - models.Invitation.objects.create( - document=document, - email=email, - role=models.RoleChoices.OWNER, - ) - - document.content = document_content - document.save() - if validated_data.get("send_notification_email", True): self._send_email_notification(document, validated_data, email, language) return document diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index 1918ab6045..a7f3b32044 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -71,6 +71,7 @@ get_document_indexer, get_visited_document_ids_of, ) +from core.services.yhub_services import YHubError, YHubService from core.tasks.access import reset_service_connections_in_cascade from core.tasks.mail import send_ask_for_access_mail from core.utils.analytics import PosthogEventName, posthog_capture @@ -694,8 +695,13 @@ def _apply_uploaded_file_conversion(self, serializer): """ Check if a file has been uploaded with a doc or a children is created. If a file is present and the conversion upload enabled, the file is converted - using the converter service and the validated_data in the serializer are filled - with the converted file and the file name. + using the converter service and the title in the serializer is filled with + the file name. + + Return the converted content, as a raw Yjs update the collaboration + server can be seeded with, or None when no file was uploaded. The + content itself is not stored by Django, it is saved by the + collaboration server. """ uploaded_file = serializer.validated_data.pop("file", None) @@ -704,49 +710,72 @@ def _apply_uploaded_file_conversion(self, serializer): {"file": ["file upload is not allowed"]} ) - # If a file is uploaded, convert it to Yjs format and set as content - if uploaded_file: - try: - file_content = uploaded_file.read() + if not uploaded_file: + return None - converter = Converter() - converted_content = converter.convert( - file_content, - content_type=uploaded_file.content_type, - accept=mime_types.YJS, - ) - serializer.validated_data["content"] = converted_content - serializer.validated_data["title"] = uploaded_file.name - logger.info("conversion ended successfully") - - posthog_capture( - PosthogEventName.DOC_IMPORTED, - self.request.user, - {"content_type": uploaded_file.content_type}, - ) - except ConversionError as err: - logger.error("could not convert file content with error: %s", err) - raise drf.exceptions.ValidationError( - {"file": ["Could not convert file content"]} - ) from err + # If a file is uploaded, convert it to Yjs format + try: + file_content = uploaded_file.read() + + converter = Converter() + converted_content = converter.convert( + file_content, + content_type=uploaded_file.content_type, + accept=mime_types.YJS, + ) + serializer.validated_data["title"] = uploaded_file.name + logger.info("conversion ended successfully") + + posthog_capture( + PosthogEventName.DOC_IMPORTED, + self.request.user, + {"content_type": uploaded_file.content_type}, + ) + except ConversionError as err: + logger.error("could not convert file content with error: %s", err) + raise drf.exceptions.ValidationError( + {"file": ["Could not convert file content"]} + ) from err + + return converted_content + + def _create_collaboration_document(self, document, update): + """ + Seed a freshly created document with the content it was imported from. + + The collaboration server owns the content from there on, so a failure + here leaves a document that lost what was uploaded: it is reported to + the caller, who is left to create it again. + """ + try: + YHubService(user=self.request.user).create_ydoc(document, update) + except YHubError as err: + logger.error("could not save the imported content with error: %s", err) + raise drf.exceptions.ValidationError( + {"file": ["Could not save the imported file content"]} + ) from err def perform_create(self, serializer): """Set the current user as creator and owner of the newly created object.""" - self._apply_uploaded_file_conversion(serializer) + update = self._apply_uploaded_file_conversion(serializer) - obj = create_tree_node_with_retry( - lambda: models.Document.add_root( - creator=self.request.user, - **serializer.validated_data, + with transaction.atomic(): + obj = create_tree_node_with_retry( + lambda: models.Document.add_root( + creator=self.request.user, + **serializer.validated_data, + ) + ) + serializer.instance = obj + models.DocumentAccess.objects.create( + document=obj, + user=self.request.user, + role=models.RoleChoices.OWNER, ) - ) - serializer.instance = obj - models.DocumentAccess.objects.create( - document=obj, - user=self.request.user, - role=models.RoleChoices.OWNER, - ) + + if update is not None: + self._create_collaboration_document(obj, update) posthog_capture( PosthogEventName.DOC_CREATED, self.request.user, {}, document=obj @@ -1019,14 +1048,18 @@ def children(self, request, *args, **kwargs): ) serializer.is_valid(raise_exception=True) - self._apply_uploaded_file_conversion(serializer) + update = self._apply_uploaded_file_conversion(serializer) - child_document = create_tree_node_with_retry( - lambda: document.add_child( - creator=request.user, - **serializer.validated_data, + with transaction.atomic(): + child_document = create_tree_node_with_retry( + lambda: document.add_child( + creator=request.user, + **serializer.validated_data, + ) ) - ) + + if update is not None: + self._create_collaboration_document(child_document, update) # Set the created instance to the serializer serializer.instance = child_document diff --git a/src/backend/core/services/converter_services.py b/src/backend/core/services/converter_services.py index a5edd41953..7a550d18cc 100644 --- a/src/backend/core/services/converter_services.py +++ b/src/backend/core/services/converter_services.py @@ -2,7 +2,6 @@ import logging import typing -from base64 import b64encode from django.conf import settings @@ -137,7 +136,13 @@ def _request(self, url, data, content_type, accept): return response def convert(self, data, content_type=mime_types.MARKDOWN, accept=mime_types.YJS): - """Convert a Markdown text into our internal format using an external microservice.""" + """ + Convert a Markdown text into our internal format using an external microservice. + + A Yjs document is returned as the raw update the collaboration server + expects. It is base64 encoded only by the callers storing it in the + text content of a document. + """ if not data: raise ValidationError("Input data cannot be empty") @@ -146,7 +151,7 @@ def convert(self, data, content_type=mime_types.MARKDOWN, accept=mime_types.YJS) try: response = self._request(url, data, content_type, accept) if accept == mime_types.YJS: - return b64encode(response.content).decode("utf-8") + return response.content if accept in {mime_types.MARKDOWN, "text/html"}: return response.text if accept == mime_types.JSON: diff --git a/src/backend/core/tests/documents/test_api_documents_children_create.py b/src/backend/core/tests/documents/test_api_documents_children_create.py index d5e25f7bb5..679edca8f8 100644 --- a/src/backend/core/tests/documents/test_api_documents_children_create.py +++ b/src/backend/core/tests/documents/test_api_documents_children_create.py @@ -314,8 +314,11 @@ def create_document(): assert document.numchild == 2 +@patch("core.api.viewsets.YHubService") @patch("core.services.converter_services.Converter.convert") -def test_api_documents_children_create_with_docx_file_success(mock_convert, settings): +def test_api_documents_children_create_with_docx_file_success( + mock_convert, mock_yhub, settings +): """ Authenticated users should be able to create children document by uploading a DOCX file. The file should be converted to YJS format and the title should be set from filename. @@ -327,7 +330,7 @@ def test_api_documents_children_create_with_docx_file_success(mock_convert, sett settings.CONVERSION_UPLOAD_ENABLED = True # Mock the conversion - converted_yjs = "base64encodedyjscontent" + converted_yjs = b"\x01\x02raw yjs update" mock_convert.return_value = converted_yjs # Create a fake DOCX file @@ -350,7 +353,10 @@ def test_api_documents_children_create_with_docx_file_success(mock_convert, sett assert Document.objects.count() == 2 children = Document.objects.get(pk=response.json()["id"]) assert children.title == "My Important Document.docx" - assert children.content == converted_yjs + # the content is saved by the collaboration server, not by Django + assert children.content is None + mock_yhub.assert_called_once_with(user=user) + mock_yhub.return_value.create_ydoc.assert_called_once_with(children, converted_yjs) # Verify the converter was called correctly mock_convert.assert_called_once_with( diff --git a/src/backend/core/tests/documents/test_api_documents_create_for_owner.py b/src/backend/core/tests/documents/test_api_documents_create_for_owner.py index b72bc84589..c27d57b165 100644 --- a/src/backend/core/tests/documents/test_api_documents_create_for_owner.py +++ b/src/backend/core/tests/documents/test_api_documents_create_for_owner.py @@ -20,18 +20,32 @@ from core.models import Document, Invitation, User from core.services import mime_types from core.services.converter_services import ConversionError, YdocConverter +from core.services.yhub_services import ( + ServiceUnavailableError as YHubServiceUnavailableError, +) from core.utils.analytics import PosthogEventName pytestmark = pytest.mark.django_db +# the converter returns a raw Yjs update, saved by the collaboration server +CONVERTED_CONTENT = b"Converted document content" + + +@pytest.fixture(autouse=True, name="mock_yhub") +def mock_yhub_fixture(): + """No test of this module should reach the collaboration server.""" + with patch("core.api.serializers.YHubService") as mock: + yield mock + + @pytest.fixture def mock_convert_md(): """Mock YdocConverter.convert to return a converted content.""" with patch.object( YdocConverter, "convert", - return_value="Converted document content", + return_value=CONVERTED_CONTENT, ) as mock: yield mock @@ -172,7 +186,7 @@ def test_api_documents_create_for_owner_invalid_sub(): @override_settings(SERVER_TO_SERVER_API_TOKENS=["DummyToken"]) -def test_api_documents_create_for_owner_existing(mock_convert_md): +def test_api_documents_create_for_owner_existing(mock_convert_md, mock_yhub): """ It should be possible to create a document on behalf of a pre-existing user by passing their sub and email. @@ -204,7 +218,11 @@ def test_api_documents_create_for_owner_existing(mock_convert_md): assert response.json() == {"id": str(document.id)} assert document.title == "My Document" - assert document.content == "Converted document content" + # the content is saved by the collaboration server, not by Django + assert document.content is None + mock_yhub.return_value.create_ydoc.assert_called_once_with( + document, CONVERTED_CONTENT + ) assert document.creator == user assert document.accesses.filter(user=user, role="owner").exists() @@ -240,7 +258,7 @@ def test_api_documents_create_for_owner_existing(mock_convert_md): @override_settings(SERVER_TO_SERVER_API_TOKENS=["DummyToken"]) -def test_api_documents_create_for_owner_new_user(mock_convert_md): +def test_api_documents_create_for_owner_new_user(mock_convert_md, mock_yhub): """ It should be possible to create a document on behalf of new users by passing their unknown sub and email address. @@ -270,7 +288,11 @@ def test_api_documents_create_for_owner_new_user(mock_convert_md): assert response.json() == {"id": str(document.id)} assert document.title == "My Document" - assert document.content == "Converted document content" + # the content is saved by the collaboration server, not by Django + assert document.content is None + mock_yhub.return_value.create_ydoc.assert_called_once_with( + document, CONVERTED_CONTENT + ) assert document.creator is None assert document.accesses.exists() is False @@ -344,7 +366,7 @@ def test_api_documents_create_for_owner_without_notification_email(mock_convert_ OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION=True, ) def test_api_documents_create_for_owner_existing_user_email_no_sub_with_fallback( - mock_convert_md, + mock_convert_md, mock_yhub ): """ It should be possible to create a document on behalf of a pre-existing user for @@ -378,7 +400,11 @@ def test_api_documents_create_for_owner_existing_user_email_no_sub_with_fallback assert response.json() == {"id": str(document.id)} assert document.title == "My Document" - assert document.content == "Converted document content" + # the content is saved by the collaboration server, not by Django + assert document.content is None + mock_yhub.return_value.create_ydoc.assert_called_once_with( + document, CONVERTED_CONTENT + ) assert document.creator == user assert document.accesses.filter(user=user, role="owner").exists() @@ -444,7 +470,7 @@ def test_api_documents_create_for_owner_existing_user_email_no_sub_no_fallback( OIDC_ALLOW_DUPLICATE_EMAILS=True, ) def test_api_documents_create_for_owner_new_user_no_sub_no_fallback_allow_duplicate( - mock_convert_md, + mock_convert_md, mock_yhub ): """ When a user does not match an existing sub and fallback to matching on email is @@ -476,7 +502,11 @@ def test_api_documents_create_for_owner_new_user_no_sub_no_fallback_allow_duplic assert response.json() == {"id": str(document.id)} assert document.title == "My Document" - assert document.content == "Converted document content" + # the content is saved by the collaboration server, not by Django + assert document.content is None + mock_yhub.return_value.create_ydoc.assert_called_once_with( + document, CONVERTED_CONTENT + ) assert document.creator is None assert document.accesses.exists() is False @@ -669,21 +699,20 @@ def test_api_documents_create_for_owner_with_converter_exception( @override_settings(SERVER_TO_SERVER_API_TOKENS=["DummyToken"]) @pytest.mark.usefixtures("mock_convert_md") -def test_api_documents_create_for_owner_access_before_content(): +def test_api_documents_create_for_owner_access_before_content(mock_yhub): """ - Accesses must exist before content is saved to object storage so the owner - has access to the very first version of the document. + Accesses must exist before the content is sent to the collaboration server + so the owner has access to the very first version of the document. """ user = factories.UserFactory() accesses_at_save_time = [] - original_save_content = Document.save_content - - def capturing_save_content(self, content): + def capturing_create_ydoc(document, _update): accesses_at_save_time.extend( - list(self.accesses.values_list("user__sub", "role")) + list(document.accesses.values_list("user__sub", "role")) ) - return original_save_content(self, content) + + mock_yhub.return_value.create_ydoc.side_effect = capturing_create_ydoc data = { "title": "My Document", @@ -692,16 +721,15 @@ def capturing_save_content(self, content): "email": user.email, } - with patch.object(Document, "save_content", capturing_save_content): - response = APIClient().post( - "/api/v1.0/documents/create-for-owner/", - data, - format="json", - HTTP_AUTHORIZATION="Bearer DummyToken", - ) + response = APIClient().post( + "/api/v1.0/documents/create-for-owner/", + data, + format="json", + HTTP_AUTHORIZATION="Bearer DummyToken", + ) assert response.status_code == 201 - # The owner access must already exist when save_content is called + # The owner access must already exist when the content is saved assert (str(user.sub), "owner") in accesses_at_save_time @@ -729,3 +757,33 @@ def test_api_documents_create_for_owner_with_empty_content(): "This field may not be blank.", ], } + + +@override_settings(SERVER_TO_SERVER_API_TOKENS=["DummyToken"]) +@pytest.mark.usefixtures("mock_convert_md") +def test_api_documents_create_for_owner_collaboration_server_unavailable(mock_yhub): + """ + A document whose content could not be saved by the collaboration server + should not be created at all, neither should its access. + """ + user = factories.UserFactory() + mock_yhub.return_value.create_ydoc.side_effect = YHubServiceUnavailableError( + "Failed to connect to the yhub service" + ) + + response = APIClient().post( + "/api/v1.0/documents/create-for-owner/", + { + "title": "My Document", + "content": "Document content", + "sub": str(user.sub), + "email": user.email, + }, + format="json", + HTTP_AUTHORIZATION="Bearer DummyToken", + ) + + assert response.status_code == 400 + assert response.json() == {"content": ["Could not save the document content"]} + assert Document.objects.exists() is False + assert len(mail.outbox) == 0 diff --git a/src/backend/core/tests/documents/test_api_documents_create_with_file.py b/src/backend/core/tests/documents/test_api_documents_create_with_file.py index ecafeb028c..89066c9654 100644 --- a/src/backend/core/tests/documents/test_api_documents_create_with_file.py +++ b/src/backend/core/tests/documents/test_api_documents_create_with_file.py @@ -2,7 +2,6 @@ Tests for Documents API endpoint in impress's core app: create with file upload """ -from base64 import b64decode, binascii from io import BytesIO from unittest.mock import patch @@ -16,6 +15,9 @@ ConversionError, ServiceUnavailableError, ) +from core.services.yhub_services import ( + ServiceUnavailableError as YHubServiceUnavailableError, +) from core.utils.analytics import PosthogEventName pytestmark = pytest.mark.django_db @@ -40,8 +42,9 @@ def test_api_documents_create_with_file_anonymous(): assert not Document.objects.exists() +@patch("core.api.viewsets.YHubService") @patch("core.services.converter_services.Converter.convert") -def test_api_documents_create_with_docx_file_success(mock_convert, settings): +def test_api_documents_create_with_docx_file_success(mock_convert, mock_yhub, settings): """ Authenticated users should be able to create documents by uploading a DOCX file. The file should be converted to YJS format and the title should be set from filename. @@ -53,7 +56,7 @@ def test_api_documents_create_with_docx_file_success(mock_convert, settings): settings.CONVERSION_UPLOAD_ENABLED = True # Mock the conversion - converted_yjs = "base64encodedyjscontent" + converted_yjs = b"\x01\x02raw yjs update" mock_convert.return_value = converted_yjs # Create a fake DOCX file @@ -73,9 +76,13 @@ def test_api_documents_create_with_docx_file_success(mock_convert, settings): assert response.status_code == 201 document = Document.objects.get() assert document.title == "My Important Document.docx" - assert document.content == converted_yjs + # the content is saved by the collaboration server, not by Django + assert document.content is None assert document.accesses.filter(role="owner", user=user).exists() + mock_yhub.assert_called_once_with(user=user) + mock_yhub.return_value.create_ydoc.assert_called_once_with(document, converted_yjs) + # Verify the converter was called correctly mock_convert.assert_called_once_with( file_content, @@ -134,8 +141,11 @@ def test_api_documents_create_with_docx_file_disabled(mock_convert, settings): mock_capture.assert_not_called() +@patch("core.api.viewsets.YHubService") @patch("core.services.converter_services.Converter.convert") -def test_api_documents_create_with_markdown_file_success(mock_convert, settings): +def test_api_documents_create_with_markdown_file_success( + mock_convert, mock_yhub, settings +): """ Authenticated users should be able to create documents by uploading a Markdown file. """ @@ -146,7 +156,7 @@ def test_api_documents_create_with_markdown_file_success(mock_convert, settings) settings.CONVERSION_UPLOAD_ENABLED = True # Mock the conversion - converted_yjs = "base64encodedyjscontent" + converted_yjs = b"\x01\x02raw yjs update" mock_convert.return_value = converted_yjs # Create a fake Markdown file @@ -166,9 +176,12 @@ def test_api_documents_create_with_markdown_file_success(mock_convert, settings) assert response.status_code == 201 document = Document.objects.get() assert document.title == "readme.md" - assert document.content == converted_yjs + # the content is saved by the collaboration server, not by Django + assert document.content is None assert document.accesses.filter(role="owner", user=user).exists() + mock_yhub.return_value.create_ydoc.assert_called_once_with(document, converted_yjs) + # Verify the converter was called correctly mock_convert.assert_called_once_with( file_content, @@ -204,7 +217,7 @@ def test_api_documents_create_with_file_and_explicit_title(mock_convert, setting settings.CONVERSION_UPLOAD_ENABLED = True # Mock the conversion - converted_yjs = "base64encodedyjscontent" + converted_yjs = b"\x01\x02raw yjs update" mock_convert.return_value = converted_yjs # Create a fake DOCX file @@ -212,7 +225,10 @@ def test_api_documents_create_with_file_and_explicit_title(mock_convert, setting file = BytesIO(file_content) file.name = "Uploaded Document.docx" - with patch("core.api.viewsets.posthog_capture") as mock_capture: + with ( + patch("core.api.viewsets.posthog_capture") as mock_capture, + patch("core.api.viewsets.YHubService"), + ): response = client.post( "/api/v1.0/documents/", { @@ -412,12 +428,13 @@ def test_api_documents_create_with_file_null_value(mock_convert, settings): ) +@patch("core.api.viewsets.YHubService") @patch("core.services.converter_services.Converter.convert") def test_api_documents_create_with_file_preserves_content_format( - mock_convert, settings + mock_convert, mock_yhub, settings ): """ - Verify that the converted content is stored correctly in the document. + Verify that the converted content reaches the collaboration server as it is. """ user = factories.UserFactory() client = APIClient() @@ -425,8 +442,8 @@ def test_api_documents_create_with_file_preserves_content_format( settings.CONVERSION_UPLOAD_ENABLED = True - # Mock the conversion with realistic base64-encoded YJS data - converted_yjs = "AQMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICA=" + # Mock the conversion with a raw Yjs update, not encodable as text + converted_yjs = b"\x01\x03\x04\x05\x06\x07" mock_convert.return_value = converted_yjs # Create a fake DOCX file @@ -446,8 +463,9 @@ def test_api_documents_create_with_file_preserves_content_format( assert response.status_code == 201 document = Document.objects.get() - # Verify the content is stored as returned by the converter - assert document.content == converted_yjs + # The update is sent untouched, it is not base64 encoded on the way + mock_yhub.return_value.create_ydoc.assert_called_once_with(document, converted_yjs) + assert document.content is None # The successful conversion should be tracked in PostHog mock_capture.assert_any_call( @@ -464,12 +482,6 @@ def test_api_documents_create_with_file_preserves_content_format( assert mock_capture.call_count == 2 - # Verify it's valid base64 (can be decoded) - try: - b64decode(converted_yjs) - except binascii.Error: - pytest.fail("Content should be valid base64-encoded data") - @patch("core.services.converter_services.Converter.convert") def test_api_documents_create_with_file_unicode_filename(mock_convert, settings): @@ -483,7 +495,7 @@ def test_api_documents_create_with_file_unicode_filename(mock_convert, settings) settings.CONVERSION_UPLOAD_ENABLED = True # Mock the conversion - converted_yjs = "base64encodedyjscontent" + converted_yjs = b"\x01\x02raw yjs update" mock_convert.return_value = converted_yjs # Create a file with Unicode characters in the name @@ -491,7 +503,10 @@ def test_api_documents_create_with_file_unicode_filename(mock_convert, settings) file = BytesIO(file_content) file.name = "文档-télécharger-документ.docx" - with patch("core.api.viewsets.posthog_capture") as mock_capture: + with ( + patch("core.api.viewsets.posthog_capture") as mock_capture, + patch("core.api.viewsets.YHubService"), + ): response = client.post( "/api/v1.0/documents/", { @@ -580,3 +595,39 @@ def test_api_documents_create_with_file_extension_not_allowed(settings): } mock_capture.assert_not_called() + + +@patch("core.api.viewsets.YHubService") +@patch("core.services.converter_services.Converter.convert") +def test_api_documents_create_with_file_collaboration_server_unavailable( + mock_convert, mock_yhub, settings +): + """ + A document whose content could not be saved by the collaboration server + should not be created at all, the uploaded file would be lost. + """ + user = factories.UserFactory() + client = APIClient() + client.force_login(user) + + settings.CONVERSION_UPLOAD_ENABLED = True + + mock_convert.return_value = b"\x01\x02raw yjs update" + mock_yhub.return_value.create_ydoc.side_effect = YHubServiceUnavailableError( + "Failed to connect to the yhub service" + ) + + file = BytesIO(b"fake docx content") + file.name = "document.docx" + + response = client.post( + "/api/v1.0/documents/", + { + "file": file, + }, + format="multipart", + ) + + assert response.status_code == 400 + assert response.json() == {"file": ["Could not save the imported file content"]} + assert not Document.objects.exists() diff --git a/src/backend/core/tests/external_api/test_external_api_documents.py b/src/backend/core/tests/external_api/test_external_api_documents.py index 66e0bbe7fc..e03a9286a1 100644 --- a/src/backend/core/tests/external_api/test_external_api_documents.py +++ b/src/backend/core/tests/external_api/test_external_api_documents.py @@ -276,7 +276,7 @@ def test_external_api_documents_create_with_markdown_file_success( settings.CONVERSION_UPLOAD_ENABLED = True # Mock the conversion - converted_yjs = "base64encodedyjscontent" + converted_yjs = b"\x01\x02raw yjs update" mock_convert.return_value = converted_yjs # Create a fake Markdown file @@ -284,7 +284,10 @@ def test_external_api_documents_create_with_markdown_file_success( file = BytesIO(file_content) file.name = "readme.md" - with patch("core.api.viewsets.posthog_capture") as mock_capture: + with ( + patch("core.api.viewsets.posthog_capture") as mock_capture, + patch("core.api.viewsets.YHubService") as mock_yhub, + ): response = client.post( "/external_api/v1.0/documents/", { @@ -299,9 +302,13 @@ def test_external_api_documents_create_with_markdown_file_success( document = models.Document.objects.get(id=data["id"]) assert document.title == "readme.md" - assert document.content == converted_yjs + # the content is saved by the collaboration server, not by Django + assert document.content is None assert document.accesses.filter(role="owner", user=user_specific_sub).exists() + mock_yhub.assert_called_once_with(user=user_specific_sub) + mock_yhub.return_value.create_ydoc.assert_called_once_with(document, converted_yjs) + # Verify the converter was called correctly mock_convert.assert_called_once_with( file_content, diff --git a/src/backend/core/tests/test_services_converter_services.py b/src/backend/core/tests/test_services_converter_services.py index f19a120f63..5747e3e9a3 100644 --- a/src/backend/core/tests/test_services_converter_services.py +++ b/src/backend/core/tests/test_services_converter_services.py @@ -1,6 +1,5 @@ """Test y-provider services.""" -from base64 import b64decode from unittest.mock import MagicMock, patch import jwt @@ -97,7 +96,8 @@ def test_convert_full_integration(mock_post, settings): result = converter.convert("test markdown") - assert b64decode(result) == expected_content + # the raw update is returned + assert result == expected_content mock_post.assert_called_once_with( "http://test.com/conversion-endpoint/", From c3ef559965d11a463a44dc1c2f20932419c69514 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Thu, 6 Aug 2026 10:58:12 +0200 Subject: [PATCH 34/59] =?UTF-8?q?=E2=9C=A8(collaboration)=20add=20a=20get-?= =?UTF-8?q?ydoc=20endpoint=20on=20yhub?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `GET /collaboration/get-ydoc/v1/docs/{id}` answers the current Yjs state of a document as a raw binary update, the read counterpart of create-ydoc, and 204 when the document has no content yet --- CHANGELOG.md | 1 + src/backend/core/services/yhub_services.py | 13 ++++++++ .../core/tests/test_services_yhub_services.py | 26 ++++++++++++++++ src/yhub-server/server.js | 31 +++++++++++++++++++ 4 files changed, 71 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba74f1ce41..9e3307cb44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,7 @@ and this project adheres to `503` instead of denying access like a permission failure, so clients retry instead of giving up. The built-in endpoints can also answer JSON on `Accept: application/json` +- ✨(collaboration) add a get-ydoc endpoint on yhub - ✨(backend) call YHubService to seed initial document content - ✨(backend) reset the yhub connections of a document and its descendants when an access or the link configuration changes diff --git a/src/backend/core/services/yhub_services.py b/src/backend/core/services/yhub_services.py index ebab7cad37..c0f74c2081 100644 --- a/src/backend/core/services/yhub_services.py +++ b/src/backend/core/services/yhub_services.py @@ -190,6 +190,19 @@ def request(self, method, url, data=None, headers=None): return response + def get_ydoc(self, document): + """ + Return the current Yjs state of a document, None when it has none. + + The raw update is what `create_ydoc` takes, so the state of a document + can be copied into another one. The built-in `ydoc` endpoint is not + used, it answers the lib0 encoding of an envelope rather than the + update itself. + """ + response = self.request("get", self.build_url("get-ydoc", document)) + + return response.content or None + def create_ydoc(self, document, update): """ Seed the initial Yjs state of a document. diff --git a/src/backend/core/tests/test_services_yhub_services.py b/src/backend/core/tests/test_services_yhub_services.py index e1b966a57b..a9dfaf7c45 100644 --- a/src/backend/core/tests/test_services_yhub_services.py +++ b/src/backend/core/tests/test_services_yhub_services.py @@ -222,3 +222,29 @@ def test_reset_connections_of_a_single_user(mock_request): _args, kwargs = mock_request.call_args assert kwargs["headers"]["X-User-Id"] == str(user.pk) + + +@patch("requests.request") +def test_get_ydoc(mock_request): + """Should return the raw update the collaboration server holds.""" + mock_request.return_value.ok = True + mock_request.return_value.content = b"\x01\x02raw yjs update" + + update = YHubService().get_ydoc(DOCUMENT) + + assert update == b"\x01\x02raw yjs update" + args, _kwargs = mock_request.call_args + assert args == ( + "get", + f"http://yhub:3002/collaboration/get-ydoc/v1/docs/{DOCUMENT.id!s}", + ) + + +@patch("requests.request") +def test_get_ydoc_without_content(mock_request): + """A document the collaboration server holds no content for should return None.""" + mock_request.return_value.ok = True + # yhub answers 204 No Content, hence an empty body + mock_request.return_value.content = b"" + + assert YHubService().get_ydoc(DOCUMENT) is None diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index 029c9cb1c3..f2d3e293ad 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -340,6 +340,37 @@ const api = [ }, }, }), + // GET /collaboration/get-ydoc/v1/{org}/{docid} — the current state of a + // document as a RAW binary update (`Y.encodeStateAsUpdate` output), the read + // counterpart of create-ydoc: yhub's built-in `GET ydoc` answers a lib0-any + // encoded `{ doc, awareness }` envelope Django cannot decode. Answers 204 + // when the room holds no content. Default access purpose: guarded like the + // built-in ydoc routes (read access on the doc — the admin JWT, or a user + // session able to retrieve it). + createApiEndpoint('get-ydoc', { + get: { + handler: async (req) => { + if (req.org !== ORG) { + return jsonResponse(400, { error: 'Unknown org' }); + } + if (!UUID4.test(req.docid)) { + return jsonResponse(400, { error: 'Room name is invalid' }); + } + const { gcDoc } = await req.yhub.getDoc( + req.room, + { gc: true, nongc: false }, + { gcOnMerge: false }, + ); + // <= 3 bytes is yhub's "no effective content" convention (an empty + // update encodes to 2 bytes) — nothing to copy. `null` answers 204. + if (gcDoc == null || gcDoc.byteLength <= 3) { + return null; + } + // a Uint8Array is served as application/octet-stream, untouched + return gcDoc; + }, + }, + }), // POST /collaboration/create-ydoc/v1/{org}/{docid} — create a document's // initial Yjs state from a RAW binary update (`Y.encodeStateAsUpdate` / // pycrdt `get_update()` output) posted as application/octet-stream. Unlike From 9301d06afbd5f44dc0422fc6ab0e646e1584c706 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Thu, 6 Aug 2026 11:00:34 +0200 Subject: [PATCH 35/59] =?UTF-8?q?=E2=9C=A8(backend)=20duplicate=20a=20docu?= =?UTF-8?q?ment=20through=20the=20collaboration=20server?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit its stateis fetched from yhub and seeded into the copy instead of being copied from the content stored by Django --- CHANGELOG.md | 1 + src/backend/core/api/viewsets.py | 66 ++++++++-- .../documents/test_api_documents_duplicate.py | 121 ++++++++++++++++-- .../test_external_api_documents.py | 9 +- src/backend/core/utils/yjs.py | 22 +++- 5 files changed, 189 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e3307cb44..2c72998f32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,7 @@ and this project adheres to instead of giving up. The built-in endpoints can also answer JSON on `Accept: application/json` - ✨(collaboration) add a get-ydoc endpoint on yhub +- ✨(backend) duplicate a document through the collaboration server - ✨(backend) call YHubService to seed initial document content - ✨(backend) reset the yhub connections of a document and its descendants when an access or the link configuration changes diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index a7f3b32044..3c1038e7f9 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -79,7 +79,7 @@ from core.utils.s3_response_stream import content_stream from core.utils.treebeard import create_tree_node_with_retry from core.utils.users import users_sharing_documents_with -from core.utils.yjs import extract_attachments +from core.utils.yjs import extract_attachments, extract_attachments_from_update from ..enums import FeatureFlag, SearchType from . import permissions, serializers, utils @@ -755,6 +755,45 @@ def _create_collaboration_document(self, document, update): {"file": ["Could not save the imported file content"]} ) from err + def _get_collaboration_document(self, document): + """ + Return the content of a document, as held by the collaboration server. + + It is the source of truth for the content, the one Django may still + store is ignored. A document it holds nothing for has no content to + copy, it answers None. + """ + try: + return YHubService(user=self.request.user).get_ydoc(document) + except YHubError as err: + logger.error( + "could not fetch the content of document %s with error: %s", + document.id, + err, + ) + raise drf.exceptions.APIException( + "Failed to fetch the document content" + ) from err + + def _copy_collaboration_document(self, document, update): + """ + Seed a duplicated document with the content of the one it copies. + + The duplicate is worthless without it, so a failure is reported to the + caller rather than leaving an empty copy behind. + """ + try: + YHubService(user=self.request.user).create_ydoc(document, update) + except YHubError as err: + logger.error( + "could not copy the content into document %s with error: %s", + document.id, + err, + ) + raise drf.exceptions.APIException( + "Failed to duplicate the document content" + ) from err + def perform_create(self, serializer): """Set the current user as creator and owner of the newly created object.""" @@ -1284,11 +1323,12 @@ def duplicate(self, request, *args, **kwargs): serializer.is_valid(raise_exception=True) user = request.user - duplicated_document = self._duplicate_document( - document_to_duplicate=document_to_duplicate, - serializer=serializer, - user=user, - ) + with transaction.atomic(): + duplicated_document = self._duplicate_document( + document_to_duplicate=document_to_duplicate, + serializer=serializer, + user=user, + ) posthog_capture( PosthogEventName.DOC_DUPLICATED, @@ -1330,7 +1370,9 @@ def _duplicate_document( user_role = document_to_duplicate.get_role(user) is_owner_or_admin = user_role in models.PRIVILEGED_ROLES - base64_yjs_content = document_to_duplicate.content + # The collaboration server holds the content, the duplicate is seeded + # with it once it exists + ydoc_update = self._get_collaboration_document(document_to_duplicate) # Duplicate the document instance link_kwargs = ( @@ -1341,7 +1383,7 @@ def _duplicate_document( if with_accesses else {} ) - extracted_attachments = set(extract_attachments(document_to_duplicate.content)) + extracted_attachments = set(extract_attachments_from_update(ydoc_update)) attachments = list( extracted_attachments & set(document_to_duplicate.attachments) ) @@ -1350,7 +1392,6 @@ def _duplicate_document( if new_parent is not None: duplicated_document = new_parent.add_child( title=title, - content=base64_yjs_content, attachments=attachments, duplicated_from=document_to_duplicate, creator=user, @@ -1382,7 +1423,6 @@ def _duplicate_document( duplicated_document = models.Document.add_root( creator=user, title=title, - content=base64_yjs_content, attachments=attachments, duplicated_from=document_to_duplicate, **link_kwargs, @@ -1396,7 +1436,6 @@ def _duplicate_document( duplicated_document = document_to_duplicate.add_sibling( "last-sibling", title=title, - content=base64_yjs_content, attachments=attachments, duplicated_from=document_to_duplicate, creator=user, @@ -1433,6 +1472,11 @@ def _duplicate_document( # Bulk create all the duplicated accesses models.DocumentAccess.objects.bulk_create(accesses_to_create) + # the accesses exist by now, so the content is only served to the users + # the duplicate is meant for + if ydoc_update: + self._copy_collaboration_document(duplicated_document, ydoc_update) + if with_descendants: for child in document_to_duplicate.get_children().filter( ancestors_deleted_at__isnull=True diff --git a/src/backend/core/tests/documents/test_api_documents_duplicate.py b/src/backend/core/tests/documents/test_api_documents_duplicate.py index a9cfe14abf..95a9827792 100644 --- a/src/backend/core/tests/documents/test_api_documents_duplicate.py +++ b/src/backend/core/tests/documents/test_api_documents_duplicate.py @@ -19,9 +19,30 @@ from rest_framework.test import APIClient from core import factories, models +from core.services.yhub_services import ( + ServiceUnavailableError as YHubServiceUnavailableError, +) pytestmark = pytest.mark.django_db + +@pytest.fixture(autouse=True, name="mock_yhub") +def mock_yhub_fixture(): + """ + The content of a document is held by the collaboration server. + + It stands for a server holding the very content the factories gave the + documents, which is what an editor connected to it would have saved. + """ + + def get_ydoc(document): + return base64.b64decode(document.content) if document.content else None + + with mock.patch("core.api.viewsets.YHubService") as mock_service: + mock_service.return_value.get_ydoc.side_effect = get_ydoc + yield mock_service + + PIXEL = ( b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00" b"\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\nIDATx\x9cc\xf8\xff\xff?\x00\x05\xfe\x02\xfe" @@ -75,7 +96,7 @@ def test_api_documents_duplicate_anonymous(): @pytest.mark.parametrize("index", range(3)) -def test_api_documents_duplicate_success(index): +def test_api_documents_duplicate_success(index, mock_yhub): """ Anonymous users should be able to retrieve attachments linked to a public document. Accesses should not be duplicated if the user does not request it specifically. @@ -120,7 +141,11 @@ def test_api_documents_duplicate_success(index): duplicated_document = models.Document.objects.get(id=response.json()["id"]) assert duplicated_document.title == "Copy of document with an image" - assert duplicated_document.content == document.content + # the content is copied through the collaboration server + assert duplicated_document.content is None + mock_yhub.return_value.create_ydoc.assert_called_once_with( + duplicated_document, base64.b64decode(document.content) + ) assert duplicated_document.creator == user assert duplicated_document.link_reach == "restricted" assert duplicated_document.link_role == "reader" @@ -185,7 +210,7 @@ def test_api_documents_duplicate_success(index): @pytest.mark.parametrize("role", ["owner", "administrator"]) -def test_api_documents_duplicate_with_accesses_admin(role): +def test_api_documents_duplicate_with_accesses_admin(role, mock_yhub): """ Accesses should be duplicated if the user requests it specifically and is owner or admin. """ @@ -219,7 +244,11 @@ def test_api_documents_duplicate_with_accesses_admin(role): duplicated_document = models.Document.objects.get(id=response.json()["id"]) assert duplicated_document.title == "Copy of document with accesses" - assert duplicated_document.content == document.content + # the content is copied through the collaboration server + assert duplicated_document.content is None + mock_yhub.return_value.create_ydoc.assert_called_once_with( + duplicated_document, base64.b64decode(document.content) + ) assert duplicated_document.link_reach == document.link_reach assert duplicated_document.link_role == document.link_role assert duplicated_document.creator == user @@ -246,7 +275,7 @@ def test_api_documents_duplicate_with_accesses_admin(role): @pytest.mark.parametrize("role", ["editor", "reader"]) -def test_api_documents_duplicate_with_accesses_non_admin(role): +def test_api_documents_duplicate_with_accesses_non_admin(role, mock_yhub): """ Accesses should not be duplicated if the user requests it specifically and is not owner or admin. @@ -274,7 +303,11 @@ def test_api_documents_duplicate_with_accesses_non_admin(role): duplicated_document = models.Document.objects.get(id=response.json()["id"]) assert duplicated_document.title == "Copy of document with accesses" - assert duplicated_document.content == document.content + # the content is copied through the collaboration server + assert duplicated_document.content is None + mock_yhub.return_value.create_ydoc.assert_called_once_with( + duplicated_document, base64.b64decode(document.content) + ) assert duplicated_document.link_reach == document.link_reach assert duplicated_document.link_role == document.link_role assert duplicated_document.creator == user @@ -295,7 +328,7 @@ def test_api_documents_duplicate_with_accesses_non_admin(role): @pytest.mark.parametrize("role", ["editor", "reader"]) -def test_api_documents_duplicate_non_root_document(role): +def test_api_documents_duplicate_non_root_document(role, mock_yhub): """ Non-root documents can be duplicated but without accesses. """ @@ -322,7 +355,11 @@ def test_api_documents_duplicate_non_root_document(role): duplicated_document = models.Document.objects.get(id=response.json()["id"]) assert duplicated_document.title == "Copy of document with accesses" - assert duplicated_document.content == child.content + # the content is copied through the collaboration server + assert duplicated_document.content is None + mock_yhub.return_value.create_ydoc.assert_called_once_with( + duplicated_document, base64.b64decode(child.content) + ) assert duplicated_document.link_reach == child.link_reach assert duplicated_document.link_role == child.link_role assert duplicated_document.creator == user @@ -517,7 +554,7 @@ def test_api_documents_duplicate_with_descendants_multi_level(): # pylint: disable=too-many-locals -def test_api_documents_duplicate_with_descendants_and_attachments(): +def test_api_documents_duplicate_with_descendants_and_attachments(mock_yhub): """ Duplicating with descendants should properly handle attachments in all children. """ @@ -590,14 +627,20 @@ def test_api_documents_duplicate_with_descendants_and_attachments(): # Check root attachments assert duplicated_root.attachments == [image_key_root] - assert duplicated_root.content == root_content # Check child attachments dup_children = duplicated_root.get_children() assert dup_children.count() == 1 dup_child = dup_children.first() assert dup_child.attachments == [image_key_child] - assert dup_child.content == child_content + + # the content of the whole subtree is copied through the collaboration server + assert duplicated_root.content is None + assert dup_child.content is None + assert mock_yhub.return_value.create_ydoc.call_args_list == [ + mock.call(duplicated_root, base64.b64decode(root_content)), + mock.call(dup_child, base64.b64decode(child_content)), + ] def test_api_documents_duplicate_with_descendants_and_accesses(): @@ -862,3 +905,59 @@ def test_api_documents_duplicate_with_descendants_complex_tree(): dup_grandchildren2 = dup_child2.get_children() assert dup_grandchildren2.count() == 1 assert dup_grandchildren2.first().title == "Copy of GrandChild 3" + + +def test_api_documents_duplicate_content_from_collaboration_server(mock_yhub): + """ + The content held by the collaboration server is the one duplicated, the + content Django may still store for the document is ignored. + """ + user = factories.UserFactory() + client = APIClient() + client.force_login(user) + + image_key, image_url = get_image_refs(uuid.uuid4()) + + # what the collaboration server holds, an image Django never saw + ydoc = pycrdt.Doc() + ydoc["document-store"] = pycrdt.XmlFragment( + [pycrdt.XmlElement("img", {"src": image_url})] + ) + edited_update = ydoc.get_update() + mock_yhub.return_value.get_ydoc.side_effect = None + mock_yhub.return_value.get_ydoc.return_value = edited_update + + document = factories.DocumentFactory( + users=[(user, "owner")], + title="an edited document", + attachments=[image_key], + ) + + response = client.post(f"/api/v1.0/documents/{document.id!s}/duplicate/") + + assert response.status_code == 201 + duplicated_document = models.Document.objects.get(id=response.json()["id"]) + + mock_yhub.return_value.get_ydoc.assert_called_once_with(document) + mock_yhub.return_value.create_ydoc.assert_called_once_with( + duplicated_document, edited_update + ) + # the attachments are the ones of the duplicated state, not of Django's + assert duplicated_document.attachments == [image_key] + + +def test_api_documents_duplicate_collaboration_server_unavailable(mock_yhub): + """A document whose content cannot be copied should not be duplicated.""" + user = factories.UserFactory() + client = APIClient() + client.force_login(user) + + document = factories.DocumentFactory(users=[(user, "owner")], title="my document") + mock_yhub.return_value.create_ydoc.side_effect = YHubServiceUnavailableError( + "Failed to connect to the yhub service" + ) + + response = client.post(f"/api/v1.0/documents/{document.id!s}/duplicate/") + + assert response.status_code == 500 + assert models.Document.objects.count() == 1 diff --git a/src/backend/core/tests/external_api/test_external_api_documents.py b/src/backend/core/tests/external_api/test_external_api_documents.py index e03a9286a1..b4bb8f1eb7 100644 --- a/src/backend/core/tests/external_api/test_external_api_documents.py +++ b/src/backend/core/tests/external_api/test_external_api_documents.py @@ -429,9 +429,12 @@ def test_external_api_documents_duplicate_allowed( role=models.RoleChoices.OWNER, ) - response = client.post( - f"/external_api/v1.0/documents/{document.id!s}/duplicate/", - ) + with patch("core.api.viewsets.YHubService") as mock_yhub: + # the collaboration server holds no content for this document + mock_yhub.return_value.get_ydoc.return_value = None + response = client.post( + f"/external_api/v1.0/documents/{document.id!s}/duplicate/", + ) assert response.status_code == 201 diff --git a/src/backend/core/utils/yjs.py b/src/backend/core/utils/yjs.py index a5f4c8b2e5..7856d0636c 100644 --- a/src/backend/core/utils/yjs.py +++ b/src/backend/core/utils/yjs.py @@ -9,16 +9,20 @@ from core import enums -def base64_yjs_to_xml(base64_string): - """Extract xml from base64 yjs document.""" - - decoded_bytes = base64.b64decode(base64_string) +def yjs_to_xml(update): + """Extract xml from a raw yjs update.""" doc = pycrdt.Doc() - doc.apply_update(decoded_bytes) + doc.apply_update(update) return str(doc.get("document-store", type=pycrdt.XmlFragment)) +def base64_yjs_to_xml(base64_string): + """Extract xml from base64 yjs document.""" + + return yjs_to_xml(base64.b64decode(base64_string)) + + def base64_yjs_to_text(base64_string): """Extract text from base64 yjs document.""" @@ -34,3 +38,11 @@ def extract_attachments(content): xml_content = base64_yjs_to_xml(content) return re.findall(enums.MEDIA_STORAGE_URL_EXTRACT, xml_content) + + +def extract_attachments_from_update(update): + """Helper method to extract media paths from a raw yjs update.""" + if not update: + return [] + + return re.findall(enums.MEDIA_STORAGE_URL_EXTRACT, yjs_to_xml(update)) From 0ffb688af06b69c25c5edeb5aed1bb77c655df6c Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Thu, 6 Aug 2026 16:49:44 +0200 Subject: [PATCH 36/59] =?UTF-8?q?=F0=9F=92=A5(backend)=20remove=20the=20`d?= =?UTF-8?q?ocuments/{id}/content/`=20endpoint?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit , both its PATCH and its GET: the content of a document is saved and served by the collaboration server. The `content_patch` and `content_retrieve` abilities go with it. --- CHANGELOG.md | 1 + UPGRADE.md | 9 + src/backend/core/api/permissions.py | 1 - src/backend/core/api/serializers.py | 29 - src/backend/core/api/utils.py | 34 -- src/backend/core/api/viewsets.py | 158 +----- src/backend/core/models.py | 2 - .../test_api_documents_content_retrieve.py | 506 ------------------ .../test_api_documents_content_update.py | 271 ---------- .../documents/test_api_documents_retrieve.py | 10 - .../documents/test_api_documents_trashbin.py | 4 - ...pi_documents_update_extract_attachments.py | 154 ------ ...pi_utils_parse_http_conditional_headers.py | 52 -- .../core/tests/test_models_documents.py | 22 - .../tests/test_utils_s3_response_stream.py | 125 ----- src/backend/core/utils/s3_response_stream.py | 47 -- src/backend/impress/settings.py | 4 - 17 files changed, 12 insertions(+), 1417 deletions(-) delete mode 100644 src/backend/core/tests/documents/test_api_documents_content_retrieve.py delete mode 100644 src/backend/core/tests/documents/test_api_documents_content_update.py delete mode 100644 src/backend/core/tests/documents/test_api_documents_update_extract_attachments.py delete mode 100644 src/backend/core/tests/test_api_utils_parse_http_conditional_headers.py delete mode 100644 src/backend/core/tests/test_utils_s3_response_stream.py delete mode 100644 src/backend/core/utils/s3_response_stream.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c72998f32..baf35ea699 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -122,6 +122,7 @@ and this project adheres to pending). The get-connections API is dropped for good: its only consumer was the removed can-edit mechanism, so it is not needed anymore - 🔥(backend) remove the unused `CollaborationService` +- 💥(backend) remove the `documents/{id}/content/` endpoint - 💥(backend) remove the `documents/{id}/can-edit/` endpoint - 💥(y-provider) the published `lasuite/impress-y-provider` image becomes converter-only and no longer serves `/collaboration/ws/`; deployments using diff --git a/UPGRADE.md b/UPGRADE.md index feafc80ff9..06629eb5de 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -16,6 +16,15 @@ the following command inside your docker container: ## [Unreleased] +- The endpoint `/api/v1.0/documents/{document_id}/content/`, added in 5.0.0, is + removed, both its `GET` and its `PATCH`. The content of a document is now + saved and served by the collaboration server, the editor exchanging it over + the websocket, so nothing reads or writes it through the API anymore. If you + integrate with Docs, stop calling this endpoint: the `content_patch` and + `content_retrieve` abilities disappear from the document payload along with + it. `/api/v1.0/documents/{document_id}/formatted-content/` is not affected. + The `CONTENT_METADATA_CACHE_TIMEOUT` setting only tuned the cache of the + removed `GET` and is no longer read, you can drop it from your configuration. - The JWKS of the resource server moved from `/api/{version}/jwks` to `/external_api/{version}/jwks`, alongside the rest of the resource server endpoints. `/api/{version}/jwks` now publishes the public key validating the diff --git a/src/backend/core/api/permissions.py b/src/backend/core/api/permissions.py index 4b92b711e6..affd9b0707 100644 --- a/src/backend/core/api/permissions.py +++ b/src/backend/core/api/permissions.py @@ -12,7 +12,6 @@ ACTION_FOR_METHOD_TO_PERMISSION = { "versions_detail": {"DELETE": "versions_destroy", "GET": "versions_retrieve"}, "children": {"GET": "children_list", "POST": "children_create"}, - "content": {"PATCH": "content_patch", "GET": "content_retrieve"}, } diff --git a/src/backend/core/api/serializers.py b/src/backend/core/api/serializers.py index 5bf57ebd86..eb8f27261a 100644 --- a/src/backend/core/api/serializers.py +++ b/src/backend/core/api/serializers.py @@ -1,9 +1,7 @@ """Client serializers for the impress core app.""" # pylint: disable=too-many-lines -import binascii import mimetypes -from base64 import b64decode from os.path import splitext from django.conf import settings @@ -308,33 +306,6 @@ class Meta: read_only_fields = ListDocumentSerializer.Meta.read_only_fields + ["parent"] -class DocumentContentSerializer(serializers.Serializer): - """Serializer for updating only the raw content of a document stored in S3.""" - - content = serializers.CharField(required=True) - - def validate_content(self, value): - """Validate the content field.""" - try: - b64decode(value, validate=True) - except binascii.Error as err: - raise serializers.ValidationError("Invalid base64 content.") from err - - return value - - def update(self, instance, validated_data): - """ - This serializer does not support updates. - """ - raise NotImplementedError("Update is not supported for this serializer.") - - def create(self, validated_data): - """ - This serializer does not support create. - """ - raise NotImplementedError("Create is not supported for this serializer.") - - class DocumentAccessSerializer(serializers.ModelSerializer): """Serialize document accesses.""" diff --git a/src/backend/core/api/utils.py b/src/backend/core/api/utils.py index 19cb03f3eb..3a78bccbb8 100644 --- a/src/backend/core/api/utils.py +++ b/src/backend/core/api/utils.py @@ -1,6 +1,5 @@ """Util to generate S3 authorization headers for object storage access control""" -import datetime as dt import time from abc import ABC, abstractmethod @@ -195,36 +194,3 @@ def get_ident(self, request): if x_forwarded_for else request.META.get("REMOTE_ADDR") ) - - -def get_content_metadata_cache_key(document_id): - """Return the cache key used to store content metadata.""" - return f"docs:content-metadata:{document_id!s}" - - -def parse_http_conditional_headers(request): - """Extract and normalize `If-None-Match` and `If-Modified-Since`. - - The `W/` weak prefix is stripped from the ETag because reverse proxies - (e.g. nginx with gzip) rewrite strong ETags into weak ones, which would - otherwise break a strict equality check in production. - """ - if_none_match = request.META.get("HTTP_IF_NONE_MATCH") - if if_none_match and if_none_match.startswith("W/"): - if_none_match = if_none_match.removeprefix("W/") - - if_modified_since_dt = None - if not (if_modified_since := request.META.get("HTTP_IF_MODIFIED_SINCE")): - return if_none_match, if_modified_since_dt - - try: - if_modified_since_dt = dt.datetime.strptime( - if_modified_since, "%a, %d %b %Y %H:%M:%S %Z" - ) - except ValueError: - if_modified_since_dt = None - else: - if not if_modified_since_dt.tzinfo: - if_modified_since_dt = if_modified_since_dt.replace(tzinfo=dt.timezone.utc) - - return if_none_match, if_modified_since_dt diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index 3c1038e7f9..84b614ff94 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -3,14 +3,12 @@ # pylint: disable=too-many-lines import base64 -import datetime as dt import ipaddress import json import logging import socket import uuid from collections import defaultdict -from io import BytesIO from urllib.parse import unquote, urlencode, urlparse from django.conf import settings @@ -20,7 +18,7 @@ from django.core.exceptions import ValidationError from django.core.files.storage import default_storage from django.core.validators import URLValidator -from django.db import DatabaseError, connection, transaction +from django.db import DatabaseError, transaction from django.db import models as db from django.db.models.expressions import RawSQL from django.db.models.functions import Greatest, Left, Length @@ -36,7 +34,6 @@ import rest_framework as drf import waffle from botocore.exceptions import ClientError -from botocore.response import StreamingBody from csp.constants import NONE from csp.decorators import csp_update from lasuite.malware_detection import malware_detection @@ -76,10 +73,9 @@ from core.tasks.mail import send_ask_for_access_mail from core.utils.analytics import PosthogEventName, posthog_capture from core.utils.paths import filter_descendants -from core.utils.s3_response_stream import content_stream from core.utils.treebeard import create_tree_node_with_retry from core.utils.users import users_sharing_documents_with -from core.utils.yjs import extract_attachments, extract_attachments_from_update +from core.utils.yjs import extract_attachments_from_update from ..enums import FeatureFlag, SearchType from . import permissions, serializers, utils @@ -2053,156 +2049,6 @@ def media_auth(self, request, *args, **kwargs): return drf.response.Response("authorized", headers=request.headers, status=200) - @drf.decorators.action(detail=True, methods=["patch"]) - def content(self, request, *args, **kwargs): - """Update the raw Yjs content of a document stored in S3.""" - document = self.get_object() - - serializer = serializers.DocumentContentSerializer(data=request.data) - serializer.is_valid(raise_exception=True) - - content = serializer.validated_data["content"] - try: - extracted_attachments = set(extract_attachments(content)) - except ValueError: - return drf_response.Response( - "invalid yjs document", status=status.HTTP_400_BAD_REQUEST - ) - - existing_attachments = set(document.attachments or []) - new_attachments = extracted_attachments - existing_attachments - - # Ensure we update attachments the request user is allowed to read - if new_attachments: - attachments_documents = ( - models.Document.objects.filter( - attachments__overlap=list(new_attachments) - ) - .only("path", "attachments") - .order_by("path") - ) - - user = self.request.user - readable_per_se_paths = ( - models.Document.objects.readable_per_se(user) - .order_by("path") - .values_list("path", flat=True) - ) - readable_attachments_paths = filter_descendants( - [doc.path for doc in attachments_documents], - readable_per_se_paths, - skip_sorting=True, - ) - - readable_attachments = set() - for attachments_document in attachments_documents: - if attachments_document.path not in readable_attachments_paths: - continue - readable_attachments.update( - set(attachments_document.attachments) & new_attachments - ) - - # Update attachments with readable keys - document.attachments = list(existing_attachments | readable_attachments) - document.content = content - document.save() - cache.delete(utils.get_content_metadata_cache_key(document.id)) - - return drf_response.Response(status=status.HTTP_204_NO_CONTENT) - - @content.mapping.get - def content_retrieve(self, request, *args, **kwargs): - """ - Retrieve the raw content file from s3 and stream it. - - We implement a HTTP cache based on the ETag and LastModified headers. - The ETag and LastModified are retrieved in the S3 get_object operation to be consistent with - the content Body retrieved at the same time. These metadata are saved in cache for - future requests. - We check in the request if the ETag is present in the If-None-Match header and if it's the - same as the one from the S3 get_object, we return a 304 response. - If the ETag is not present or not the same, we do the same check based on the LastModified - value if present in the If-Modified-Since header. - """ - document = self.get_object() - # The S3 call to fetch the document can take time and the database - # connection is useless in this process. Hence we are closing it now - # to prevent having a massive number of database connections during - # the web-socket re-connection burst. - connection.close() - - if_none_match, if_modified_since_dt = utils.parse_http_conditional_headers( - request - ) - - # First check if a cache is existing to return earlier a 304 without reaching s3 - # if etag or last_modified have not changed. - cache_key = utils.get_content_metadata_cache_key(document.id) - if content_metadata := cache.get(cache_key): - if (if_none_match and if_none_match == content_metadata.get("etag")) or ( - if_modified_since_dt - and dt.datetime.fromisoformat(content_metadata.get("last_modified")) - <= if_modified_since_dt - ): - return drf_response.Response(status=status.HTTP_304_NOT_MODIFIED) - - # Prepare get_object S3 operation. The get_object manages ETag and last_modified - # headers will raise a 304 client error if one of them matches the value existing in - # S3. - get_object_kwargs = { - "Bucket": default_storage.bucket_name, - "Key": document.file_key, - } - if if_none_match: - get_object_kwargs["IfNoneMatch"] = if_none_match - if if_modified_since_dt: - get_object_kwargs["IfModifiedSince"] = if_modified_since_dt - - try: - s3_response = default_storage.connection.meta.client.get_object( - **get_object_kwargs - ) - except ClientError as exc: - code = exc.response["Error"]["Code"] - match code: - case "304" | "PreconditionFailed" | "NotModified": - return drf_response.Response(status=status.HTTP_304_NOT_MODIFIED) - case "NoSuchKey" | "404": - return StreamingHttpResponse( - content_stream(StreamingBody(BytesIO(b""), content_length=0)), - content_type="text/plain", - status=200, - ) - case _: - raise - - last_modified = s3_response["LastModified"] - etag = s3_response["ETag"] - size = s3_response["ContentLength"] - - # Refresh the metadata cache - cache.set( - cache_key, - { - "last_modified": last_modified.isoformat(), - "etag": etag, - }, - settings.CONTENT_METADATA_CACHE_TIMEOUT, - ) - - response = StreamingHttpResponse( - streaming_content=content_stream(s3_response["Body"]), - content_type="text/plain", - status=status.HTTP_200_OK, - ) - - response["Content-Length"] = size - response["ETag"] = etag - response["Last-Modified"] = last_modified.strftime("%a, %d %b %Y %H:%M:%S %Z") - response["Cache-Control"] = "private, no-cache" - - return response - @drf.decorators.action(detail=True, methods=["get"], url_path="media-check") def media_check(self, request, *args, **kwargs): """ diff --git a/src/backend/core/models.py b/src/backend/core/models.py index e112ab27f0..299cd2437d 100644 --- a/src/backend/core/models.py +++ b/src/backend/core/models.py @@ -1391,8 +1391,6 @@ def get_abilities(self, user): # pylint: disable=too-many-locals "collaboration_auth": can_get, "comment": can_comment, "formatted_content": can_get, - "content_patch": can_update, - "content_retrieve": retrieve, "cors_proxy": can_get, "descendants": can_get, "destroy": can_destroy, diff --git a/src/backend/core/tests/documents/test_api_documents_content_retrieve.py b/src/backend/core/tests/documents/test_api_documents_content_retrieve.py deleted file mode 100644 index 3d1ce1c3bd..0000000000 --- a/src/backend/core/tests/documents/test_api_documents_content_retrieve.py +++ /dev/null @@ -1,506 +0,0 @@ -""" -Tests for the GET /api/v1.0/documents/{id}/content/ endpoint. -""" - -from datetime import timedelta -from uuid import uuid4 - -from django.core.cache import cache -from django.core.files.storage import default_storage -from django.utils import timezone - -import pytest -from asgiref.sync import sync_to_async -from rest_framework import status -from rest_framework.test import APIClient - -from core import factories -from core.api.utils import get_content_metadata_cache_key -from core.tests.conftest import TEAM, USER, VIA - -pytestmark = pytest.mark.django_db - - -@pytest.mark.parametrize("reach", ["authenticated", "restricted"]) -def test_api_documents_content_retrieve_anonymous_non_public(reach): - """Anonymous users cannot retrieve content of non-public documents.""" - document = factories.DocumentFactory(link_reach=reach) - - response = APIClient().get(f"/api/v1.0/documents/{document.id!s}/content/") - - assert response.status_code == status.HTTP_401_UNAUTHORIZED - - -def test_api_documents_content_retrieve_anonymous_public(): - """Anonymous users can retrieve content of a public document.""" - document = factories.DocumentFactory(link_reach="public") - - assert not cache.get(get_content_metadata_cache_key(document.id)) - - response = APIClient().get(f"/api/v1.0/documents/{document.id!s}/content/") - - assert response.status_code == status.HTTP_200_OK - assert response["Content-Type"] == "text/plain" - assert b"".join( - response.streaming_content - ) == factories.YDOC_HELLO_WORLD_BASE64.encode("utf-8") - assert response["Content-Length"] is not None - assert response["ETag"] is not None - assert response["Last-Modified"] is not None - assert response["Cache-Control"] == "private, no-cache" - - assert cache.get(get_content_metadata_cache_key(document.id)) - - -def test_api_documents_content_retrieve_authenticated_no_access(): - """Authenticated users without access cannot retrieve content of a restricted document.""" - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - - client = APIClient() - client.force_login(user) - - response = client.get(f"/api/v1.0/documents/{document.id!s}/content/") - - assert response.status_code == status.HTTP_403_FORBIDDEN - - -@pytest.mark.parametrize("link_reach", ["authenticated", "public"]) -def test_api_documents_content_retrieve_authenticated_not_restricted(link_reach): - """ - Authenticated users can retrieve content of a public document - without any explicit access grant. - """ - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach=link_reach) - - client = APIClient() - client.force_login(user) - - assert not cache.get(get_content_metadata_cache_key(document.id)) - - response = client.get(f"/api/v1.0/documents/{document.id!s}/content/") - - assert response.status_code == status.HTTP_200_OK - assert b"".join( - response.streaming_content - ) == factories.YDOC_HELLO_WORLD_BASE64.encode("utf-8") - assert response["Content-Length"] is not None - assert response["ETag"] is not None - assert response["Last-Modified"] is not None - assert response["Cache-Control"] == "private, no-cache" - - assert cache.get(get_content_metadata_cache_key(document.id)) - - -@pytest.mark.parametrize("via", VIA) -@pytest.mark.parametrize( - "role", ["reader", "commenter", "editor", "administrator", "owner"] -) -def test_api_documents_content_retrieve_success(role, via, mock_user_teams): - """Users with any role can retrieve document content, directly or via a team.""" - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - - if via == USER: - factories.UserDocumentAccessFactory(document=document, user=user, role=role) - elif via == TEAM: - mock_user_teams.return_value = ["lasuite"] - factories.TeamDocumentAccessFactory( - document=document, team="lasuite", role=role - ) - - client = APIClient() - client.force_login(user) - - assert not cache.get(get_content_metadata_cache_key(document.id)) - - response = client.get(f"/api/v1.0/documents/{document.id!s}/content/") - - assert response.status_code == status.HTTP_200_OK - assert b"".join( - response.streaming_content - ) == factories.YDOC_HELLO_WORLD_BASE64.encode("utf-8") - assert response["Content-Length"] is not None - assert response["ETag"] is not None - assert response["Last-Modified"] is not None - assert response["Cache-Control"] == "private, no-cache" - - assert cache.get(get_content_metadata_cache_key(document.id)) - - -def test_api_documents_content_retrieve_nonexistent_document(): - """Retrieving content of a non-existent document returns 404.""" - user = factories.UserFactory() - client = APIClient() - client.force_login(user) - - response = client.get(f"/api/v1.0/documents/{uuid4()!s}/content/") - - assert response.status_code == status.HTTP_404_NOT_FOUND - - -def test_api_documents_content_retrieve_file_not_in_storage(): - """Returns an empty string when the file does not exist on the storage.""" - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - factories.UserDocumentAccessFactory(document=document, user=user, role="reader") - - client = APIClient() - client.force_login(user) - - default_storage.delete(document.file_key) - - assert not default_storage.exists(document.file_key) - - response = client.get(f"/api/v1.0/documents/{document.id!s}/content/") - - assert response.status_code == status.HTTP_200_OK - assert b"".join(response.streaming_content) == b"" - assert not response.get("Content-Length") - assert not response.get("ETag") - assert not response.get("Last-Modified") - assert not response.get("Cache-Control") - - assert not cache.get(get_content_metadata_cache_key(document.id)) - - -# The data created in this test through `sync_to_async` is written on a -# separate thread-local database connection, outside the atomic transaction -# pytest-django uses to isolate tests. `transaction=True` makes pytest-django -# flush the tables after the test instead of relying on a rollback, so the row -# does not leak into the rest of the suite. -@pytest.mark.django_db(transaction=True) -@pytest.mark.asyncio(loop_scope="function") -async def test_api_documents_content_retrieve_async(monkeypatch): - """ - Test the content retrieve method in async should use the async generator in the streaming - response. - """ - monkeypatch.setenv("PYTHON_SERVER_MODE", "async") - - document = await sync_to_async(factories.DocumentFactory)(link_reach="public") - client = APIClient() - - response = await sync_to_async(client.get)( - f"/api/v1.0/documents/{document.id!s}/content/" - ) - - assert response.status_code == status.HTTP_200_OK - # Wait for the streaming content to be fully received => async iterator -> list - # This fails if the streaming is not an async generator - response_content = b"".join( - [content async for content in response.streaming_content] - ).decode("utf-8") - assert response_content == factories.YDOC_HELLO_WORLD_BASE64 - - -@pytest.mark.django_db(transaction=True) -@pytest.mark.asyncio(loop_scope="function") -async def test_api_documents_content_retrieve_file_not_in_storage_async(monkeypatch): - """Returns an empty string when the file does not exist on the storage.""" - monkeypatch.setenv("PYTHON_SERVER_MODE", "async") - - user = await sync_to_async(factories.UserFactory)() - document = await sync_to_async(factories.DocumentFactory)(link_reach="restricted") - await sync_to_async(factories.UserDocumentAccessFactory)( - document=document, user=user, role="reader" - ) - - client = APIClient() - await client.aforce_login(user) - - await sync_to_async(default_storage.delete)(document.file_key) - - assert not await sync_to_async(default_storage.exists)(document.file_key) - - response = await sync_to_async(client.get)( - f"/api/v1.0/documents/{document.id!s}/content/" - ) - - assert response.status_code == status.HTTP_200_OK - # Wait for the streaming content to be fully received => async iterator -> list - # This fails if the streaming is not an async generator - assert b"".join([content async for content in response.streaming_content]) == b"" - assert not response.get("Content-Length") - assert not response.get("ETag") - assert not response.get("Last-Modified") - assert not response.get("Cache-Control") - - assert not await cache.aget(get_content_metadata_cache_key(document.id)) - - -def test_api_documents_content_retrieve_content_length_header(): - """The response includes the Content-Length header when available from storage.""" - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - factories.UserDocumentAccessFactory(document=document, user=user, role="reader") - - client = APIClient() - client.force_login(user) - - response = client.get(f"/api/v1.0/documents/{document.id!s}/content/") - - assert response.status_code == status.HTTP_200_OK - expected_size = default_storage.size(document.file_key) - assert int(response["Content-Length"]) == expected_size - - -@pytest.mark.parametrize("role", ["reader", "commenter", "editor", "administrator"]) -def test_api_documents_content_retrieve_deleted_document_for_non_owners_all_roles(role): - """ - Retrieving content of a soft-deleted document returns 404 for any non-owner role. - """ - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - factories.UserDocumentAccessFactory(document=document, user=user, role=role) - - document.soft_delete() - document.refresh_from_db() - - client = APIClient() - client.force_login(user) - - response = client.get(f"/api/v1.0/documents/{document.id!s}/content/") - - assert response.status_code == status.HTTP_404_NOT_FOUND - - -def test_api_documents_content_retrieve_deleted_document_for_owner(): - """ - Owners can still retrieve content of a soft-deleted document. - - The 'retrieve' ability is True for owners regardless of deletion state. - """ - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - factories.UserDocumentAccessFactory(document=document, user=user, role="owner") - - document.soft_delete() - document.refresh_from_db() - - client = APIClient() - client.force_login(user) - - assert not cache.get(get_content_metadata_cache_key(document.id)) - - response = client.get(f"/api/v1.0/documents/{document.id!s}/content/") - - assert response.status_code == status.HTTP_200_OK - assert b"".join( - response.streaming_content - ) == factories.YDOC_HELLO_WORLD_BASE64.encode("utf-8") - assert response["Content-Length"] is not None - assert response["ETag"] is not None - assert response["Last-Modified"] is not None - assert response["Cache-Control"] == "private, no-cache" - - assert cache.get(get_content_metadata_cache_key(document.id)) - - -def test_api_documents_content_retrieve_reusing_etag(): - """Fetching content reusing a valid ETag header should return a 304.""" - - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - factories.UserDocumentAccessFactory(document=document, user=user, role="owner") - - client = APIClient() - client.force_login(user) - - file_metadata = default_storage.connection.meta.client.head_object( - Bucket=default_storage.bucket_name, Key=document.file_key - ) - last_modified = file_metadata["LastModified"] - etag = file_metadata["ETag"] - size = file_metadata["ContentLength"] - - cache.set( - get_content_metadata_cache_key(document.id), - { - "last_modified": last_modified.isoformat(), - "etag": etag, - "size": size, - }, - ) - - response = client.get( - f"/api/v1.0/documents/{document.id!s}/content/", - headers={"If-None-Match": etag}, - ) - - assert response.status_code == status.HTTP_304_NOT_MODIFIED - - -def test_api_documents_content_retrieve_reusing_invalid_etag(): - """Fetching content using an invalid ETag header should return a 200.""" - - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - factories.UserDocumentAccessFactory(document=document, user=user, role="owner") - - client = APIClient() - client.force_login(user) - - file_metadata = default_storage.connection.meta.client.head_object( - Bucket=default_storage.bucket_name, Key=document.file_key - ) - last_modified = file_metadata["LastModified"] - etag = file_metadata["ETag"] - size = file_metadata["ContentLength"] - - cache.set( - get_content_metadata_cache_key(document.id), - { - "last_modified": last_modified.isoformat(), - "etag": etag, - "size": size, - }, - ) - - response = client.get( - f"/api/v1.0/documents/{document.id!s}/content/", - headers={"If-None-Match": "invalid"}, - ) - - assert response.status_code == status.HTTP_200_OK - assert b"".join( - response.streaming_content - ) == factories.YDOC_HELLO_WORLD_BASE64.encode("utf-8") - assert response["Content-Length"] is not None - assert response["ETag"] is not None - assert response["Last-Modified"] is not None - assert response["Cache-Control"] == "private, no-cache" - - -def test_api_documents_content_retrieve_using_etag_without_cache(): - """ - Fetching content using a valid ETag header but without existing cache should return a 304. - """ - - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - factories.UserDocumentAccessFactory(document=document, user=user, role="owner") - - client = APIClient() - client.force_login(user) - - file_metadata = default_storage.connection.meta.client.head_object( - Bucket=default_storage.bucket_name, Key=document.file_key - ) - etag = file_metadata["ETag"] - - assert not cache.get(get_content_metadata_cache_key(document.id)) - - response = client.get( - f"/api/v1.0/documents/{document.id!s}/content/", - headers={"If-None-Match": etag}, - ) - - assert response.status_code == status.HTTP_304_NOT_MODIFIED - - -def test_api_documents_content_retrieve_reusing_last_modified_since(): - """Fetching a content using a If-Modified-Since valid should return a 304.""" - - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - factories.UserDocumentAccessFactory(document=document, user=user, role="owner") - - client = APIClient() - client.force_login(user) - - file_metadata = default_storage.connection.meta.client.head_object( - Bucket=default_storage.bucket_name, Key=document.file_key - ) - last_modified = file_metadata["LastModified"] - etag = file_metadata["ETag"] - size = file_metadata["ContentLength"] - - cache.set( - get_content_metadata_cache_key(document.id), - { - "last_modified": last_modified.isoformat(), - "etag": etag, - "size": size, - }, - ) - - response = client.get( - f"/api/v1.0/documents/{document.id!s}/content/", - headers={ - "If-Modified-Since": timezone.now().strftime("%a, %d %b %Y %H:%M:%S %Z") - }, - ) - - assert response.status_code == status.HTTP_304_NOT_MODIFIED - - -def test_api_documents_content_retrieve_using_last_modified_since_without_cache(): - """ - Fetching a content using a If-Modified-Since valid should return a 304 - even if content metadata are not present in cache. - """ - - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - factories.UserDocumentAccessFactory(document=document, user=user, role="owner") - - client = APIClient() - client.force_login(user) - - assert not cache.get(get_content_metadata_cache_key(document.id)) - - response = client.get( - f"/api/v1.0/documents/{document.id!s}/content/", - headers={ - "If-Modified-Since": timezone.now().strftime("%a, %d %b %Y %H:%M:%S %Z") - }, - ) - - assert response.status_code == status.HTTP_304_NOT_MODIFIED - - -def test_api_documents_content_retrieve_reusing_last_modified_since_invalid(): - """Fetching a content using a If-Modified-Since invalid should return a 200.""" - - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - factories.UserDocumentAccessFactory(document=document, user=user, role="owner") - - client = APIClient() - client.force_login(user) - - file_metadata = default_storage.connection.meta.client.head_object( - Bucket=default_storage.bucket_name, Key=document.file_key - ) - last_modified = file_metadata["LastModified"] - etag = file_metadata["ETag"] - size = file_metadata["ContentLength"] - - cache.set( - get_content_metadata_cache_key(document.id), - { - "last_modified": last_modified.isoformat(), - "etag": etag, - "size": size, - }, - ) - - response = client.get( - f"/api/v1.0/documents/{document.id!s}/content/", - headers={ - "If-Modified-Since": (timezone.now() - timedelta(minutes=60)).strftime( - "%a, %d %b %Y %H:%M:%S %Z" - ) - }, - ) - - assert response.status_code == status.HTTP_200_OK - assert b"".join( - response.streaming_content - ) == factories.YDOC_HELLO_WORLD_BASE64.encode("utf-8") - assert response["Content-Length"] is not None - assert response["ETag"] is not None - assert response["Last-Modified"] is not None - assert response["Cache-Control"] == "private, no-cache" diff --git a/src/backend/core/tests/documents/test_api_documents_content_update.py b/src/backend/core/tests/documents/test_api_documents_content_update.py deleted file mode 100644 index 79530d9cf0..0000000000 --- a/src/backend/core/tests/documents/test_api_documents_content_update.py +++ /dev/null @@ -1,271 +0,0 @@ -""" -Tests for the PATCH /api/v1.0/documents/{id}/content/ endpoint. -""" - -import base64 -from functools import cache -from uuid import uuid4 - -from django.core.files.storage import default_storage - -import pycrdt -import pytest -from rest_framework import status -from rest_framework.test import APIClient - -from core import factories, models -from core.tests.conftest import TEAM, USER, VIA - -pytestmark = pytest.mark.django_db - - -@cache -def get_sample_ydoc(): - """Return a ydoc from text for testing purposes.""" - ydoc = pycrdt.Doc() - ydoc["document-store"] = pycrdt.Text("Hello") - update = ydoc.get_update() - return base64.b64encode(update).decode("utf-8") - - -def get_s3_content(document): - """Read the raw content currently stored in S3 for the given document.""" - with default_storage.open(document.file_key, mode="rb") as file: - return file.read().decode() - - -def test_api_documents_content_update_anonymous(): - """Anonymous users without access cannot update document content.""" - document = factories.DocumentFactory(link_reach="restricted") - - response = APIClient().patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_sample_ydoc()}, - ) - - assert response.status_code == status.HTTP_401_UNAUTHORIZED - - -def test_api_documents_content_update_authenticated_no_access(): - """Authenticated users without access cannot update document content.""" - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - - client = APIClient() - client.force_login(user) - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_sample_ydoc()}, - ) - - assert response.status_code == status.HTTP_403_FORBIDDEN - - -@pytest.mark.parametrize("role", ["reader", "commenter"]) -def test_api_documents_content_update_read_only_role(role): - """Users with reader or commenter role cannot update document content.""" - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - factories.UserDocumentAccessFactory(document=document, user=user, role=role) - - client = APIClient() - client.force_login(user) - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_sample_ydoc()}, - ) - - assert response.status_code == status.HTTP_403_FORBIDDEN - - -@pytest.mark.parametrize("via", VIA) -@pytest.mark.parametrize("role", ["editor", "administrator", "owner"]) -def test_api_documents_content_update_success(role, via, mock_user_teams): - """Users with editor, administrator, or owner role can update document content.""" - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - - if via == USER: - factories.UserDocumentAccessFactory(document=document, user=user, role=role) - elif via == TEAM: - mock_user_teams.return_value = ["lasuite"] - factories.TeamDocumentAccessFactory( - document=document, team="lasuite", role=role - ) - - client = APIClient() - client.force_login(user) - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_sample_ydoc()}, - ) - - assert response.status_code == status.HTTP_204_NO_CONTENT - assert get_s3_content(document) == get_sample_ydoc() - - -def test_api_documents_content_update_missing_content_field(): - """A request body without the content field returns 400.""" - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - factories.UserDocumentAccessFactory(document=document, user=user, role="editor") - - client = APIClient() - client.force_login(user) - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {}, - ) - - assert response.status_code == status.HTTP_400_BAD_REQUEST - assert response.json() == { - "content": [ - "This field is required.", - ] - } - - -def test_api_documents_content_update_invalid_base64(): - """A non-base64 content value returns 400.""" - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - factories.UserDocumentAccessFactory(document=document, user=user, role="editor") - - client = APIClient() - client.force_login(user) - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": "not-valid-base64!!!"}, - ) - - assert response.status_code == status.HTTP_400_BAD_REQUEST - assert response.json() == { - "content": [ - "Invalid base64 content.", - ] - } - - -def test_api_documents_content_update_nonexistent_document(): - """Updating the content of a non-existent document returns 404.""" - user = factories.UserFactory() - client = APIClient() - client.force_login(user) - - response = client.patch( - f"/api/v1.0/documents/{uuid4()!s}/content/", - {"content": get_sample_ydoc()}, - ) - - assert response.status_code == status.HTTP_404_NOT_FOUND - - -def test_api_documents_content_update_replaces_existing(): - """Patching content replaces whatever was previously in S3.""" - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - factories.UserDocumentAccessFactory(document=document, user=user, role="editor") - - client = APIClient() - client.force_login(user) - - assert get_s3_content(document) == factories.YDOC_HELLO_WORLD_BASE64 - - new_content = get_sample_ydoc() - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": new_content}, - ) - - assert response.status_code == status.HTTP_204_NO_CONTENT - assert get_s3_content(document) == new_content - - -@pytest.mark.parametrize("role", ["editor", "administrator"]) -def test_api_documents_content_update_deleted_document_for_non_owners(role): - """Updating content on a soft-deleted document returns 404 for non-owners. - - Soft-deleted documents are excluded from the queryset for non-owners, - so the endpoint returns 404 rather than 403. - """ - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - factories.UserDocumentAccessFactory(document=document, user=user, role=role) - - document.soft_delete() - document.refresh_from_db() - - client = APIClient() - client.force_login(user) - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_sample_ydoc()}, - ) - - assert response.status_code == status.HTTP_404_NOT_FOUND - - -def test_api_documents_content_update_deleted_document_for_owners(): - """Updating content on a soft-deleted document returns 403 for owners.""" - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - factories.UserDocumentAccessFactory(document=document, user=user, role="owner") - - document.soft_delete() - document.refresh_from_db() - - client = APIClient() - client.force_login(user) - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_sample_ydoc()}, - ) - - assert response.status_code == status.HTTP_403_FORBIDDEN - - -def test_api_documents_content_update_link_editor(): - """ - A public document with link_role=editor allows any authenticated user to - update content via the link role. - """ - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="public", link_role="editor") - - client = APIClient() - client.force_login(user) - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_sample_ydoc()}, - ) - - assert response.status_code == status.HTTP_204_NO_CONTENT - assert get_s3_content(document) == get_sample_ydoc() - assert models.Document.objects.filter(id=document.id).exists() - - -def test_api_documents_content_upadte_invalid_yjs_doc(): - """sending an invalid yjs doc as content should return a 400.""" - user = factories.UserFactory() - document = factories.DocumentFactory(link_reach="restricted") - factories.UserDocumentAccessFactory(document=document, user=user, role="editor") - - client = APIClient() - client.force_login(user) - - assert get_s3_content(document) == factories.YDOC_HELLO_WORLD_BASE64 - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": base64.b64encode(b"invalid yjs").decode("utf-8")}, - ) - - assert response.status_code == status.HTTP_400_BAD_REQUEST diff --git a/src/backend/core/tests/documents/test_api_documents_retrieve.py b/src/backend/core/tests/documents/test_api_documents_retrieve.py index b3105afe6c..c4cd90b3a1 100644 --- a/src/backend/core/tests/documents/test_api_documents_retrieve.py +++ b/src/backend/core/tests/documents/test_api_documents_retrieve.py @@ -51,8 +51,6 @@ def test_api_documents_retrieve_anonymous_public_standalone(): "public": ["reader", "commenter", "editor"], "restricted": None, }, - "content_patch": document.link_role == "editor", - "content_retrieve": True, "leave": False, "media_auth": True, "media_check": True, @@ -129,8 +127,6 @@ def test_api_documents_retrieve_anonymous_public_parent(): "link_select_options": models.LinkReachChoices.get_select_options( **links_definition ), - "content_patch": grand_parent.link_role == "editor", - "content_retrieve": True, "leave": False, "media_auth": True, "media_check": True, @@ -240,8 +236,6 @@ def test_api_documents_retrieve_authenticated_unrelated_public_or_authenticated( "public": ["reader", "commenter", "editor"], "restricted": None, }, - "content_patch": document.link_role == "editor", - "content_retrieve": True, "leave": True, "media_auth": True, "media_check": True, @@ -326,8 +320,6 @@ def test_api_documents_retrieve_authenticated_public_or_authenticated_parent(rea **links_definition ), "move": False, - "content_patch": grand_parent.link_role == "editor", - "content_retrieve": True, "leave": True, "media_auth": True, "media_check": True, @@ -524,8 +516,6 @@ def test_api_documents_retrieve_authenticated_related_parent(): "link_select_options": models.LinkReachChoices.get_select_options( **link_definition ), - "content_patch": access.role not in ["reader", "commenter"], - "content_retrieve": True, "leave": access.role not in ["administrator", "owner"], "media_auth": True, "media_check": True, diff --git a/src/backend/core/tests/documents/test_api_documents_trashbin.py b/src/backend/core/tests/documents/test_api_documents_trashbin.py index b32da42e3d..3a9caafecd 100644 --- a/src/backend/core/tests/documents/test_api_documents_trashbin.py +++ b/src/backend/core/tests/documents/test_api_documents_trashbin.py @@ -100,8 +100,6 @@ def test_api_documents_trashbin_format(): "public": ["reader", "commenter", "editor"], "restricted": None, }, - "content_patch": False, - "content_retrieve": True, "leave": False, "media_auth": False, "media_check": False, @@ -167,8 +165,6 @@ def test_api_documents_trashbin_format(): "public": ["reader", "commenter", "editor"], "restricted": None, }, - "content_patch": False, - "content_retrieve": True, "leave": False, "media_auth": False, "media_check": False, diff --git a/src/backend/core/tests/documents/test_api_documents_update_extract_attachments.py b/src/backend/core/tests/documents/test_api_documents_update_extract_attachments.py deleted file mode 100644 index a9d04c4ac3..0000000000 --- a/src/backend/core/tests/documents/test_api_documents_update_extract_attachments.py +++ /dev/null @@ -1,154 +0,0 @@ -""" -Test extract-attachments on document update in docs core app. -""" - -import base64 -from uuid import uuid4 - -import pycrdt -import pytest -from rest_framework.test import APIClient - -from core import factories - -pytestmark = pytest.mark.django_db - - -def get_ydoc_with_images(image_keys): - """Return a ydoc from text for testing purposes.""" - ydoc = pycrdt.Doc() - fragment = pycrdt.XmlFragment( - [ - pycrdt.XmlElement("img", {"src": f"http://localhost/media/{key:s}"}) - for key in image_keys - ] - ) - ydoc["document-store"] = fragment - update = ydoc.get_update() - return base64.b64encode(update).decode("utf-8") - - -def test_api_documents_update_new_attachment_keys_anonymous(django_assert_num_queries): - """ - When an anonymous user updates a document, the attachment keys extracted from the - updated content should be added to the list of "attachments" to the document if these - attachments are already readable by anonymous users. - """ - image_keys = [f"{uuid4()!s}/attachments/{uuid4()!s}.png" for _ in range(4)] - document = factories.DocumentFactory( - content=get_ydoc_with_images(image_keys[:1]), - attachments=[image_keys[0]], - link_reach="public", - link_role="editor", - ) - - factories.DocumentFactory(attachments=[image_keys[1]], link_reach="public") - factories.DocumentFactory(attachments=[image_keys[2]], link_reach="authenticated") - factories.DocumentFactory(attachments=[image_keys[3]], link_reach="restricted") - expected_keys = {image_keys[i] for i in [0, 1]} - - with django_assert_num_queries(9): - response = APIClient().patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_ydoc_with_images(image_keys)}, - format="json", - ) - assert response.status_code == 204 - - document.refresh_from_db() - assert set(document.attachments) == expected_keys - - # Check that the db query to check attachments readability for extracted - # keys is not done if the content changes but no new keys are found - with django_assert_num_queries(7): - response = APIClient().patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_ydoc_with_images(image_keys[:2]), "websocket": True}, - format="json", - ) - assert response.status_code == 204 - - document.refresh_from_db() - assert len(document.attachments) == 2 - assert set(document.attachments) == expected_keys - - -def test_api_documents_update_new_attachment_keys_authenticated( - django_assert_num_queries, -): - """ - When an authenticated user updates a document, the attachment keys extracted from the - updated content should be added to the list of "attachments" to the document if these - attachments are already readable by the editing user. - """ - user = factories.UserFactory() - client = APIClient() - client.force_login(user) - - image_keys = [f"{uuid4()!s}/attachments/{uuid4()!s}.png" for _ in range(5)] - document = factories.DocumentFactory( - content=get_ydoc_with_images(image_keys[:1]), - attachments=[image_keys[0]], - users=[(user, "editor")], - ) - - factories.DocumentFactory(attachments=[image_keys[1]], link_reach="public") - factories.DocumentFactory(attachments=[image_keys[2]], link_reach="authenticated") - factories.DocumentFactory(attachments=[image_keys[3]], link_reach="restricted") - factories.DocumentFactory(attachments=[image_keys[4]], users=[user]) - expected_keys = {image_keys[i] for i in [0, 1, 2, 4]} - - with django_assert_num_queries(10): - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_ydoc_with_images(image_keys)}, - format="json", - ) - assert response.status_code == 204 - - document.refresh_from_db() - assert set(document.attachments) == expected_keys - - # Check that the db query to check attachments readability for extracted - # keys is not done if the content changes but no new keys are found - with django_assert_num_queries(8): - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_ydoc_with_images(image_keys[:2])}, - format="json", - ) - assert response.status_code == 204 - - document.refresh_from_db() - assert len(document.attachments) == 4 - assert set(document.attachments) == expected_keys - - -def test_api_documents_update_new_attachment_keys_duplicate(): - """ - Duplicate keys in the content should not result in duplicates in the document's attachments. - """ - user = factories.UserFactory() - client = APIClient() - client.force_login(user) - - image_key1 = f"{uuid4()!s}/attachments/{uuid4()!s}.png" - image_key2 = f"{uuid4()!s}/attachments/{uuid4()!s}.png" - document = factories.DocumentFactory( - content=get_ydoc_with_images([image_key1]), - attachments=[image_key1], - users=[(user, "editor")], - ) - - factories.DocumentFactory(attachments=[image_key2], users=[user]) - - response = client.patch( - f"/api/v1.0/documents/{document.id!s}/content/", - {"content": get_ydoc_with_images([image_key1, image_key2, image_key2])}, - format="json", - ) - assert response.status_code == 204 - - document.refresh_from_db() - assert len(document.attachments) == 2 - assert set(document.attachments) == {image_key1, image_key2} diff --git a/src/backend/core/tests/test_api_utils_parse_http_conditional_headers.py b/src/backend/core/tests/test_api_utils_parse_http_conditional_headers.py deleted file mode 100644 index b654b99951..0000000000 --- a/src/backend/core/tests/test_api_utils_parse_http_conditional_headers.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -Unit tests for the parse_http_conditional_headers utility function. -""" - -import datetime as dt - -import pytest -from rest_framework.test import APIRequestFactory - -from core.api.utils import parse_http_conditional_headers - - -@pytest.fixture(name="prepare_request") -def fixture_prepare_request(request): - """ - Fixture returning a request with headers configured from the indirect parametrize parameters. - """ - return APIRequestFactory().get("/", headers=request.param) - - -@pytest.mark.parametrize( - "prepare_request, expected_if_none_match, expected_if_modified_since", - [ - ({}, None, None), - ({"if-none-match": '"abc123"'}, '"abc123"', None), - ({"if-none-match": 'W/"abc123"'}, '"abc123"', None), - ( - {"if-modified-since": "Wed, 21 Oct 2015 07:28:00 GMT"}, - None, - dt.datetime(2015, 10, 21, 7, 28, 0, tzinfo=dt.timezone.utc), - ), - ({"if-modified-since": "not-a-date"}, None, None), - ( - { - "if-none-match": 'W/"deadbeef"', - "if-modified-since": "Wed, 21 Oct 2015 07:28:00 GMT", - }, - '"deadbeef"', - dt.datetime(2015, 10, 21, 7, 28, 0, tzinfo=dt.timezone.utc), - ), - ], - indirect=["prepare_request"], -) -def test_api_utils_parse_http_conditional_headers( - prepare_request, expected_if_none_match, expected_if_modified_since -): - """Test parse_http_conditional_headers utils.""" - if_none_match, if_modified_since_dt = parse_http_conditional_headers( - prepare_request - ) - assert if_none_match == expected_if_none_match - assert if_modified_since_dt == expected_if_modified_since diff --git a/src/backend/core/tests/test_models_documents.py b/src/backend/core/tests/test_models_documents.py index 5c124d15fa..407e239da7 100644 --- a/src/backend/core/tests/test_models_documents.py +++ b/src/backend/core/tests/test_models_documents.py @@ -170,8 +170,6 @@ def test_models_documents_get_abilities_forbidden( "favorite": False, "comment": False, "invite_owner": False, - "content_patch": False, - "content_retrieve": False, "leave": False, "media_auth": False, "media_check": False, @@ -244,8 +242,6 @@ def test_models_documents_get_abilities_reader( "public": ["reader", "commenter", "editor"], "restricted": None, }, - "content_patch": False, - "content_retrieve": True, "leave": False, "media_auth": True, "media_check": True, @@ -317,8 +313,6 @@ def test_models_documents_get_abilities_commenter( "public": ["reader", "commenter", "editor"], "restricted": None, }, - "content_patch": False, - "content_retrieve": True, "leave": False, "media_auth": True, "media_check": True, @@ -387,8 +381,6 @@ def test_models_documents_get_abilities_editor( "public": ["reader", "commenter", "editor"], "restricted": None, }, - "content_patch": True, - "content_retrieve": True, "leave": False, "media_auth": True, "media_check": True, @@ -446,8 +438,6 @@ def test_models_documents_get_abilities_owner(django_assert_num_queries): "public": ["reader", "commenter", "editor"], "restricted": None, }, - "content_patch": True, - "content_retrieve": True, "leave": False, "media_auth": True, "media_check": True, @@ -491,8 +481,6 @@ def test_models_documents_get_abilities_owner(django_assert_num_queries): "public": ["reader", "commenter", "editor"], "restricted": None, }, - "content_patch": False, - "content_retrieve": True, "leave": False, "media_auth": False, "media_check": False, @@ -540,8 +528,6 @@ def test_models_documents_get_abilities_administrator(django_assert_num_queries) "public": ["reader", "commenter", "editor"], "restricted": None, }, - "content_patch": True, - "content_retrieve": True, "leave": False, "media_auth": True, "media_check": True, @@ -599,8 +585,6 @@ def test_models_documents_get_abilities_editor_user(django_assert_num_queries): "public": ["reader", "commenter", "editor"], "restricted": None, }, - "content_patch": True, - "content_retrieve": True, "leave": True, "media_auth": True, "media_check": True, @@ -666,8 +650,6 @@ def test_models_documents_get_abilities_reader_user( "public": ["reader", "commenter", "editor"], "restricted": None, }, - "content_patch": access_from_link, - "content_retrieve": True, "leave": True, "media_auth": True, "media_check": True, @@ -734,8 +716,6 @@ def test_models_documents_get_abilities_commenter_user( "public": ["reader", "commenter", "editor"], "restricted": None, }, - "content_patch": access_from_link, - "content_retrieve": True, "leave": True, "media_auth": True, "media_check": True, @@ -798,8 +778,6 @@ def test_models_documents_get_abilities_preset_role(django_assert_num_queries): "public": ["reader", "commenter", "editor"], "restricted": None, }, - "content_patch": False, - "content_retrieve": True, "leave": True, "media_auth": True, "media_check": True, diff --git a/src/backend/core/tests/test_utils_s3_response_stream.py b/src/backend/core/tests/test_utils_s3_response_stream.py deleted file mode 100644 index e42c1d79fd..0000000000 --- a/src/backend/core/tests/test_utils_s3_response_stream.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Test the s3 response stream utilities.""" - -from collections.abc import AsyncIterator, Iterator - -import pytest -from asgiref.sync import async_to_sync - -from core.utils.s3_response_stream import async_stream, content_stream, sync_stream - -pytestmark = pytest.mark.django_db - - -class FakeS3Body: - """Minimal stand-in for a botocore StreamingBody.""" - - def __init__(self, chunks): - self._chunks = chunks - self.closed = False - - def iter_chunks(self): - """Yield the configured chunks, like StreamingBody.iter_chunks.""" - yield from self._chunks - - def close(self): - """Record that the body has been closed.""" - self.closed = True - - -def collect_async(async_gen): - """Consume an async generator synchronously and return its items as a list.""" - - async def _collect(): - return [chunk async for chunk in async_gen] - - return async_to_sync(_collect)() - - -# -- sync_stream -- - - -def test_sync_stream_yields_all_chunks(): - """Should yield every chunk of the body in order.""" - body = FakeS3Body([b"hello", b"world", b"!"]) - - assert list(sync_stream(body)) == [b"hello", b"world", b"!"] - - -def test_sync_stream_empty_body(): - """Should yield nothing when the body is empty.""" - body = FakeS3Body([]) - - assert not list(sync_stream(body)) - - -def test_sync_stream_closes_body(): - """Should close the body once it has been fully consumed.""" - body = FakeS3Body([b"hello"]) - - assert body.closed is False - list(sync_stream(body)) - assert body.closed is True - - -# -- async_stream -- - - -def test_async_stream_yields_all_chunks(): - """Should yield every chunk of the body in order.""" - body = FakeS3Body([b"hello", b"world", b"!"]) - - assert collect_async(async_stream(body)) == [b"hello", b"world", b"!"] - - -def test_async_stream_empty_body(): - """Should yield nothing when the body is empty.""" - body = FakeS3Body([]) - - assert not collect_async(async_stream(body)) - - -def test_async_stream_closes_body(): - """Should close the body once it has been fully consumed.""" - body = FakeS3Body([b"hello"]) - - assert body.closed is False - collect_async(async_stream(body)) - assert body.closed is True - - -# -- content_stream -- - - -def test_content_stream_async_mode(monkeypatch): - """In async mode, content_stream should return an async iterator.""" - monkeypatch.setenv("PYTHON_SERVER_MODE", "async") - body = FakeS3Body([b"hello", b"world"]) - - stream = content_stream(body) - - assert isinstance(stream, AsyncIterator) - assert collect_async(stream) == [b"hello", b"world"] - - -def test_content_stream_sync_mode(monkeypatch): - """In sync mode, content_stream should return a sync iterator.""" - monkeypatch.setenv("PYTHON_SERVER_MODE", "sync") - body = FakeS3Body([b"hello", b"world"]) - - stream = content_stream(body) - - assert not isinstance(stream, AsyncIterator) - assert isinstance(stream, Iterator) - assert list(stream) == [b"hello", b"world"] - - -def test_content_stream_defaults_to_sync(monkeypatch): - """When PYTHON_SERVER_MODE is not set, content_stream should default to sync.""" - monkeypatch.delenv("PYTHON_SERVER_MODE", raising=False) - body = FakeS3Body([b"hello", b"world"]) - - stream = content_stream(body) - - assert not isinstance(stream, AsyncIterator) - assert isinstance(stream, Iterator) - assert list(stream) == [b"hello", b"world"] diff --git a/src/backend/core/utils/s3_response_stream.py b/src/backend/core/utils/s3_response_stream.py deleted file mode 100644 index ddf364f39f..0000000000 --- a/src/backend/core/utils/s3_response_stream.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Utils module to stream content to a StreamingHttpResponse""" - -import os - -from asgiref.sync import sync_to_async -from botocore.response import StreamingBody - - -def _is_async_server(): - """ - Return whether the app runs as an ASGI application, based on the - PYTHON_SERVER_MODE environment variable (set in impress/asgi.py and - impress/wsgi.py). - """ - return os.environ.get("PYTHON_SERVER_MODE", "sync") == "async" - - -def sync_stream(body: StreamingBody): - """Synchronous generator consuming s3 response body.""" - yield from body.iter_chunks() - body.close() - - -async def async_stream(body: StreamingBody): - """Asynchronous generator consuming s3 response body""" - # The botocore stream is blocking, so each read is offloaded with - # sync_to_async to avoid blocking the event loop. - chunks = await sync_to_async(body.iter_chunks)() - sentinel = object() - while True: - chunk = await sync_to_async(next)(chunks, sentinel) - if chunk is sentinel: - break - yield chunk - await sync_to_async(body.close)() - - -def content_stream(body: StreamingBody): - """ - Depending on the server mode (set through the PYTHON_SERVER_MODE - environment variable in impress/asgi.py and impress/wsgi.py), the - content is streamed back with either an asynchronous or a synchronous - iterator. Under ASGI, a synchronous iterator would trigger a Django - warning and be consumed synchronously, defeating the purpose of - streaming. - """ - return async_stream(body) if _is_async_server() else sync_stream(body) diff --git a/src/backend/impress/settings.py b/src/backend/impress/settings.py index 9b79c48ebb..724331036a 100755 --- a/src/backend/impress/settings.py +++ b/src/backend/impress/settings.py @@ -1123,10 +1123,6 @@ class Base(Configuration): ), } - CONTENT_METADATA_CACHE_TIMEOUT = values.IntegerValue( - 60 * 60 * 24, environ_name="CONTENT_METADATA_CACHE_TIMEOUT", environ_prefix=None - ) - TREEBEARD_PATH_COMPUTE_RETRY_MAX_ATTEMPTS = values.IntegerValue( 10, environ_name="TREEBEARD_PATH_COMPUTE_RETRY_MAX_ATTEMPTS", From 661a06094145113dbf125ad5345c6c5decbd0af1 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Thu, 6 Aug 2026 17:24:52 +0200 Subject: [PATCH 37/59] =?UTF-8?q?=E2=9C=A8(backend)=20serve=20`documents/{?= =?UTF-8?q?id}/formatted-content/`=20from=20yhub?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The formatted-content endpoint was using the `document.content` to fetch the ydoc from s3, we want to move from this usage to using yhub to retrieve the content, so yhub is becoming our source of thruth. --- CHANGELOG.md | 1 + src/backend/core/api/viewsets.py | 18 ++++-- .../test_api_documents_formatted_content.py | 63 +++++++++++++++++++ 3 files changed, 77 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index baf35ea699..a95399848a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,7 @@ and this project adheres to instead of giving up. The built-in endpoints can also answer JSON on `Accept: application/json` - ✨(collaboration) add a get-ydoc endpoint on yhub +- ✨(backend) serve `documents/{id}/formatted-content/` from yhub - ✨(backend) duplicate a document through the collaboration server - ✨(backend) call YHubService to seed initial document content - ✨(backend) reset the yhub connections of a document and its descendants diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index 84b614ff94..0fde145aed 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -2,7 +2,6 @@ # pylint: disable=too-many-lines -import base64 import ipaddress import json import logging @@ -2416,15 +2415,24 @@ def formatted_content(self, request, pk=None): "Invalid format. Must be one of: json, markdown, html" ) - # Get the base64 content from the document + # Get the content from the collaboration server, it is the source of + # truth for it + try: + update = YHubService(user=request.user).get_ydoc(document) + except YHubError as e: + logger.error("Error getting content for document %s: %s", pk, e) + return drf_response.Response( + {"error": "Failed to get document content"}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + content = None - base64_content = document.content - if base64_content is not None: + if update is not None: # Convert using the y-provider service try: yprovider = Converter() result = yprovider.convert( - base64.b64decode(base64_content), + update, mime_types.YJS, { "markdown": mime_types.MARKDOWN, diff --git a/src/backend/core/tests/documents/test_api_documents_formatted_content.py b/src/backend/core/tests/documents/test_api_documents_formatted_content.py index b2318b6275..7a800a10fb 100644 --- a/src/backend/core/tests/documents/test_api_documents_formatted_content.py +++ b/src/backend/core/tests/documents/test_api_documents_formatted_content.py @@ -11,10 +11,30 @@ from rest_framework.test import APIClient from core import factories +from core.services.yhub_services import ( + ServiceUnavailableError as YHubServiceUnavailableError, +) pytestmark = pytest.mark.django_db +@pytest.fixture(autouse=True, name="mock_yhub") +def mock_yhub_fixture(): + """ + The content of a document is held by the collaboration server. + + It stands for a server holding the very content the factories gave the + documents, which is what an editor connected to it would have saved. + """ + + def get_ydoc(document): + return base64.b64decode(document.content) if document.content else None + + with patch("core.api.viewsets.YHubService") as mock_service: + mock_service.return_value.get_ydoc.side_effect = get_ydoc + yield mock_service + + @pytest.mark.parametrize( "reach, role", [ @@ -182,3 +202,46 @@ def test_api_documents_formatted_content_empty_document(mock_request): assert data["title"] == document.title assert data["content"] is None mock_request.assert_not_called() + + +@patch("core.services.converter_services.YdocConverter.convert") +def test_api_documents_formatted_content_from_collaboration_server( + mock_content, mock_yhub +): + """The content converted is the one held by the collaboration server.""" + document = factories.DocumentFactory(link_reach="public") + mock_content.return_value = {"some": "data"} + # what the collaboration server holds, edited since Django last saw it + mock_yhub.return_value.get_ydoc.side_effect = None + mock_yhub.return_value.get_ydoc.return_value = b"\x01\x02edited update" + + response = APIClient().get( + f"/api/v1.0/documents/{document.id!s}/formatted-content/" + ) + + assert response.status_code == status.HTTP_200_OK + mock_yhub.return_value.get_ydoc.assert_called_once_with(document) + mock_content.assert_called_once_with( + b"\x01\x02edited update", + "application/vnd.yjs.doc", + "application/json", + ) + + +@patch("core.services.converter_services.YdocConverter.convert") +def test_api_documents_formatted_content_collaboration_server_error( + mock_content, mock_yhub +): + """A content the collaboration server cannot serve should answer a 500.""" + document = factories.DocumentFactory(link_reach="public") + mock_yhub.return_value.get_ydoc.side_effect = YHubServiceUnavailableError( + "Failed to connect to the yhub service" + ) + + response = APIClient().get( + f"/api/v1.0/documents/{document.id!s}/formatted-content/" + ) + + assert response.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR + assert response.json() == {"error": "Failed to get document content"} + mock_content.assert_not_called() From 57065e8845cd11e987dca13e8c97edc5fe0f5174 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Fri, 7 Aug 2026 10:51:32 +0200 Subject: [PATCH 38/59] =?UTF-8?q?=E2=9C=A8(collaboration)=20notify=20the?= =?UTF-8?q?=20backend=20when=20the=20worker=20persists=20new=20content?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit notify the backend when the worker persists new content for a document, so the lists ordered by `updated_at` follow the edits made on the collaboration server. The backend serves it on `POST /api/v1.0/documents/{id}/content-updated/`, authenticated with a short lived RS256 JWT the collaboration server signs (`aud: "docs-backend"`) and the backend verifies against the JWKS the collaboration server publishes on `/collaboration/jwks/v1` — the mirror of the admin token the backend signs to call it, so no long lived secret is shared and either side can roll its key on its own --- CHANGELOG.md | 11 + Makefile | 11 +- UPGRADE.md | 22 ++ bin/generate-jwt-private-key.sh | 40 ++- compose.yml | 4 + src/backend/core/api/viewsets.py | 31 ++ src/backend/core/authentication/__init__.py | 65 +++++ src/backend/core/services/jwt_services.py | 108 +++++++ src/backend/core/services/yhub_services.py | 22 +- .../test_api_documents_content_updated.py | 273 ++++++++++++++++++ .../core/tests/test_services_jwks_client.py | 165 +++++++++++ .../core/tests/test_services_yhub_services.py | 8 + src/backend/core/tests/utils/jwt_helper.py | 22 ++ src/yhub-server/README.md | 20 +- src/yhub-server/server.js | 177 +++++++++++- 15 files changed, 949 insertions(+), 30 deletions(-) create mode 100644 src/backend/core/tests/documents/test_api_documents_content_updated.py create mode 100644 src/backend/core/tests/test_services_jwks_client.py diff --git a/CHANGELOG.md b/CHANGELOG.md index a95399848a..7872708cad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,17 @@ and this project adheres to `503` instead of denying access like a permission failure, so clients retry instead of giving up. The built-in endpoints can also answer JSON on `Accept: application/json` +- ✨(collaboration) notify the backend when the worker persists new content for + a document, so the lists ordered by `updated_at` follow the edits made on the + collaboration server. The backend serves it on + `POST /api/v1.0/documents/{id}/content-updated/`, authenticated with a short + lived RS256 JWT the collaboration server signs (`aud: "docs-backend"`) and + the backend verifies against the JWKS the collaboration server publishes on + `/collaboration/jwks/v1` — the mirror of the admin token the backend signs to + call it, so no long lived secret is shared and either side can roll its key + on its own +- 🔧(dev) generate the JWT signing key of the collaboration server when + bootstrapping the dev stack, alongside the backend one - ✨(collaboration) add a get-ydoc endpoint on yhub - ✨(backend) serve `documents/{id}/formatted-content/` from yhub - ✨(backend) duplicate a document through the collaboration server diff --git a/Makefile b/Makefile index 82729249a6..5753f44753 100644 --- a/Makefile +++ b/Makefile @@ -69,11 +69,16 @@ data/media: data/static: @mkdir -p data/static -# RSA key signing the JWT tokens the backend issues. Generated locally, never -# committed: "data/" is gitignored. Regenerate it by deleting the file. +# RSA keys signing the JWT tokens the services issue: one for the backend, one +# for the collaboration server. Generated locally, never committed: "data/" is +# gitignored. Regenerate one by deleting the file. Both are listed, so a stack +# set up before the collaboration server had a key of its own gets it too. data/jwt/private.pem: @bin/generate-jwt-private-key.sh +data/jwt/yhub-private.pem: + @bin/generate-jwt-private-key.sh + # -- Project create-env-local-files: ## create env.local files in env.d/development @@ -87,7 +92,7 @@ create-env-local-files: generate-secret-keys: generate-secret-keys: ## generate the secret keys needed by the dev stack -generate-secret-keys: data/jwt/private.pem +generate-secret-keys: data/jwt/private.pem data/jwt/yhub-private.pem @bin/generate-oidc-store-refresh-token-key.sh .PHONY: generate-secret-keys diff --git a/UPGRADE.md b/UPGRADE.md index 06629eb5de..b7d3f5ac09 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -31,6 +31,28 @@ the following command inside your docker container: tokens Docs issues to call external services. If you enabled the resource server (`OIDC_RESOURCE_SERVER_ENABLED`), update the JWKS URI declared to your OIDC provider accordingly. +- ⚠️ The collaboration server now calls the backend on its own, to declare that + a document was edited, and signs those calls: **it needs an RSA private key + of its own**, which it had not before. Generate one and give it to the + collaboration server in `YHUB_JWT_PRIVATE_KEY`, or in a file + `YHUB_JWT_PRIVATE_KEY_FILE` points at: + + ```bash + openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out yhub-private.pem + ``` + + There is nothing to configure on the backend side: it reads the public half + from the JWKS the collaboration server publishes on `/collaboration/jwks/v1`, + which it fetches over `YHUB_API_BASE_URL` — so the two only need to reach + each other, and this key can be rolled without the backend being touched. + Do not share the backend key (`JWT_PRIVATE_KEY`) with it: each service signs + with a key of its own. + + Without this key the collaboration server keeps serving documents, and warns + at startup that it will not notify the backend: the `updated_at` of a document + then stops following the edits made in the editor, and the lists ordered by it + drift out of date. In a development environment, + `make generate-secret-keys` creates the key in `data/jwt/`. ### [5.0.0] - 2026-04-30 diff --git a/bin/generate-jwt-private-key.sh b/bin/generate-jwt-private-key.sh index 3d2963bd0c..76d62eb48e 100755 --- a/bin/generate-jwt-private-key.sh +++ b/bin/generate-jwt-private-key.sh @@ -1,23 +1,39 @@ #!/usr/bin/env bash -# Generate the RSA private key signing the JWT tokens issued by the backend. +# Generate the RSA keys signing the JWT tokens exchanged between the services. # -# Development only. The key is generated locally and never committed: it lands -# in "data/", which is gitignored. The dev stack mounts it in the backend -# containers, where JWT_PRIVATE_KEY_FILE points at it. +# Two directions, hence two keys: +# - "private.pem" signs the tokens the backend issues to call the converter and +# the collaboration server. +# - "yhub-private.pem" signs the calls the collaboration server makes to the +# backend. # -# Idempotent: an existing key is kept. Delete the file to roll the key. +# Only the private halves exist as files: each service publishes the public half +# of its own key on its JWKS endpoint, where the other one reads it. +# +# Development only. The keys are generated locally and never committed: they +# land in "data/", which is gitignored. The dev stack mounts them in the +# containers, where the *_FILE settings point at them. +# +# Idempotent: existing keys are kept. Delete a file to roll it. set -eo pipefail REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -KEY_PATH="${REPO_DIR}/data/jwt/private.pem" +KEY_DIR="${REPO_DIR}/data/jwt" -if [ -f "${KEY_PATH}" ]; then - exit 0 +mkdir -p "${KEY_DIR}" + +if [ ! -f "${KEY_DIR}/private.pem" ]; then + openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 \ + -out "${KEY_DIR}/private.pem" 2>/dev/null + chmod 600 "${KEY_DIR}/private.pem" + echo "✓ backend JWT private key generated in ${KEY_DIR}/private.pem" fi -mkdir -p "$(dirname "${KEY_PATH}")" -openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${KEY_PATH}" 2>/dev/null -chmod 600 "${KEY_PATH}" -echo "✓ JWT private key generated in ${KEY_PATH}" +if [ ! -f "${KEY_DIR}/yhub-private.pem" ]; then + openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 \ + -out "${KEY_DIR}/yhub-private.pem" 2>/dev/null + chmod 600 "${KEY_DIR}/yhub-private.pem" + echo "✓ collaboration JWT private key generated in ${KEY_DIR}/yhub-private.pem" +fi diff --git a/compose.yml b/compose.yml index c9f0e9a2d6..3c4b63239f 100644 --- a/compose.yml +++ b/compose.yml @@ -248,9 +248,13 @@ services: # seed rooms from the legacy Django/S3 document store on first access — # S3 endpoint/credentials come from env.d/development/common SOFT_MIGRATION: "true" + # signs the calls made to the backend, which holds the public half + YHUB_JWT_PRIVATE_KEY_FILE: /data/jwt/yhub-private.pem env_file: - env.d/development/common - env.d/development/common.local + volumes: + - ./data/jwt:/data/jwt:ro restart: unless-stopped ports: - "3002:3002" diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index 0fde145aed..b22ce17e4c 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -932,6 +932,37 @@ def create_for_owner(self, request): {"id": str(document.id)}, status=status.HTTP_201_CREATED ) + @drf.decorators.action( + authentication_classes=[authentication.CollaborationServerAuthentication], + detail=True, + methods=["post"], + permission_classes=[], + url_path="content-updated", + ) + def content_updated(self, request, *args, **kwargs): + """ + Record that the collaboration server saved a new content for a document. + + The content of a document does not go through Django anymore, so nothing + would refresh its "updated_at" as it is edited and the lists ordered by + it would freeze. The collaboration server calls this once it persisted + the changes of a document, at most once per debounce window. + + The update is written without going through the model, saving it would + trigger a re-indexing of a content Django did not see change. + """ + try: + document_id = uuid.UUID(kwargs["pk"]) + except ValueError as err: + raise Http404 from err + + if not models.Document.objects.filter(pk=document_id).update( + updated_at=timezone.now() + ): + raise Http404 + + return drf_response.Response(status=status.HTTP_204_NO_CONTENT) + @drf.decorators.action(detail=True, methods=["post"]) @transaction.atomic def move(self, request, *args, **kwargs): diff --git a/src/backend/core/authentication/__init__.py b/src/backend/core/authentication/__init__.py index c5fa0c7113..6c615dacb8 100644 --- a/src/backend/core/authentication/__init__.py +++ b/src/backend/core/authentication/__init__.py @@ -2,9 +2,13 @@ from django.conf import settings +import jwt from rest_framework.authentication import BaseAuthentication from rest_framework.exceptions import AuthenticationFailed +from core.services.jwt_services import JWTError +from core.services.yhub_services import YHubError, YHubService + class ServerToServerAuthentication(BaseAuthentication): """ @@ -50,3 +54,64 @@ def authenticate(self, request): def authenticate_header(self, request): """Return the WWW-Authenticate header value.""" return f"{self.TOKEN_TYPE} realm='Create document server to server'" + + +class CollaborationServerAuthentication(BaseAuthentication): + """ + Authenticate the collaboration server on the JWT it signs. + + The mirror of the token the backend signs to call it: the collaboration + server holds the private key and publishes the public half on its JWKS, + which is where we read it, and the tokens it mints live for a few minutes. + Nothing long-lived is shared between them, and either side can roll its key + without the other being reconfigured. + """ + + AUTH_HEADER = "Authorization" + TOKEN_TYPE = "Bearer" # noqa S105 + ALGORITHM = "RS256" + # The collaboration server mints its tokens for us, and only us: a token it + # signed for another service is refused here. + AUDIENCE = "docs-backend" + + def authenticate(self, request): + """ + Authenticate the request on the signature, the audience and the expiry + of the token it carries. The key validating the signature is the one + the collaboration server publishes for the "kid" the token names. + + Returns: + None: If authentication is successful, no user acts behind a call + of the collaboration server. + + Raises: + AuthenticationFailed: If the Authorization header is missing, + malformed, or carries a token we cannot validate. + """ + auth_header = request.headers.get(self.AUTH_HEADER) + if not auth_header: + raise AuthenticationFailed("Authorization header is missing.") + + auth_parts = auth_header.split(" ") + if len(auth_parts) != 2 or auth_parts[0] != self.TOKEN_TYPE: + raise AuthenticationFailed("Invalid authorization header.") + + token = auth_parts[1] + try: + signing_key = YHubService().jwks.get_signing_key(token) + jwt.decode( + token, + signing_key.key, + algorithms=[self.ALGORITHM], + audience=self.AUDIENCE, + ) + # a collaboration server we cannot reach, or one that publishes nothing + # we can verify a token with, authenticates nobody either + except (jwt.PyJWTError, JWTError, YHubError) as err: + raise AuthenticationFailed("Invalid collaboration server token.") from err + + # Authentication is successful, but no user is authenticated + + def authenticate_header(self, request): + """Return the WWW-Authenticate header value.""" + return f"{self.TOKEN_TYPE} realm='Collaboration server'" diff --git a/src/backend/core/services/jwt_services.py b/src/backend/core/services/jwt_services.py index 7ac666fbaf..6edafb5d7f 100644 --- a/src/backend/core/services/jwt_services.py +++ b/src/backend/core/services/jwt_services.py @@ -12,12 +12,24 @@ from django.utils import timezone import jwt +import requests from joserfc.jwk import KeySet, RSAKey logger = logging.getLogger(__name__) ALGORITHM = "RS256" CACHE_KEY_PREFIX = "jwt_token" +JWKS_CACHE_KEY_PREFIX = "jwks" +# How long a fetched JWKS is served from the cache before being fetched again. +JWKS_CACHE_TIMEOUT = 300 +# Minimum delay, in seconds, between two fetches of the same JWKS. A token +# names the key that signed it and anybody can name one that does not exist, so +# the refresh a rotation needs is rate limited: an unknown key costs at most one +# fetch per window, not one per request. +JWKS_REFRESH_COOLDOWN = 30 +# Timeout, in seconds, of the fetch of a JWKS. Short: it happens while +# authenticating a request. +JWKS_FETCH_TIMEOUT = 10 class Audiences(StrEnum): @@ -39,6 +51,10 @@ class TokenGenerationError(JWTError): """Raised when a token cannot be signed.""" +class JWKSError(JWTError): + """Raised when the keys validating the tokens of a service cannot be used.""" + + @functools.cache def import_private_key(private_key): """ @@ -66,6 +82,98 @@ def import_private_key(private_key): raise ConfigurationError("The JWT private key cannot be imported.") from err +@functools.cache +def import_jwks(jwks): + """ + Import a JSON Web Key Set, as published by the service issuing the tokens. + + Importing keys is expensive, hence the cache. It is keyed on the document + itself, so a service publishing a new key gets it imported instead of the + previous set being served forever. + """ + try: + return jwt.PyJWKSet.from_json(jwks) + # a JWKS is fetched from another service: anything malformed in it, from + # the JSON to the key material, must surface as a JWKS error + except (jwt.PyJWTError, AttributeError, TypeError, ValueError) as err: + raise JWKSError("The JWKS cannot be imported.") from err + + +class JWKSClient: + """ + Client of the JSON Web Key Set a service publishes to let us verify the + tokens it signs. + + Fetching and importing the keys on every token would be wasteful, so the + document is cached — in the Django cache, hence shared by our processes — + and its import memoized. The service can still roll its key without + anything changing here: a token signed by a key we do not know refreshes + the set. + """ + + def __init__(self, url, timeout=JWKS_FETCH_TIMEOUT): + """Bind the client to the url a service publishes its keys at.""" + self.url = url + self.timeout = timeout + + @property + def cache_key(self): + """Build the cache key holding the document published at our url.""" + digest = hashlib.sha256(self.url.encode("utf-8")).hexdigest() + return f"{JWKS_CACHE_KEY_PREFIX}:{digest}" + + def fetch(self): + """Fetch the published document and cache it, as published.""" + try: + response = requests.get(self.url, timeout=self.timeout) + response.raise_for_status() + except requests.RequestException as err: + logger.exception("Unable to fetch the JWKS at %s", self.url) + raise JWKSError(f"Unable to fetch the JWKS at {self.url}") from err + + cache.set(self.cache_key, response.text, JWKS_CACHE_TIMEOUT) + + return response.text + + def get_keys(self, refresh=False): + """Return the published keys, from the cache unless a refresh is asked.""" + jwks = None if refresh else cache.get(self.cache_key) + if jwks is None: + jwks = self.fetch() + + return import_jwks(jwks) + + def get_signing_key(self, token): + """ + Return the key a token was signed with, among the published ones. + + The header of the token names it, which is what makes a rotation + transparent: a key we do not know yet is looked for again in a freshly + fetched set. That name is not authenticated though, so the refresh is + rate limited, and a token naming a key nobody published is refused. + """ + try: + kid = jwt.get_unverified_header(token)["kid"] + except (jwt.PyJWTError, KeyError) as err: + raise JWKSError("The token does not name the key that signed it.") from err + + try: + return self.get_keys()[kid] + except KeyError: + pass + + # `add` only succeeds for the first caller of the cooldown window, + # whichever process it runs in + if not cache.add(f"{self.cache_key}:refresh", True, JWKS_REFRESH_COOLDOWN): + raise JWKSError(f'The JWKS at {self.url} has no key "{kid}".') + + logger.info('Unknown key "%s", refreshing the JWKS at %s', kid, self.url) + try: + return self.get_keys(refresh=True)[kid] + except KeyError as err: + raise JWKSError(f'The JWKS at {self.url} has no key "{kid}".') from err + + class JWTService: """ Service class issuing RS256 signed JSON Web Tokens. diff --git a/src/backend/core/services/yhub_services.py b/src/backend/core/services/yhub_services.py index c0f74c2081..fed8c3ce5d 100644 --- a/src/backend/core/services/yhub_services.py +++ b/src/backend/core/services/yhub_services.py @@ -13,6 +13,10 @@ accepts a `branch` query parameter, but our auth plugin only ever grants access to the `main` branch, so this service never sends it. +A few routes are about the server itself rather than about a document, and +carry no room: `/{prefix}/jwks/{version}` publishes the public keys validating +the tokens yhub signs to call us back. + This service only owns the transport for now, the endpoints are added as we need them. """ @@ -23,7 +27,7 @@ import requests -from core.services.jwt_services import Audiences, JWTService +from core.services.jwt_services import Audiences, JWKSClient, JWTService logger = logging.getLogger(__name__) @@ -134,6 +138,22 @@ def auth_header(self): ) return f"Bearer {token}" + @property + def jwks_url(self): + """Return the url yhub publishes its public keys at.""" + return f"{self.base_url}/{self.api_prefix}/jwks/{self.api_version}" + + @property + def jwks(self): + """ + Return the client of the keys validating the tokens yhub signs. + + The mirror of the JWKS we publish for the tokens we sign to call it: + neither side holds a copy of the key of the other, so either can roll + its own without the other being reconfigured. + """ + return JWKSClient(self.jwks_url) + def build_url(self, endpoint, document): """Build the url of a document scoped endpoint of the yhub API.""" return ( diff --git a/src/backend/core/tests/documents/test_api_documents_content_updated.py b/src/backend/core/tests/documents/test_api_documents_content_updated.py new file mode 100644 index 0000000000..c77ea75ab4 --- /dev/null +++ b/src/backend/core/tests/documents/test_api_documents_content_updated.py @@ -0,0 +1,273 @@ +""" +Tests for Documents API endpoint in impress's core app: content updated +""" + +from datetime import datetime, timedelta +from datetime import timezone as tz +from uuid import uuid4 + +import jwt +import pytest +import responses +from freezegun import freeze_time +from rest_framework.test import APIClient + +from core import factories +from core.authentication import CollaborationServerAuthentication +from core.models import Document +from core.tests.utils.jwt_helper import build_jwks, generate_key_pair, key_id + +pytestmark = pytest.mark.django_db + +# Generating an RSA key is expensive, do it once for the whole module +PRIVATE_KEY, PUBLIC_KEY = generate_key_pair() +JWKS_URL = "http://yhub:3002/collaboration/jwks/v1" + + +@pytest.fixture(name="yhub_jwks", autouse=True) +def yhub_jwks_fixture(settings): + """ + Publish the collaboration server keys where the backend reads them. + + It only ever holds the public half of that key, and not even in its + configuration: it fetches it from the collaboration server itself. + """ + settings.YHUB_API_BASE_URL = "http://yhub:3002" + + with responses.RequestsMock(assert_all_requests_are_fired=False) as mock: + mock.get(JWKS_URL, json=build_jwks(PUBLIC_KEY)) + yield mock + + +def collaboration_token(private_key=PRIVATE_KEY, public_key=PUBLIC_KEY, **claims): + """Sign a token the way the collaboration server does.""" + issued_at = datetime.now(tz=tz.utc) + + return jwt.encode( + { + "iss": "yhub", + "aud": CollaborationServerAuthentication.AUDIENCE, + "iat": issued_at, + "exp": issued_at + timedelta(seconds=60), + **claims, + }, + private_key, + algorithm="RS256", + headers={"kid": key_id(public_key)}, + ) + + +def test_api_documents_content_updated_anonymous(): + """Anonymous users should not be allowed to declare a content update.""" + document = factories.DocumentFactory() + + response = APIClient().post(f"/api/v1.0/documents/{document.id!s}/content-updated/") + + assert response.status_code == 401 + + +def test_api_documents_content_updated_authenticated(): + """A logged-in user is not the collaboration server, their session is no credential.""" + user = factories.UserFactory() + client = APIClient() + client.force_login(user) + document = factories.DocumentFactory(users=[(user, "owner")]) + + response = client.post(f"/api/v1.0/documents/{document.id!s}/content-updated/") + + assert response.status_code == 401 + + +def test_api_documents_content_updated_token_signed_by_another_key(): + """🔒 A token signed by another key should not be allowed, published "kid" or not.""" + document = factories.DocumentFactory() + other_private_key, _other_public_key = generate_key_pair() + + response = APIClient().post( + f"/api/v1.0/documents/{document.id!s}/content-updated/", + # the key that signs it is not the one it names + HTTP_AUTHORIZATION=f"Bearer {collaboration_token(other_private_key)}", + ) + + assert response.status_code == 401 + + +def test_api_documents_content_updated_token_naming_an_unpublished_key(): + """A token naming a key the collaboration server does not publish is refused.""" + document = factories.DocumentFactory() + token = jwt.encode( + { + "aud": CollaborationServerAuthentication.AUDIENCE, + "exp": datetime.now(tz=tz.utc) + timedelta(seconds=60), + }, + PRIVATE_KEY, + algorithm="RS256", + headers={"kid": "a-key-nobody-published"}, + ) + + response = APIClient().post( + f"/api/v1.0/documents/{document.id!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {token}", + ) + + assert response.status_code == 401 + + +def test_api_documents_content_updated_token_naming_no_key(): + """A token that does not name the key it was signed with is refused.""" + document = factories.DocumentFactory() + token = jwt.encode( + { + "aud": CollaborationServerAuthentication.AUDIENCE, + "exp": datetime.now(tz=tz.utc) + timedelta(seconds=60), + }, + PRIVATE_KEY, + algorithm="RS256", + ) + + response = APIClient().post( + f"/api/v1.0/documents/{document.id!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {token}", + ) + + assert response.status_code == 401 + + +def test_api_documents_content_updated_token_for_another_audience(): + """A token the collaboration server minted for another service should be refused.""" + document = factories.DocumentFactory() + + response = APIClient().post( + f"/api/v1.0/documents/{document.id!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {collaboration_token(aud='somewhere-else')}", + ) + + assert response.status_code == 401 + + +def test_api_documents_content_updated_expired_token(): + """An expired token should be refused, they are short-lived on purpose.""" + document = factories.DocumentFactory() + expired = datetime.now(tz=tz.utc) - timedelta(seconds=60) + + response = APIClient().post( + f"/api/v1.0/documents/{document.id!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {collaboration_token(exp=expired)}", + ) + + assert response.status_code == 401 + + +def test_api_documents_content_updated_collaboration_server_not_configured(settings): + """Without a collaboration server to read the keys from, nothing is authenticated.""" + settings.YHUB_API_BASE_URL = None + document = factories.DocumentFactory() + + response = APIClient().post( + f"/api/v1.0/documents/{document.id!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {collaboration_token()}", + ) + + assert response.status_code == 401 + + +def test_api_documents_content_updated_jwks_unavailable(yhub_jwks): + """A collaboration server that publishes no key authenticates nobody.""" + yhub_jwks.reset() + yhub_jwks.get(JWKS_URL, status=500) + document = factories.DocumentFactory() + + response = APIClient().post( + f"/api/v1.0/documents/{document.id!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {collaboration_token()}", + ) + + assert response.status_code == 401 + + +def test_api_documents_content_updated_rolled_key(yhub_jwks): + """ + A key rolled on the collaboration server should be picked up on its own. + + This is what publishing a JWKS buys over a key pinned in our settings: + the tokens signed with the new key name a key we do not know, and looking + it up fetches the set again. + """ + document = factories.DocumentFactory() + url = f"/api/v1.0/documents/{document.id!s}/content-updated/" + + # a first call caches the keys published so far + response = APIClient().post( + url, HTTP_AUTHORIZATION=f"Bearer {collaboration_token()}" + ) + assert response.status_code == 204 + + new_private_key, new_public_key = generate_key_pair() + yhub_jwks.reset() + yhub_jwks.get(JWKS_URL, json=build_jwks(new_public_key)) + + response = APIClient().post( + url, + HTTP_AUTHORIZATION=( + f"Bearer {collaboration_token(new_private_key, new_public_key)}" + ), + ) + + assert response.status_code == 204 + + +def test_api_documents_content_updated(): + """The collaboration server should be able to refresh the date of a document.""" + with freeze_time("2026-08-01 12:00:00"): + # no content: writing one to S3 under a frozen clock breaks its signature + document = factories.DocumentFactory(title="my document", content="") + + with freeze_time("2026-08-06 12:00:00"): + response = APIClient().post( + f"/api/v1.0/documents/{document.id!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {collaboration_token()}", + ) + + assert response.status_code == 204 + + document.refresh_from_db() + assert document.updated_at == datetime(2026, 8, 6, 12, 0, 0, tzinfo=tz.utc) + # the document itself is left alone + assert document.created_at == datetime(2026, 8, 1, 12, 0, 0, tzinfo=tz.utc) + assert document.title == "my document" + + +def test_api_documents_content_updated_restricted_document(): + """ + The collaboration server acts for whoever is editing, the access of a + document is not its business. + """ + document = factories.DocumentFactory(link_reach="restricted") + + response = APIClient().post( + f"/api/v1.0/documents/{document.id!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {collaboration_token()}", + ) + + assert response.status_code == 204 + + +def test_api_documents_content_updated_unknown_document(): + """A document deleted in the meantime should answer a 404.""" + response = APIClient().post( + f"/api/v1.0/documents/{uuid4()!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {collaboration_token()}", + ) + + assert response.status_code == 404 + assert not Document.objects.exists() + + +def test_api_documents_content_updated_invalid_document_id(): + """A room name that is no document id should answer a 404, not a 500.""" + response = APIClient().post( + "/api/v1.0/documents/not-an-uuid/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {collaboration_token()}", + ) + + assert response.status_code == 404 diff --git a/src/backend/core/tests/test_services_jwks_client.py b/src/backend/core/tests/test_services_jwks_client.py new file mode 100644 index 0000000000..a96566426b --- /dev/null +++ b/src/backend/core/tests/test_services_jwks_client.py @@ -0,0 +1,165 @@ +""" +This module contains tests for the JWKSClient class in the +core.services.jwt_services module. +""" + +from datetime import datetime, timedelta, timezone + +import jwt +import pytest +import responses + +from core.services.jwt_services import JWKSClient, JWKSError +from core.tests.utils.jwt_helper import build_jwks, generate_key_pair, key_id + +# Generating RSA keys is expensive, do it once for the whole module +PRIVATE_KEY, PUBLIC_KEY = generate_key_pair() +OTHER_PRIVATE_KEY, OTHER_PUBLIC_KEY = generate_key_pair() + +JWKS_URL = "http://service.example.com/jwks" + + +def signed_token(private_key=PRIVATE_KEY, public_key=PUBLIC_KEY, headers=None): + """ + Sign a token the way a service publishing a JWKS does. + + Unless the caller wants something else in there, the header names the key + the token can be validated with. + """ + return jwt.encode( + {"exp": datetime.now(tz=timezone.utc) + timedelta(seconds=60)}, + private_key, + algorithm="RS256", + headers={"kid": key_id(public_key)} if headers is None else headers, + ) + + +@pytest.fixture(name="jwks") +def jwks_fixture(): + """Serve the JWKS of the key this module signs its tokens with.""" + with responses.RequestsMock(assert_all_requests_are_fired=False) as mock: + mock.get(JWKS_URL, json=build_jwks(PUBLIC_KEY)) + yield mock + + +@pytest.mark.usefixtures("jwks") +def test_get_signing_key_returns_the_published_key(): + """The key a token names should validate its signature.""" + token = signed_token() + + key = JWKSClient(JWKS_URL).get_signing_key(token) + + assert jwt.decode(token, key.key, algorithms=["RS256"]) + + +def test_the_document_is_fetched_once(jwks): + """Validating tokens should not call the publisher every time.""" + client = JWKSClient(JWKS_URL) + + client.get_signing_key(signed_token()) + client.get_signing_key(signed_token()) + # the document is cached for the url, not for the client instance + JWKSClient(JWKS_URL).get_signing_key(signed_token()) + + assert len(jwks.calls) == 1 + + +def test_an_unknown_key_is_looked_for_in_a_fresh_document(jwks): + """A key published after the document was cached should still be found.""" + client = JWKSClient(JWKS_URL) + client.get_signing_key(signed_token()) + + # the service rolls its key and publishes the new one + jwks.reset() + jwks.get(JWKS_URL, json=build_jwks(OTHER_PUBLIC_KEY)) + token = signed_token(OTHER_PRIVATE_KEY, OTHER_PUBLIC_KEY) + + key = client.get_signing_key(token) + + assert jwt.decode(token, key.key, algorithms=["RS256"]) + assert len(jwks.calls) == 1 # the fetch of the refreshed document + + +@pytest.mark.usefixtures("jwks") +def test_a_key_nobody_published_is_refused(): + """A token can name any key, only a published one validates it.""" + with pytest.raises(JWKSError, match="has no key"): + JWKSClient(JWKS_URL).get_signing_key( + signed_token(headers={"kid": "a-key-nobody-published"}) + ) + + +def test_an_unknown_key_only_refreshes_once_per_cooldown(jwks): + """ + 🔒 A forged "kid" must not turn every request into a call to the publisher. + + Nothing authenticates the key a token names, so without a cooldown an + unauthenticated caller would have us fetch the document as often as it asks. + """ + client = JWKSClient(JWKS_URL) + + for _ in range(5): + with pytest.raises(JWKSError): + client.get_signing_key( + signed_token(headers={"kid": "a-key-nobody-published"}) + ) + + # the first call fills the cache, the second is the refresh of the window + assert len(jwks.calls) == 2 + + +@pytest.mark.usefixtures("jwks") +def test_a_token_naming_no_key_is_refused(): + """A token that does not name its key cannot be matched to one.""" + with pytest.raises(JWKSError, match="does not name"): + JWKSClient(JWKS_URL).get_signing_key(signed_token(headers={})) + + +@pytest.mark.usefixtures("jwks") +def test_a_token_that_is_not_a_token_is_refused(): + """A header we cannot even read is reported like any unusable token.""" + with pytest.raises(JWKSError, match="does not name"): + JWKSClient(JWKS_URL).get_signing_key("not-a-token") + + +@responses.activate +def test_an_unreachable_publisher_is_reported(): + """A service we cannot fetch the keys from validates no token.""" + responses.get(JWKS_URL, status=500) + + with pytest.raises(JWKSError, match="Unable to fetch"): + JWKSClient(JWKS_URL).get_signing_key(signed_token()) + + +@responses.activate +def test_a_malformed_document_is_reported(): + """A published document we cannot import validates no token either.""" + responses.get(JWKS_URL, body="not a jwks") + + with pytest.raises(JWKSError, match="cannot be imported"): + JWKSClient(JWKS_URL).get_signing_key(signed_token()) + + +@responses.activate +def test_a_document_without_any_usable_key_is_reported(): + """A publisher answering an empty set is not a publisher we can use.""" + responses.get(JWKS_URL, json={"keys": []}) + + with pytest.raises(JWKSError, match="cannot be imported"): + JWKSClient(JWKS_URL).get_signing_key(signed_token()) + + +@responses.activate +def test_the_documents_of_two_services_do_not_share_a_cache_entry(): + """Two publishers are two documents, whatever each of them holds.""" + other_url = "http://other-service.example.com/jwks" + responses.get(JWKS_URL, json=build_jwks(PUBLIC_KEY)) + responses.get(other_url, json=build_jwks(OTHER_PUBLIC_KEY)) + + JWKSClient(JWKS_URL).get_signing_key(signed_token()) + key = JWKSClient(other_url).get_signing_key( + signed_token(OTHER_PRIVATE_KEY, OTHER_PUBLIC_KEY) + ) + + assert key.key_id == key_id(OTHER_PUBLIC_KEY) + assert len(responses.calls) == 2 diff --git a/src/backend/core/tests/test_services_yhub_services.py b/src/backend/core/tests/test_services_yhub_services.py index a9dfaf7c45..08d08bf57c 100644 --- a/src/backend/core/tests/test_services_yhub_services.py +++ b/src/backend/core/tests/test_services_yhub_services.py @@ -60,6 +60,14 @@ def test_build_url(): assert url == f"http://yhub:3002/collaboration/ydoc/v1/docs/{DOCUMENT.id!s}" +def test_jwks_url(): + """The keys validating what yhub signs should be read from yhub itself.""" + service = YHubService() + + assert service.jwks_url == "http://yhub:3002/collaboration/jwks/v1" + assert service.jwks.url == service.jwks_url + + def test_auth_header(): """The auth header should carry an admin JWT signed with the configured key.""" scheme, token = YHubService().auth_header.split(" ") diff --git a/src/backend/core/tests/utils/jwt_helper.py b/src/backend/core/tests/utils/jwt_helper.py index f97bb18ea2..139e6e0c1b 100644 --- a/src/backend/core/tests/utils/jwt_helper.py +++ b/src/backend/core/tests/utils/jwt_helper.py @@ -2,6 +2,7 @@ from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa +from joserfc.jwk import KeySet, RSAKey def generate_key_pair(): @@ -21,3 +22,24 @@ def generate_key_pair(): .decode("utf-8") ) return private_pem, public_pem + + +def key_id(public_pem): + """ + Return the "kid" naming a key, the way every service here names its own. + + It is the RFC 7638 thumbprint of the key, computed from its public + components: the signer stamps it in the header of its tokens and publishes + it in its JWKS, which is how the two are matched. + """ + return RSAKey.import_key(public_pem).thumbprint() + + +def build_jwks(public_pem): + """Build the JWKS a service publishes for a PEM encoded RSA public key.""" + key = RSAKey.import_key( + public_pem, + parameters={"alg": "RS256", "use": "sig", "kid": key_id(public_pem)}, + ) + + return KeySet([key]).as_dict(private=False) diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index c87929235d..0d517869f1 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -39,14 +39,30 @@ It is not a fork of yhub — it is a thin wrapper: - exposes `POST /collaboration/migrate/v1/{org}/{docid}`, which replays a document's **full** legacy version history out of the S3 media bucket (see "Full migration" below) — admin JWT only, like `reset-connections`, +- exposes `GET /collaboration/get-ydoc/v1/{org}/{docid}`, the read counterpart + of `create-ydoc`: the current state of a document as a raw binary update + (204 when it has no content), which the Django backend reads to export or + duplicate a document. Guarded by standard document read access, +- notifies the Django backend on + `POST /api/v1.0/documents/{id}/content-updated/` whenever the worker + persists new content for a document, so that lists ordered by `updated_at` + follow the edits made here. Signed with an RS256 JWT of our own + (`YHUB_JWT_PRIVATE_KEY`, `aud: "docs-backend"`, one minute), best effort: a + notification the backend refuses or never receives is logged and dropped, +- publishes the public half of that key on `GET /collaboration/jwks/v1`, where + the backend reads it. The exact mirror of the JWKS the backend publishes for + its own tokens: neither side is configured with a copy of the key of the + other, so either can roll its key on its own. Served unauthenticated, as any + JWKS is, - mirrors the environment conventions used elsewhere in this repository (`*_FILE` secret indirection, `COLLABORATION_SERVER_ORIGIN` allowlist, …). Public exposure: route the whole `/collaboration/` prefix to this server — the websocket and the built-in document APIs (`ydoc`, `rollback`, `prune`, `changeset`, `activity`) are all guarded by the same cookie-based document -authorization and are meant to be reachable by browsers. The one exception -is `/collaboration/reset-connections/` and `/collaboration/migrate/`, which are +authorization and are meant to be reachable by browsers, as is +`/collaboration/jwks/v1`, which carries public keys and nothing else. The one +exception is `/collaboration/reset-connections/` and `/collaboration/migrate/`, which are backend-internal and should not be routed through the public ingress. ## Container image diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index f2d3e293ad..dbafe6da44 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -1,12 +1,22 @@ -import { createHash } from 'node:crypto'; +import { createHash, createPublicKey, randomUUID } from 'node:crypto'; +import { readFileSync } from 'node:fs'; import { apiError, createApiEndpoint, createAuthPlugin, createYHub, + logger, } from '@y/hub'; -import { createRemoteJWKSet, jwtVerify } from 'jose'; +import { + calculateJwkThumbprint, + createRemoteJWKSet, + exportJWK, + importPKCS8, + jwtVerify, + SignJWT, +} from 'jose'; +import { Client as S3Client } from 'minio'; import { secret } from './env.js'; // legacy Django/S3 document store — see migration.js and README.md @@ -29,6 +39,13 @@ const allowedOrigins = ( ).split(','); const Y_PROVIDER_API_KEY = secret('Y_PROVIDER_API_KEY', 'yprovider-api-key'); const ORG = process.env.YHUB_ORG || 'docs'; +// Segment every route is mounted under (`server.apiPrefix` below), matching the +// URL scheme Docs already routes to the collaboration server. Hardcoded like +// the audiences: the backend builds its urls with the same prefix. +const API_PREFIX = 'collaboration'; +// Path the JWKS endpoint declared in `api` is mounted at. readAuthInfo reads +// the raw request, without any route context, hence the duplication. +const JWKS_PATH = `/${API_PREFIX}/jwks/v1`; // Requiring this audience stops a valid admin JWT that Django issued for // another service (today: the y-converter token in converter_services.py, // which is handed to the converter process) from being replayed against yhub. @@ -51,6 +68,77 @@ const EMPTY_YDOC = new Uint8Array([0, 0]); // websocket path. const MAX_CREATE_BYTES = 10 * 1024 * 1024; +const touchLog = logger.child({ module: 'updated-at-notifier' }); + +const BACKEND_NOTIFY_TIMEOUT_MS = 5000; +// Audience of the tokens the backend accepts from us. It must match the one +// its CollaborationServerAuthentication requires, a token minted for anything +// else is refused there. +const BACKEND_AUDIENCE = 'docs-backend'; +const BACKEND_TOKEN_LIFETIME_S = 60; +// Renew this long before expiry so a token never dies in flight. +const BACKEND_TOKEN_MARGIN_MS = 10000; + +// We sign the calls we make to the backend, the mirror of the admin JWT it +// signs to call us: no long-lived shared secret, only our private key here and +// its public half published on the JWKS endpoint below. +const YHUB_JWT_PRIVATE_KEY = secret('YHUB_JWT_PRIVATE_KEY', ''); +const backendSigningKey = YHUB_JWT_PRIVATE_KEY + ? await importPKCS8(YHUB_JWT_PRIVATE_KEY, 'RS256') + : null; + +if (backendSigningKey == null) { + // not fatal, documents keep being served — only their `updated_at` freezes + touchLog.warn( + 'YHUB_JWT_PRIVATE_KEY is empty, the backend will not be notified of content updates', + ); +} + +// The public half of the signing key, as published on the JWKS endpoint. Its +// "kid" is the RFC 7638 thumbprint of the key: computed from the public +// components only, it is stable across restarts and changes on its own when +// the key is rolled. Every token we sign carries it, which is how the backend +// picks the matching key — and how it knows to fetch the set again when it +// does not know the key yet, so rolling this key needs no change on its side. +const backendPublicJwk = + backendSigningKey == null + ? null + : await (async () => { + // derived from the PEM rather than exported from `backendSigningKey`: + // exporting a private key as a JWK would carry its private components + const jwk = await exportJWK(createPublicKey(YHUB_JWT_PRIVATE_KEY)); + return { + ...jwk, + alg: 'RS256', + use: 'sig', + kid: await calculateJwkThumbprint(jwk), + }; + })(); + +/** + * @type {{ token: string, expiresAt: number } | null} + */ +let backendToken = null; + +// The token carries no per-document claim, so one is reused until it is about +// to expire rather than signing on every notification. +const getBackendToken = async () => { + const now = Date.now(); + if (backendToken != null && backendToken.expiresAt - BACKEND_TOKEN_MARGIN_MS > now) { + return backendToken.token; + } + const token = await new SignJWT({}) + // the "kid" names the key in our JWKS the backend must verify it with + .setProtectedHeader({ alg: 'RS256', kid: backendPublicJwk.kid }) + .setIssuer('yhub') + .setAudience(BACKEND_AUDIENCE) + .setIssuedAt() + .setExpirationTime(`${BACKEND_TOKEN_LIFETIME_S}s`) + .sign(backendSigningKey); + backendToken = { token, expiresAt: now + BACKEND_TOKEN_LIFETIME_S * 1000 }; + return token; +}; + // Public keys verifying the RS256 admin tokens Django issues (JWTService). // Lazily fetched on first use; jose caches the keys and refetches on unknown // "kid", so Django can rotate the signing key without a yhub restart. @@ -110,10 +198,19 @@ const seedFromLegacyStore = async (room) => { const auth = createAuthPlugin({ // uws req is only valid synchronously — read headers AND query before first await. async readAuthInfo(req) { + const url = req.getUrl(); const authorization = req.getHeader('authorization'); const cookie = req.getHeader('cookie'); const origin = req.getHeader('origin'); const gcOff = req.getQuery('gc') === 'false'; + // The JWKS holds public keys and nothing else, and the backend must be + // able to fetch it before it can authenticate anything we send it — so it + // is served to anyone, as the backend serves its own. This identity is + // granted the 'jwks' purpose and nothing else (getGlobalAccessType), and + // the check is on the exact path of that one route. + if (url === JWKS_PATH) { + return { userid: 'anonymous' }; + } if (authorization !== '') { // backend-to-server call: RS256 JWT signed by Django, verified against // its JWKS. A browser cannot attach an Authorization header to a ws @@ -183,6 +280,10 @@ const auth = createAuthPlugin({ return { userid: `anon:${anon}`, cookie, origin }; } }, + // Authorizes the global-scoped endpoints, of which the JWKS is the only one. + async getGlobalAccessType(authInfo, purpose) { + return purpose === 'jwks' ? 'r' : null; + }, async getAccessType(authInfo, { org, docid, branch }, purpose) { if (authInfo.admin === true) { // Django's admin token: full access. It still goes through the legacy @@ -254,6 +355,24 @@ const jsonResponse = (status, body) => }); const api = [ + // GET /collaboration/jwks/v1 — the public keys verifying the tokens we sign + // to call the backend, in the JSON Web Key Set format (RFC 7517). Global + // scope: it is about this server, not about a document, so the route carries + // no org and no docid. The counterpart of the backend's own /api/v1.0/jwks, + // which we read above to verify its tokens: neither side stores a copy of + // the other's key, so either can be rolled without the other being changed. + createApiEndpoint('jwks', { + scope: 'global', + accessPurpose: 'jwks', + get: { + // an empty set when no key is configured: honest, and the backend + // refuses our (equally absent) tokens rather than trusting anything + handler: () => + jsonResponse(200, { + keys: backendPublicJwk == null ? [] : [backendPublicJwk], + }), + }, + }), // POST /collaboration/reset-connections/v1/{org}/{docid} — replaces // y-provider's /collaboration/api/reset-connections/?room=. Doc-scoped, so // the room comes from the path; access is gated to the admin token via the @@ -475,8 +594,46 @@ const api = [ }), ]; -// referenced by getAccessType above — safe: auth callbacks only fire once the -// server is up, i.e. after this assignment +// Django orders the document lists by `updated_at` and no edit goes through it +// anymore, so it is told here that a document moved on. +const touchDocument = async (docid) => { + if (backendSigningKey == null) return; + try { + const res = await fetch( + `${COLLABORATION_BACKEND_BASE_URL}/api/v1.0/documents/${docid}/content-updated/`, + { + method: 'POST', + headers: { authorization: `Bearer ${await getBackendToken()}` }, + signal: AbortSignal.timeout(BACKEND_NOTIFY_TIMEOUT_MS), + }, + ); + if (!res.ok) { + touchLog.warn({ docid, status: res.status }, 'backend refused the notification'); + } + } catch (err) { + // best effort: a lost notification only leaves `updated_at` behind until + // the document is edited again, it must never fail a compaction + touchLog.warn({ err, docid }, 'could not notify the backend'); + } +}; + +// `docUpdate` is the worker event for "this compaction found new content": the +// task returns before it when it has nothing to persist, so the awareness-only +// traffic of someone merely opening a document never reaches it. Since yhub +// 0.5.0 it is handed the room of the task alongside the merged document. +const workerEvents = { + docUpdate: ({ room }) => { + // Django knows the documents of this org, on the main branch, by their uuid + if (room.org !== ORG || room.branch !== 'main' || !UUID4.test(room.docid)) { + return; + } + // deliberately not awaited: a slow backend must not hold the worker + touchDocument(room.docid); + }, +}; + +// the instance is referenced by the soft-migration helpers above — safe: auth +// callbacks only fire once the server is up, i.e. after this assignment const yhub = await createYHub({ redis: { url: REDIS, @@ -486,12 +643,8 @@ const yhub = await createYHub({ }, postgres: POSTGRES, persistence: [], // blobs live in yhub's postgres - // apiPrefix mounts every route — built-ins, reset-connections, and the - // websocket (/collaboration/ws/v1/{org}/{docid}) — under /collaboration/, - // matching the URL scheme Docs already routes to the collaboration server. - server: { port: PORT, auth, api, apiPrefix: 'collaboration' }, - worker: { taskConcurrency: 5 }, - // TODO(yhub): worker.events.docUpdate could push snapshots to Django and replace the - // client useSaveDoc PATCH flow. No longer blocked upstream — yhub 0.5.0 adds `room` - // to the event payload, which was the missing piece. + // apiPrefix mounts every route — built-ins, our custom endpoints, and the + // websocket (/collaboration/ws/v1/{org}/{docid}) — under /collaboration/. + server: { port: PORT, auth, api, apiPrefix: API_PREFIX }, + worker: { taskConcurrency: 5, events: workerEvents }, }); From e7982328ebb8355a5b810c736cd884efd42eab6a Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Fri, 7 Aug 2026 11:20:28 +0200 Subject: [PATCH 39/59] =?UTF-8?q?=E2=99=BB=EF=B8=8F(backend)=20index=20the?= =?UTF-8?q?=20content=20of=20a=20document=20from=20updated=5Fcontent=20end?= =?UTF-8?q?point?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the search indexer reads it with `YHubService`, and the indexation of an edited document is triggered by the `content-updated` call the collaboration server makes — nothing else sees the content change anymore. It is queued as a celery task, throttled like the other updates, so no indexation ever runs in the process serving the request. A document whose content cannot be read is left out of the batch rather than indexed empty, which would have erased it from the search backend --- CHANGELOG.md | 8 +++ src/backend/core/api/viewsets.py | 16 ++++- src/backend/core/services/search_indexers.py | 57 ++++++++++++++---- src/backend/core/signals.py | 7 ++- src/backend/core/tasks/search.py | 16 +++-- src/backend/core/tests/commands/test_index.py | 9 ++- src/backend/core/tests/conftest.py | 15 ++++- .../test_api_documents_content_updated.py | 45 +++++++++++++- .../test_services_find_document_indexer.py | 51 +++++++++------- .../tests/test_services_search_indexers.py | 58 +++++++++++++++++-- src/backend/core/utils/yjs.py | 11 +++- 11 files changed, 233 insertions(+), 60 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7872708cad..d2aa03023c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -133,6 +133,14 @@ and this project adheres to endpoint added above is its server-side replacement, backend wiring pending). The get-connections API is dropped for good: its only consumer was the removed can-edit mechanism, so it is not needed anymore +- ♻️(backend) index the content of a document as the collaboration server has + it: the search indexer reads it with `YHubService`, and the indexation of an + edited document is triggered by the `content-updated` call the collaboration + server makes — nothing else sees the content change anymore. It is queued as + a celery task, throttled like the other updates, so no indexation ever runs + in the process serving the request. A document whose content cannot be read + is left out of the batch rather than indexed empty, which would have erased + it from the search backend - 🔥(backend) remove the unused `CollaborationService` - 💥(backend) remove the `documents/{id}/content/` endpoint - 💥(backend) remove the `documents/{id}/can-edit/` endpoint diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index b22ce17e4c..88be692425 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -70,6 +70,7 @@ from core.services.yhub_services import YHubError, YHubService from core.tasks.access import reset_service_connections_in_cascade from core.tasks.mail import send_ask_for_access_mail +from core.tasks.search import trigger_batch_document_indexer from core.utils.analytics import PosthogEventName, posthog_capture from core.utils.paths import filter_descendants from core.utils.treebeard import create_tree_node_with_retry @@ -948,19 +949,28 @@ def content_updated(self, request, *args, **kwargs): it would freeze. The collaboration server calls this once it persisted the changes of a document, at most once per debounce window. - The update is written without going through the model, saving it would - trigger a re-indexing of a content Django did not see change. + The new content is then read back from the collaboration server to + refresh the search index, in a task: this call is on the path of a + worker persisting a document, it only records what happened. + + The update is written without going through the model: saving it would + index the document a second time, through the post_save signal. """ try: document_id = uuid.UUID(kwargs["pk"]) except ValueError as err: raise Http404 from err + updated_at = timezone.now() if not models.Document.objects.filter(pk=document_id).update( - updated_at=timezone.now() + updated_at=updated_at ): raise Http404 + # Throttled like any other change: the collaboration server calls this + # once per debounce window, for as long as a document is being edited. + trigger_batch_document_indexer(document_id, updated_at) + return drf_response.Response(status=status.HTTP_204_NO_CONTENT) @drf.decorators.action(detail=True, methods=["post"]) diff --git a/src/backend/core/services/search_indexers.py b/src/backend/core/services/search_indexers.py index cd00fa872a..af5b8ce6a2 100644 --- a/src/backend/core/services/search_indexers.py +++ b/src/backend/core/services/search_indexers.py @@ -14,9 +14,10 @@ from core import models from core.enums import SearchType +from core.services.yhub_services import YHubError, YHubService from core.utils.dicts import get_value_by_pattern from core.utils.paths import get_ancestor_to_descendants_map -from core.utils.yjs import base64_yjs_to_text +from core.utils.yjs import yjs_to_text logger = logging.getLogger(__name__) @@ -157,11 +158,27 @@ def index(self, queryset=None, batch_size=None): last_id = documents_batch[-1].id accesses_by_document_path = get_batch_accesses_by_users_and_teams(doc_paths) - serialized_batch = [ - self.serialize_document(document, accesses_by_document_path) - for document in documents_batch - if document.content or document.title - ] + serialized_batch = [] + for document in documents_batch: + try: + content = self.get_document_content(document) + except YHubError: + # A document whose content we could not read is left alone: + # pushing it with an empty content would erase what the + # search backend knows of it over a transient failure. + logger.exception( + "Document %s was not indexed, its content could not be " + "read from the collaboration server", + document.pk, + ) + continue + + if content or document.title: + serialized_batch.append( + self.serialize_document( + document, content, accesses_by_document_path + ) + ) if serialized_batch: self.push(serialized_batch) @@ -169,11 +186,27 @@ def index(self, queryset=None, batch_size=None): return count + @staticmethod + def get_document_content(document): + """ + Return the text of a document, as the collaboration server has it. + + The collaboration server owns the content of the documents, so it is + read from there and never from the database. A document it holds no + content for has none, and is indexed on its metadata alone. + """ + update = YHubService().get_ydoc(document) + + return yjs_to_text(update) if update else "" + @abstractmethod - def serialize_document(self, document, accesses): + def serialize_document(self, document, content, accesses): """ Convert a Document instance to a JSON-serializable format for indexing. + The content is passed in rather than read from the document: it is + fetched once, from the collaboration server, by `index`. + Must be implemented by subclasses. """ @@ -308,25 +341,25 @@ def get_title(source): return source["title"] return "" - def serialize_document(self, document, accesses): + def serialize_document(self, document, content, accesses): """ Convert a Document to the JSON format expected by La Suite Find. Args: document (Document): The document instance. + content (str): The text of the document, as read from the + collaboration server. accesses (dict): Mapping of document ID to user/team access. Returns: dict: A JSON-serializable dictionary. """ doc_path = document.path - doc_content = document.content - text_content = base64_yjs_to_text(doc_content) if doc_content else "" return { "id": str(document.id), "title": document.title or "", - "content": text_content, + "content": content, "depth": document.depth, "path": document.path, "numchild": document.numchild, @@ -335,7 +368,7 @@ def serialize_document(self, document, accesses): "users": list(accesses.get(doc_path, {}).get("users", set())), "groups": list(accesses.get(doc_path, {}).get("teams", set())), "reach": document.computed_link_reach, - "size": len(text_content.encode("utf-8")), + "size": len(content.encode("utf-8")), "is_active": not bool(document.ancestors_deleted_at), } diff --git a/src/backend/core/signals.py b/src/backend/core/signals.py index 03faa666b0..5acf5f0507 100644 --- a/src/backend/core/signals.py +++ b/src/backend/core/signals.py @@ -21,7 +21,9 @@ def document_post_save(sender, instance, **kwargs): # pylint: disable=unused-ar Note : Within the transaction we can have an empty content and a serialization error. """ - transaction.on_commit(partial(trigger_batch_document_indexer, instance)) + transaction.on_commit( + partial(trigger_batch_document_indexer, instance.pk, instance.updated_at) + ) @receiver(signals.post_save, sender=models.DocumentAccess) @@ -31,8 +33,9 @@ def document_access_post_save(sender, instance, created, **kwargs): # pylint: d Clear cache for the affected user. """ if not created: + document = instance.document transaction.on_commit( - partial(trigger_batch_document_indexer, instance.document) + partial(trigger_batch_document_indexer, document.pk, document.updated_at) ) # Invalidate cache for the user diff --git a/src/backend/core/tasks/search.py b/src/backend/core/tasks/search.py index e1c39e6bea..d6717bc286 100644 --- a/src/backend/core/tasks/search.py +++ b/src/backend/core/tasks/search.py @@ -63,12 +63,13 @@ def batch_document_indexer_task(timestamp): logger.info("Indexed %d documents", count) -def trigger_batch_document_indexer(document): +def trigger_batch_document_indexer(document_id, updated_at): """ Trigger indexation task with debounce a delay set by the SEARCH_INDEXER_COUNTDOWN setting. Args: - document (Document): The document instance. + document_id (UUID): The id of the document that changed. + updated_at (datetime): When it changed, the horizon of the batch. """ countdown = int(settings.SEARCH_INDEXER_COUNTDOWN) @@ -82,14 +83,17 @@ def trigger_batch_document_indexer(document): if batch_indexer_throttle_acquire(timeout=countdown): logger.info( "Add task for batch document indexation from updated_at=%s in %d seconds", - document.updated_at.isoformat(), + updated_at.isoformat(), countdown, ) batch_document_indexer_task.apply_async( - args=[document.updated_at], countdown=countdown + args=[updated_at], countdown=countdown ) else: - logger.info("Skip task for batch document %s indexation", document.pk) + logger.info("Skip task for batch document %s indexation", document_id) else: - document_indexer_task.apply(args=[document.pk]) + # Indexing reads the content of the document from the collaboration + # server and pushes it to the search backend: never in the process + # asking for it. + document_indexer_task.delay(document_id) diff --git a/src/backend/core/tests/commands/test_index.py b/src/backend/core/tests/commands/test_index.py index 78d3024958..1b4a42e6c8 100644 --- a/src/backend/core/tests/commands/test_index.py +++ b/src/backend/core/tests/commands/test_index.py @@ -12,6 +12,11 @@ from core import factories from core.services.search_indexers import FindDocumentIndexer +from core.utils.yjs import base64_yjs_to_text + +# what the fake collaboration server of the indexer_settings fixture serves +# for a document created with the content of the factory +CONTENT = base64_yjs_to_text(factories.YDOC_HELLO_WORLD_BASE64) @pytest.mark.django_db @@ -46,8 +51,8 @@ def test_index(): assert sorted(push_call_args[0], key=itemgetter("id")) == sorted( [ - indexer.serialize_document(doc, accesses), - indexer.serialize_document(no_title_doc, accesses), + indexer.serialize_document(doc, CONTENT, accesses), + indexer.serialize_document(no_title_doc, CONTENT, accesses), ], key=itemgetter("id"), ) diff --git a/src/backend/core/tests/conftest.py b/src/backend/core/tests/conftest.py index 0af57d9ff7..9e1ada633d 100644 --- a/src/backend/core/tests/conftest.py +++ b/src/backend/core/tests/conftest.py @@ -9,6 +9,7 @@ import responses from core import factories +from core.services.yhub_services import YHubService from core.tests.utils.urls import reload_urls USER = "user" @@ -35,6 +36,11 @@ def mock_user_teams(): def indexer_settings_fixture(settings): """ Setup valid settings for the document indexer. Clear the indexer cache. + + The indexer reads the content of a document from the collaboration server, + which is faked here: it serves what the factories wrote in the database, so + a document built with `content=""` is one the collaboration server holds no + content for. """ # pylint: disable-next=import-outside-toplevel @@ -50,7 +56,14 @@ def indexer_settings_fixture(settings): settings.SEARCH_URL = "http://localhost:8081/api/v1.0/documents/search/" settings.SEARCH_INDEXER_COUNTDOWN = 1 - yield settings + def get_ydoc(_service, document): + """Answer the raw update the collaboration server would serve.""" + return base64.b64decode(document.content) if document.content else None + + with mock.patch.object( + YHubService, "get_ydoc", autospec=True, side_effect=get_ydoc + ): + yield settings # clear cache to prevent issues with other tests get_document_indexer.cache_clear() diff --git a/src/backend/core/tests/documents/test_api_documents_content_updated.py b/src/backend/core/tests/documents/test_api_documents_content_updated.py index c77ea75ab4..fa7b5ac693 100644 --- a/src/backend/core/tests/documents/test_api_documents_content_updated.py +++ b/src/backend/core/tests/documents/test_api_documents_content_updated.py @@ -4,6 +4,7 @@ from datetime import datetime, timedelta from datetime import timezone as tz +from unittest import mock from uuid import uuid4 import jwt @@ -15,7 +16,9 @@ from core import factories from core.authentication import CollaborationServerAuthentication from core.models import Document +from core.services.search_indexers import FindDocumentIndexer from core.tests.utils.jwt_helper import build_jwks, generate_key_pair, key_id +from core.utils.yjs import base64_yjs_to_text pytestmark = pytest.mark.django_db @@ -34,9 +37,9 @@ def yhub_jwks_fixture(settings): """ settings.YHUB_API_BASE_URL = "http://yhub:3002" - with responses.RequestsMock(assert_all_requests_are_fired=False) as mock: - mock.get(JWKS_URL, json=build_jwks(PUBLIC_KEY)) - yield mock + with responses.RequestsMock(assert_all_requests_are_fired=False) as jwks: + jwks.get(JWKS_URL, json=build_jwks(PUBLIC_KEY)) + yield jwks def collaboration_token(private_key=PRIVATE_KEY, public_key=PUBLIC_KEY, **claims): @@ -237,6 +240,42 @@ def test_api_documents_content_updated(): assert document.title == "my document" +@pytest.mark.usefixtures("indexer_settings") +def test_api_documents_content_updated_indexes_the_document(): + """ + The content changed on the collaboration server: the search index follows. + + Nothing else refreshes it anymore, the content does not go through Django. + """ + document = factories.DocumentFactory(title="my document") + + with mock.patch.object(FindDocumentIndexer, "push") as mock_push: + response = APIClient().post( + f"/api/v1.0/documents/{document.id!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {collaboration_token()}", + ) + + assert response.status_code == 204 + # the task reads the content back from the collaboration server + indexed = {doc["id"]: doc for doc in mock_push.call_args[0][0]} + assert indexed[str(document.id)]["content"] == base64_yjs_to_text( + factories.YDOC_HELLO_WORLD_BASE64 + ) + + +@pytest.mark.usefixtures("indexer_settings") +def test_api_documents_content_updated_does_not_index_when_the_document_is_unknown(): + """A document that does not exist is not worth an indexation task.""" + with mock.patch("core.api.viewsets.trigger_batch_document_indexer") as mock_trigger: + response = APIClient().post( + f"/api/v1.0/documents/{uuid4()!s}/content-updated/", + HTTP_AUTHORIZATION=f"Bearer {collaboration_token()}", + ) + + assert response.status_code == 404 + mock_trigger.assert_not_called() + + def test_api_documents_content_updated_restricted_document(): """ The collaboration server acts for whoever is editing, the access of a diff --git a/src/backend/core/tests/test_services_find_document_indexer.py b/src/backend/core/tests/test_services_find_document_indexer.py index 9a080cd7c9..6accd7239f 100644 --- a/src/backend/core/tests/test_services_find_document_indexer.py +++ b/src/backend/core/tests/test_services_find_document_indexer.py @@ -14,9 +14,14 @@ from core import factories, models from core.enums import SearchType from core.services.search_indexers import FindDocumentIndexer +from core.utils.yjs import base64_yjs_to_text pytestmark = pytest.mark.django_db +# The documents of this module all carry the content of the factory, which the +# fake collaboration server of the indexer_settings fixture serves back. +CONTENT = base64_yjs_to_text(factories.YDOC_HELLO_WORLD_BASE64) + def reset_batch_indexer_throttle(): """Reset throttle flag""" @@ -49,9 +54,9 @@ def test_models_documents_post_save_indexer(mock_push): # One call assert sorted(data[0], key=itemgetter("id")) == sorted( [ - indexer.serialize_document(doc1, accesses), - indexer.serialize_document(doc2, accesses), - indexer.serialize_document(doc3, accesses), + indexer.serialize_document(doc1, CONTENT, accesses), + indexer.serialize_document(doc2, CONTENT, accesses), + indexer.serialize_document(doc3, CONTENT, accesses), ], key=itemgetter("id"), ) @@ -81,9 +86,9 @@ def test_models_documents_post_save_indexer_no_batches(indexer_settings): # all documents are indexed assert sorted([d[0] for d in data], key=itemgetter("id")) == sorted( [ - indexer.serialize_document(doc1, accesses), - indexer.serialize_document(doc2, accesses), - indexer.serialize_document(doc3, accesses), + indexer.serialize_document(doc1, CONTENT, accesses), + indexer.serialize_document(doc2, CONTENT, accesses), + indexer.serialize_document(doc3, CONTENT, accesses), ], key=itemgetter("id"), ) @@ -151,9 +156,9 @@ def test_models_documents_post_save_indexer_with_accesses(mock_push): assert len(data) == 1 assert sorted(data[0], key=itemgetter("id")) == sorted( [ - indexer.serialize_document(doc1, accesses), - indexer.serialize_document(doc2, accesses), - indexer.serialize_document(doc3, accesses), + indexer.serialize_document(doc1, CONTENT, accesses), + indexer.serialize_document(doc2, CONTENT, accesses), + indexer.serialize_document(doc3, CONTENT, accesses), ], key=itemgetter("id"), ) @@ -215,9 +220,9 @@ def test_models_documents_post_save_indexer_deleted(mock_push): # First indexation on document creation assert sorted(data[0], key=itemgetter("id")) == sorted( [ - indexer.serialize_document(doc, accesses), - indexer.serialize_document(main_doc, accesses), - indexer.serialize_document(child_doc, accesses), + indexer.serialize_document(doc, CONTENT, accesses), + indexer.serialize_document(main_doc, CONTENT, accesses), + indexer.serialize_document(child_doc, CONTENT, accesses), ], key=itemgetter("id"), ) @@ -225,8 +230,10 @@ def test_models_documents_post_save_indexer_deleted(mock_push): # Even deleted items are re-indexed : only update their status in the future assert sorted(data[1], key=itemgetter("id")) == sorted( [ - indexer.serialize_document(main_doc_deleted, accesses), # soft_delete() - indexer.serialize_document(child_doc_deleted, accesses), + indexer.serialize_document( + main_doc_deleted, CONTENT, accesses + ), # soft_delete() + indexer.serialize_document(child_doc_deleted, CONTENT, accesses), ], key=itemgetter("id"), ) @@ -317,9 +324,9 @@ def test_models_documents_post_save_indexer_restored(mock_push): # First indexation on items creation & soft delete (in the same transaction) assert sorted(data[0], key=itemgetter("id")) == sorted( [ - indexer.serialize_document(doc, accesses), - indexer.serialize_document(doc_deleted, accesses), - indexer.serialize_document(doc_ancestor_deleted, accesses), + indexer.serialize_document(doc, CONTENT, accesses), + indexer.serialize_document(doc_deleted, CONTENT, accesses), + indexer.serialize_document(doc_ancestor_deleted, CONTENT, accesses), ], key=itemgetter("id"), ) @@ -327,8 +334,8 @@ def test_models_documents_post_save_indexer_restored(mock_push): # Restored items are re-indexed : only update their status in the future assert sorted(data[1], key=itemgetter("id")) == sorted( [ - indexer.serialize_document(doc_restored, accesses), # restore() - indexer.serialize_document(doc_ancestor_restored, accesses), + indexer.serialize_document(doc_restored, CONTENT, accesses), # restore() + indexer.serialize_document(doc_ancestor_restored, CONTENT, accesses), ], key=itemgetter("id"), ) @@ -376,9 +383,9 @@ def test_models_documents_post_save_indexer_throttle(): assert sorted(data[0], key=itemgetter("id")) == sorted( [ - indexer.serialize_document(docs[0], accesses), - indexer.serialize_document(docs[2], accesses), - indexer.serialize_document(docs[3], accesses), + indexer.serialize_document(docs[0], CONTENT, accesses), + indexer.serialize_document(docs[2], CONTENT, accesses), + indexer.serialize_document(docs[3], CONTENT, accesses), ], key=itemgetter("id"), ) diff --git a/src/backend/core/tests/test_services_search_indexers.py b/src/backend/core/tests/test_services_search_indexers.py index a48d2c8660..2dcc972d1b 100644 --- a/src/backend/core/tests/test_services_search_indexers.py +++ b/src/backend/core/tests/test_services_search_indexers.py @@ -1,5 +1,6 @@ """Tests for Documents search indexers""" +from base64 import b64decode from functools import partial from json import dumps as json_dumps from unittest.mock import patch @@ -19,6 +20,7 @@ get_document_indexer, get_visited_document_ids_of, ) +from core.services.yhub_services import ServiceUnavailableError, YHubService from core.utils.yjs import base64_yjs_to_text pytestmark = pytest.mark.django_db @@ -27,7 +29,7 @@ class FakeDocumentIndexer(BaseDocumentIndexer): """Fake indexer for test purpose""" - def serialize_document(self, document, accesses): + def serialize_document(self, document, content, accesses): return {} def push(self, data): @@ -190,7 +192,9 @@ def test_services_search_indexers_serialize_document_returns_expected_json(): } indexer = FindDocumentIndexer() - result = indexer.serialize_document(document, accesses) + # the content is read from the collaboration server and passed in, the + # serialization never reaches for it itself + result = indexer.serialize_document(document, "Hello world", accesses) assert set(result.pop("users")) == {str(user_a.sub), str(user_b.sub)} assert set(result.pop("groups")) == {"team1", "team2"} @@ -200,11 +204,11 @@ def test_services_search_indexers_serialize_document_returns_expected_json(): "depth": 1, "path": document.path, "numchild": 1, - "content": base64_yjs_to_text(document.content), + "content": "Hello world", "created_at": document.created_at.isoformat(), "updated_at": document.updated_at.isoformat(), "reach": document.link_reach, - "size": 13, + "size": 11, "is_active": True, } @@ -219,7 +223,7 @@ def test_services_search_indexers_serialize_document_deleted(): document.refresh_from_db() indexer = FindDocumentIndexer() - result = indexer.serialize_document(document, {}) + result = indexer.serialize_document(document, "", {}) assert result["is_active"] is False @@ -230,7 +234,7 @@ def test_services_search_indexers_serialize_document_empty(): document = factories.DocumentFactory(content="", title=None) indexer = FindDocumentIndexer() - result = indexer.serialize_document(document, {}) + result = indexer.serialize_document(document, "", {}) assert result["content"] == "" assert result["title"] == "" @@ -334,6 +338,48 @@ def test_services_search_indexers_batch_size_argument(mock_push): assert seen_doc_ids == {str(d.id) for d in documents} +@patch.object(FindDocumentIndexer, "push") +@pytest.mark.usefixtures("indexer_settings") +def test_services_search_indexers_index_the_content_of_the_collaboration_server( + mock_push, +): + """The indexed content is the one the collaboration server holds.""" + document = factories.DocumentFactory() + + assert FindDocumentIndexer().index() == 1 + + indexed = mock_push.call_args[0][0][0] + assert indexed["id"] == str(document.id) + assert indexed["content"] == base64_yjs_to_text(factories.YDOC_HELLO_WORLD_BASE64) + + +@patch.object(FindDocumentIndexer, "push") +@pytest.mark.usefixtures("indexer_settings") +def test_services_search_indexers_skip_documents_the_content_of_which_is_unreadable( + mock_push, +): + """ + A document whose content cannot be read is left out of the batch. + + Indexing it with an empty content would erase what the search backend knows + of it, on nothing more than a collaboration server hiccup. + """ + unreadable, readable = factories.DocumentFactory.create_batch(2) + + def get_ydoc(_service, document): + if document.pk == unreadable.pk: + raise ServiceUnavailableError("yhub is unreachable") + return b64decode(document.content) + + # a plain replacement: the indexer_settings fixture already serves the + # content of the documents, this test needs one of them to fail + with patch.object(YHubService, "get_ydoc", get_ydoc): + assert FindDocumentIndexer().index() == 1 + + results = {doc["id"] for doc in mock_push.call_args[0][0]} + assert results == {str(readable.id)} + + @patch.object(FindDocumentIndexer, "push") @pytest.mark.usefixtures("indexer_settings") def test_services_search_indexers_ignore_empty_documents(mock_push): diff --git a/src/backend/core/utils/yjs.py b/src/backend/core/utils/yjs.py index 7856d0636c..d906509551 100644 --- a/src/backend/core/utils/yjs.py +++ b/src/backend/core/utils/yjs.py @@ -23,12 +23,17 @@ def base64_yjs_to_xml(base64_string): return yjs_to_xml(base64.b64decode(base64_string)) +def yjs_to_text(update): + """Extract text from a raw yjs update.""" + + soup = BeautifulSoup(yjs_to_xml(update), "lxml-xml") + return soup.get_text(separator=" ", strip=True) + + def base64_yjs_to_text(base64_string): """Extract text from base64 yjs document.""" - blocknote_structure = base64_yjs_to_xml(base64_string) - soup = BeautifulSoup(blocknote_structure, "lxml-xml") - return soup.get_text(separator=" ", strip=True) + return yjs_to_text(base64.b64decode(base64_string)) def extract_attachments(content): From 12af5fa60bda98a9e60c7c5fab3500dd7030ca0e Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Fri, 7 Aug 2026 11:43:15 +0200 Subject: [PATCH 40/59] =?UTF-8?q?=E2=99=BB=EF=B8=8F(backend)=20duplicate?= =?UTF-8?q?=20the=20onboarding=20sandbox=20using=20YHub=20service?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Duplicate the onboarding sandbox document through the collaboration server: its content is read from there and copied under the identity of the user the sandbox is created for. A collaboration server that cannot be reached skips the sandbox, as a missing template already did, and never fails the signup --- CHANGELOG.md | 5 + src/backend/core/models.py | 44 ++++++-- src/backend/core/tests/test_models_users.py | 111 ++++++++++++++++++++ 3 files changed, 149 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2aa03023c..712c7afb35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -141,6 +141,11 @@ and this project adheres to in the process serving the request. A document whose content cannot be read is left out of the batch rather than indexed empty, which would have erased it from the search backend +- ♻️(backend) duplicate the onboarding sandbox document through the + collaboration server: its content is read from there and copied under the + identity of the user the sandbox is created for. A collaboration server that + cannot be reached skips the sandbox, as a missing template already did, and + never fails the signup - 🔥(backend) remove the unused `CollaborationService` - 💥(backend) remove the `documents/{id}/content/` endpoint - 💥(backend) remove the `documents/{id}/can-edit/` endpoint diff --git a/src/backend/core/models.py b/src/backend/core/models.py index 299cd2437d..1b890f6190 100644 --- a/src/backend/core/models.py +++ b/src/backend/core/models.py @@ -40,6 +40,7 @@ RoleChoices, get_equivalent_link_definition, ) +from core.services.yhub_services import YHubError, YHubService from core.utils.treebeard import create_tree_node_with_retry from core.validators import sub_validator @@ -314,6 +315,10 @@ def _duplicate_onboarding_sandbox_document(self): """ If the user is new and there is a sandbox document configured, duplicate the sandbox document for the user + + The content of the template is read from the collaboration server, which + owns it, and seeded into the copy under the identity of the user: the + sandbox is theirs from its very first revision. """ if settings.USER_ONBOARDING_SANDBOX_DOCUMENT: sandbox_id = settings.USER_ONBOARDING_SANDBOX_DOCUMENT @@ -325,19 +330,36 @@ def _duplicate_onboarding_sandbox_document(self): sandbox_id, ) return - with transaction.atomic(): - sandbox_document = create_tree_node_with_retry( - lambda: Document.add_root( - title=template_document.title, - content=template_document.content, - attachments=template_document.attachments, - duplicated_from=template_document, - creator=self, + + service = YHubService(user=self) + try: + # a template the collaboration server holds nothing for is an + # empty template: the sandbox is created, empty as well + ydoc_update = service.get_ydoc(template_document) + + with transaction.atomic(): + sandbox_document = create_tree_node_with_retry( + lambda: Document.add_root( + title=template_document.title, + attachments=template_document.attachments, + duplicated_from=template_document, + creator=self, + ) ) - ) - DocumentAccess.objects.create( - user=self, document=sandbox_document, role=RoleChoices.OWNER + DocumentAccess.objects.create( + user=self, document=sandbox_document, role=RoleChoices.OWNER + ) + + if ydoc_update: + service.create_ydoc(sandbox_document, ydoc_update) + except YHubError: + # Onboarding is not worth failing a signup for, and a sandbox + # the content of which could not be copied is not one we want + # to leave behind: the transaction takes it back. + logger.exception( + "Onboarding sandbox document with id %s could not be copied. Skipping.", + sandbox_id, ) def _convert_valid_invitations(self): diff --git a/src/backend/core/tests/test_models_users.py b/src/backend/core/tests/test_models_users.py index 2cf0d50d3b..196aa8681c 100644 --- a/src/backend/core/tests/test_models_users.py +++ b/src/backend/core/tests/test_models_users.py @@ -13,9 +13,33 @@ import pytest from core import factories, models +from core.services.yhub_services import ServiceUnavailableError, YHubService pytestmark = pytest.mark.django_db +# what the collaboration server serves for the onboarding template +TEMPLATE_UPDATE = b"\x01\x02the content of the template" + + +@pytest.fixture(name="collaboration_server", autouse=True) +def collaboration_server_fixture(): + """ + Serve the content of the onboarding template, and take the copies. + + The sandbox is duplicated through the collaboration server, which owns the + content of the documents; every test creating a user goes through it as + soon as USER_ONBOARDING_SANDBOX_DOCUMENT is set. + """ + with ( + patch.object( + YHubService, "get_ydoc", autospec=True, return_value=TEMPLATE_UPDATE + ) as mock_get_ydoc, + patch.object(YHubService, "create_ydoc", autospec=True) as mock_create_ydoc, + ): + # autospec, so the calls carry the service itself: who it acts for is + # what the collaboration server attributes the content to + yield mock_get_ydoc, mock_create_ydoc + def test_models_users_str(): """The str representation should be the email.""" @@ -283,6 +307,93 @@ def test_models_users_duplicate_onboarding_sandbox_document_with_invalid_templat assert sandbox_docs.count() == 0 +def test_models_users_duplicate_onboarding_sandbox_document_copies_the_content( + collaboration_server, +): + """ + The content of the sandbox is the one the collaboration server holds for + the template, copied under the identity of the user it is created for. + """ + mock_get_ydoc, mock_create_ydoc = collaboration_server + template_document = factories.DocumentFactory(title="Getting started with Docs") + + with override_settings(USER_ONBOARDING_SANDBOX_DOCUMENT=str(template_document.id)): + user = factories.UserFactory() + + sandbox_document = models.Document.objects.get( + creator=user, title="Getting started with Docs" + ) + + # read from the template, written to the sandbox + _service, read_document = mock_get_ydoc.call_args[0] + service, written_document, update = mock_create_ydoc.call_args[0] + + assert read_document.id == template_document.id + assert written_document.id == sandbox_document.id + assert update == TEMPLATE_UPDATE + # the service acts for the new user: the content is attributed to them, and + # not to the backend itself + assert service.user == user + + +def test_models_users_duplicate_onboarding_sandbox_document_empty_template( + collaboration_server, +): + """A template the collaboration server holds no content for yields an empty sandbox.""" + mock_get_ydoc, mock_create_ydoc = collaboration_server + mock_get_ydoc.return_value = None + template_document = factories.DocumentFactory(title="Getting started with Docs") + + with override_settings(USER_ONBOARDING_SANDBOX_DOCUMENT=str(template_document.id)): + user = factories.UserFactory() + + assert models.Document.objects.filter( + creator=user, title="Getting started with Docs" + ).exists() + mock_create_ydoc.assert_not_called() + + +def test_models_users_duplicate_onboarding_sandbox_document_unreadable_template( + collaboration_server, +): + """ + A signup is not failed over a collaboration server that cannot be reached. + + The sandbox is skipped instead, as it is when its template does not exist. + """ + mock_get_ydoc, _mock_create_ydoc = collaboration_server + mock_get_ydoc.side_effect = ServiceUnavailableError("yhub is unreachable") + template_document = factories.DocumentFactory(title="Getting started with Docs") + + with override_settings(USER_ONBOARDING_SANDBOX_DOCUMENT=str(template_document.id)): + user = factories.UserFactory() + + assert user.pk is not None + assert not models.Document.objects.filter(creator=user).exists() + assert not models.DocumentAccess.objects.filter(user=user).exists() + + +def test_models_users_duplicate_onboarding_sandbox_document_content_not_copied( + collaboration_server, +): + """ + A sandbox whose content could not be copied is not left behind. + + An empty document titled after the template would be more confusing than + no document at all. + """ + _mock_get_ydoc, mock_create_ydoc = collaboration_server + mock_create_ydoc.side_effect = ServiceUnavailableError("yhub is unreachable") + template_document = factories.DocumentFactory(title="Getting started with Docs") + + with override_settings(USER_ONBOARDING_SANDBOX_DOCUMENT=str(template_document.id)): + user = factories.UserFactory() + + assert user.pk is not None + assert not models.Document.objects.filter(creator=user).exists() + assert not models.DocumentAccess.objects.filter(user=user).exists() + + def test_models_users_duplicate_onboarding_sandbox_document_creates_unique_sandbox_per_user(): """ Each new user should get their own independent sandbox document. From bd4c6c01b7e273910da805f4d5dea8961c28b91d Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Mon, 10 Aug 2026 10:54:33 +0200 Subject: [PATCH 41/59] =?UTF-8?q?=E2=99=BB=EF=B8=8F(backend)=20take=20adav?= =?UTF-8?q?antage=20of=20yhub=200.5.0=20json=20encoding=20returns?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The version 0.5.0 can manage response format by using accept and content-type headers. In python we can't use for now the lib0 decoder so we have to use the json format. When the lib0 decoder will be available in pycrdt we will use it. So we can now use directly the /ydoc api to fetch a document content instead the custom api made for this. --- CHANGELOG.md | 6 +- src/backend/core/services/yhub_services.py | 74 +++++++++++++++---- .../core/tests/test_services_yhub_services.py | 58 +++++++++++++-- 3 files changed, 115 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 712c7afb35..198966af03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -98,7 +98,6 @@ and this project adheres to on its own - 🔧(dev) generate the JWT signing key of the collaboration server when bootstrapping the dev stack, alongside the backend one -- ✨(collaboration) add a get-ydoc endpoint on yhub - ✨(backend) serve `documents/{id}/formatted-content/` from yhub - ✨(backend) duplicate a document through the collaboration server - ✨(backend) call YHubService to seed initial document content @@ -146,6 +145,11 @@ and this project adheres to identity of the user the sandbox is created for. A collaboration server that cannot be reached skips the sandbox, as a missing template already did, and never fails the signup +- ♻️(backend) read the content of a document from the built-in `ydoc` endpoint + of the collaboration server: `YHubService` asks for JSON, which yhub speaks + since 0.5.0, so the custom `get-ydoc` endpoint it used to need is gone. + `create-ydoc` stays, no built-in offers what it does — a strict create, and + content credited to the user rather than to the backend - 🔥(backend) remove the unused `CollaborationService` - 💥(backend) remove the `documents/{id}/content/` endpoint - 💥(backend) remove the `documents/{id}/can-edit/` endpoint diff --git a/src/backend/core/services/yhub_services.py b/src/backend/core/services/yhub_services.py index fed8c3ce5d..c67758f10c 100644 --- a/src/backend/core/services/yhub_services.py +++ b/src/backend/core/services/yhub_services.py @@ -13,6 +13,12 @@ accepts a `branch` query parameter, but our auth plugin only ever grants access to the `main` branch, so this service never sends it. +Since yhub 0.5.0 those endpoints answer JSON to a request asking for it, with +the binary fields base64 encoded, so this service sends `Accept: +application/json` and reads them without a lib0 decoder. Their errors come back +the same way, as a JSON `{"error": ...}` this service reports along with the +status. + A few routes are about the server itself rather than about a document, and carry no room: `/{prefix}/jwks/{version}` publishes the public keys validating the tokens yhub signs to call us back. @@ -21,6 +27,7 @@ need them. """ +import base64 import logging from django.conf import settings @@ -72,6 +79,11 @@ class YHubService: # Version of the endpoints we call, the one all the built-ins are at. api_version = "v1" + # A Yjs update carrying no content encodes to 2 bytes, and yhub reads + # anything up to 3 as an empty document. `ydoc` answers the encoding of an + # empty document for a room it holds nothing for, never an empty body. + empty_update_max_bytes = 3 + def __init__(self, user=None): """Bind the service to the user a call is made on behalf of, if any.""" self.user = user @@ -173,7 +185,7 @@ def build_user_header(user_id): def request(self, method, url, data=None, headers=None): """ - Send an authenticated request to the yhub API. + Send an authenticated request to the yhub API, asking it for JSON. Return the raw response, it is up to the caller to decode its body: the endpoints do not all answer with the same payload. @@ -185,7 +197,8 @@ def request(self, method, url, data=None, headers=None): data=data, headers={ "Authorization": self.auth_header, - "Content-Type": "application/octet-stream", + # what makes yhub answer JSON rather than its lib0 encoding + "Accept": "application/json", **(headers or {}), }, timeout=self.timeout, @@ -203,44 +216,73 @@ def request(self, method, url, data=None, headers=None): response.status_code, response.text[:200] if response.text else "empty", ) + detail = self.json_body(response).get("error") raise APIError( - f"The yhub API answered {response.status_code} on {url}", + f"The yhub API answered {response.status_code} on {url}" + + (f": {detail}" if detail else ""), status_code=response.status_code, ) return response + @staticmethod + def json_body(response): + """ + Return the JSON body of a response, an empty dict when it has none. + + yhub reports its errors as `{"error": ...}` and its endpoints answer + JSON, but a failure can also come from something else on the way (a + proxy, a gateway): what it says is a bonus, never something to fail on. + """ + try: + body = response.json() + except ValueError: + return {} + + return body if isinstance(body, dict) else {} + def get_ydoc(self, document): """ Return the current Yjs state of a document, None when it has none. The raw update is what `create_ydoc` takes, so the state of a document - can be copied into another one. The built-in `ydoc` endpoint is not - used, it answers the lib0 encoding of an envelope rather than the - update itself. + can be copied into another one. The built-in `ydoc` endpoint answers + `{"doc": ...}`, the update base64 encoded, and the encoding of an empty + document for a room it holds no content for. """ - response = self.request("get", self.build_url("get-ydoc", document)) + response = self.request("get", self.build_url("ydoc", document)) - return response.content or None + try: + update = base64.b64decode(self.json_body(response)["doc"]) + except (KeyError, TypeError, ValueError) as err: + raise APIError( + f"The yhub API answered no readable document on {response.url}" + ) from err + + return update if len(update) > self.empty_update_max_bytes else None def create_ydoc(self, document, update): """ Seed the initial Yjs state of a document. The body is the raw binary update, what pycrdt's `get_update()` - returns, and not the lib0 encoding the built-in `ydoc` endpoint speaks. - The content is attributed to the user the service is bound to, yhub - only takes our word for it because the token grants admin. - - It is a strict create: yhub answers 409 when the document already has - content, 413 over 10MB and 400 on an update it cannot apply, all - reported as an `APIError` carrying the status. + returns. This is not the built-in `ydoc` endpoint, which would take the + same update base64 encoded but knows nothing of the two things this one + is for: it is a strict create, and it attributes the content to the + user the service is bound to rather than to the backend calling it. + + yhub answers 409 when the document already has content, 413 over 10MB + and 400 on an update it cannot apply, all reported as an `APIError` + carrying the status. """ return self.request( "post", self.build_url("create-ydoc", document), data=update, - headers=self.build_user_header(self.user_id), + headers={ + "Content-Type": "application/octet-stream", + **self.build_user_header(self.user_id), + }, ) def reset_connections(self, document, user_id=None): diff --git a/src/backend/core/tests/test_services_yhub_services.py b/src/backend/core/tests/test_services_yhub_services.py index 08d08bf57c..ffb90cc412 100644 --- a/src/backend/core/tests/test_services_yhub_services.py +++ b/src/backend/core/tests/test_services_yhub_services.py @@ -1,5 +1,6 @@ """Test yhub services.""" +from base64 import b64encode from unittest.mock import patch from uuid import uuid4 @@ -127,7 +128,8 @@ def test_request(mock_request): assert kwargs["data"] == b"body" assert kwargs["timeout"] == 30 assert kwargs["headers"]["Authorization"].startswith("Bearer ") - assert kwargs["headers"]["Content-Type"] == "application/octet-stream" + # asked of every endpoint: yhub answers its lib0 encoding otherwise + assert kwargs["headers"]["Accept"] == "application/json" @patch("requests.request") @@ -236,23 +238,67 @@ def test_reset_connections_of_a_single_user(mock_request): def test_get_ydoc(mock_request): """Should return the raw update the collaboration server holds.""" mock_request.return_value.ok = True - mock_request.return_value.content = b"\x01\x02raw yjs update" + mock_request.return_value.json.return_value = { + "doc": b64encode(b"\x01\x02raw yjs update").decode() + } update = YHubService().get_ydoc(DOCUMENT) assert update == b"\x01\x02raw yjs update" - args, _kwargs = mock_request.call_args + args, kwargs = mock_request.call_args + # the built-in endpoint, which answers the update base64 encoded in json assert args == ( "get", - f"http://yhub:3002/collaboration/get-ydoc/v1/docs/{DOCUMENT.id!s}", + f"http://yhub:3002/collaboration/ydoc/v1/docs/{DOCUMENT.id!s}", ) + assert kwargs["headers"]["Accept"] == "application/json" @patch("requests.request") def test_get_ydoc_without_content(mock_request): """A document the collaboration server holds no content for should return None.""" mock_request.return_value.ok = True - # yhub answers 204 No Content, hence an empty body - mock_request.return_value.content = b"" + # a room with no content answers the encoding of an empty document + mock_request.return_value.json.return_value = { + "doc": b64encode(b"\x00\x00").decode() + } assert YHubService().get_ydoc(DOCUMENT) is None + + +@patch("requests.request") +def test_get_ydoc_unreadable_answer(mock_request): + """An answer we cannot read a document out of should raise, never look empty.""" + mock_request.return_value.ok = True + mock_request.return_value.json.return_value = {"unexpected": "payload"} + + with pytest.raises(APIError): + YHubService().get_ydoc(DOCUMENT) + + +@patch("requests.request") +def test_request_error_reports_what_yhub_said(mock_request): + """The message yhub puts in its json error should travel with the status.""" + mock_request.return_value.ok = False + mock_request.return_value.status_code = 409 + mock_request.return_value.text = '{"error": "Document already exists"}' + mock_request.return_value.json.return_value = {"error": "Document already exists"} + + with pytest.raises(APIError, match="Document already exists") as excinfo: + YHubService().create_ydoc(DOCUMENT, b"\x01\x02update") + + assert excinfo.value.status_code == 409 + + +@patch("requests.request") +def test_request_error_without_json_body(mock_request): + """An error from something else on the way should be reported all the same.""" + mock_request.return_value.ok = False + mock_request.return_value.status_code = 502 + mock_request.return_value.text = "Bad Gateway" + mock_request.return_value.json.side_effect = ValueError("not json") + + with pytest.raises(APIError, match="answered 502") as excinfo: + YHubService().get_ydoc(DOCUMENT) + + assert excinfo.value.status_code == 502 From b4bc3862325d9f2accffd1d9acb75814865b3b57 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Mon, 10 Aug 2026 10:58:29 +0200 Subject: [PATCH 42/59] =?UTF-8?q?=F0=9F=94=A5(yhub)=20remove=20custom=20en?= =?UTF-8?q?dpoint=20get-ydoc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We don't need anymore the get-ydoc endpoint to fetch a document content since yhub 0.5.0 can manage json encoding. We can safely remove it. --- src/yhub-server/README.md | 17 +++++++------- src/yhub-server/server.js | 47 +++++++++------------------------------ 2 files changed, 18 insertions(+), 46 deletions(-) diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index 0d517869f1..a0653e42ef 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -31,18 +31,17 @@ It is not a fork of yhub — it is a thin wrapper: `X-User-Id` header naming the user the initial content is attributed to), which seeds a document's initial Yjs state from a raw binary update (`Y.encodeStateAsUpdate` / pycrdt `get_update()` output posted as - `application/octet-stream` — no lib0 encoding, unlike yhub's built-in - `PATCH .../ydoc/`), so the Django backend can create documents - server-side. Strict create: 409 when the document already has content. - Guarded by standard document write access (the admin JWT, or a user - session with update ability), + `application/octet-stream`), so the Django backend can create documents + server-side. The built-in `PATCH .../ydoc/` takes the same update, but this + one is a **strict create** — 409 when the document already has content — and + it credits the content to `X-User-Id` instead of to the caller. Guarded by + standard document write access (the admin JWT, or a user session with update + ability). Reading needs neither, and goes through the built-in `GET + .../ydoc/`, which since 0.5.0 answers JSON (the update base64 encoded) to a + request sending `Accept: application/json`, - exposes `POST /collaboration/migrate/v1/{org}/{docid}`, which replays a document's **full** legacy version history out of the S3 media bucket (see "Full migration" below) — admin JWT only, like `reset-connections`, -- exposes `GET /collaboration/get-ydoc/v1/{org}/{docid}`, the read counterpart - of `create-ydoc`: the current state of a document as a raw binary update - (204 when it has no content), which the Django backend reads to export or - duplicate a document. Guarded by standard document read access, - notifies the Django backend on `POST /api/v1.0/documents/{id}/content-updated/` whenever the worker persists new content for a document, so that lists ordered by `updated_at` diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index dbafe6da44..3c952b3871 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -459,45 +459,18 @@ const api = [ }, }, }), - // GET /collaboration/get-ydoc/v1/{org}/{docid} — the current state of a - // document as a RAW binary update (`Y.encodeStateAsUpdate` output), the read - // counterpart of create-ydoc: yhub's built-in `GET ydoc` answers a lib0-any - // encoded `{ doc, awareness }` envelope Django cannot decode. Answers 204 - // when the room holds no content. Default access purpose: guarded like the - // built-in ydoc routes (read access on the doc — the admin JWT, or a user - // session able to retrieve it). - createApiEndpoint('get-ydoc', { - get: { - handler: async (req) => { - if (req.org !== ORG) { - return jsonResponse(400, { error: 'Unknown org' }); - } - if (!UUID4.test(req.docid)) { - return jsonResponse(400, { error: 'Room name is invalid' }); - } - const { gcDoc } = await req.yhub.getDoc( - req.room, - { gc: true, nongc: false }, - { gcOnMerge: false }, - ); - // <= 3 bytes is yhub's "no effective content" convention (an empty - // update encodes to 2 bytes) — nothing to copy. `null` answers 204. - if (gcDoc == null || gcDoc.byteLength <= 3) { - return null; - } - // a Uint8Array is served as application/octet-stream, untouched - return gcDoc; - }, - }, - }), // POST /collaboration/create-ydoc/v1/{org}/{docid} — create a document's // initial Yjs state from a RAW binary update (`Y.encodeStateAsUpdate` / - // pycrdt `get_update()` output) posted as application/octet-stream. Unlike - // yhub's built-in `PATCH ydoc`, the body is not lib0-any encoded, so Django - // can call it with a plain `requests.post(url, data=raw_bytes)`. Strict - // create: 409 when the room already has content. Default access purpose: - // guarded like the built-in ydoc routes (write access on the doc — the - // admin JWT, or a user session with update ability). + // pycrdt `get_update()` output) posted as application/octet-stream. + // + // The built-in `PATCH ydoc` takes the same update (base64, in a json body + // since 0.5.0) but neither of the two things this endpoint exists for: it is + // a strict create, answering 409 when the room already has content, and it + // attributes the content to the user named in `X-User-Id` rather than to the + // backend making the call. Reads have no such needs and use the built-in + // `GET ydoc`. Default access purpose: guarded like the built-in ydoc routes + // (write access on the doc — the admin JWT, or a user session with update + // ability). createApiEndpoint('create-ydoc', { post: { handler: async (req) => { From 7376ba6ad4d22321bb4b10df194ae96f195639c2 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Mon, 10 Aug 2026 11:58:29 +0200 Subject: [PATCH 43/59] =?UTF-8?q?=E2=99=BB=EF=B8=8F(backend)=20seed=20the?= =?UTF-8?q?=20content=20of=20the=20demo=20documents=20using=20yhub?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit seed the content of the demo documents in the collaboration server: `create_demo` no longer writes it to the object storage, which nothing reads anymore, and fails with an explicit message when the collaboration server is not running rather than building a corpus of documents that would open empty --- CHANGELOG.md | 5 + .../demo/management/commands/create_demo.py | 112 ++++++++++++++++-- .../demo/tests/test_commands_create_demo.py | 56 ++++++++- 3 files changed, 163 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 198966af03..4cd01346b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -150,6 +150,11 @@ and this project adheres to since 0.5.0, so the custom `get-ydoc` endpoint it used to need is gone. `create-ydoc` stays, no built-in offers what it does — a strict create, and content credited to the user rather than to the backend +- ♻️(backend) seed the content of the demo documents in the collaboration + server: `create_demo` no longer writes it to the object storage, which + nothing reads anymore, and fails with an explicit message when the + collaboration server is not running rather than building a corpus of + documents that would open empty - 🔥(backend) remove the unused `CollaborationService` - 💥(backend) remove the `documents/{id}/content/` endpoint - 💥(backend) remove the `documents/{id}/can-edit/` endpoint diff --git a/src/backend/demo/management/commands/create_demo.py b/src/backend/demo/management/commands/create_demo.py index e216edf945..ace042e2b3 100644 --- a/src/backend/demo/management/commands/create_demo.py +++ b/src/backend/demo/management/commands/create_demo.py @@ -1,12 +1,12 @@ # ruff: noqa: S311, S106 """create_demo management command""" -import base64 import logging import math import random import time from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor, as_completed from uuid import uuid4 from django import db @@ -17,6 +17,7 @@ from faker import Faker from core import models +from core.services.yhub_services import YHubError, YHubService from demo import defaults @@ -31,14 +32,101 @@ def random_true_with_probability(probability): return random.random() < probability -def get_ydoc_for_text(text): - """Return a ydoc from plain text for demo purposes.""" +# The collaboration server is a service of its own: seeding a corpus one +# document at a time would make the demo wait on the network for most of its +# run, so a few seeds are in flight at once. +SEED_CONCURRENCY = 10 + + +def create_block(kind, text, **attributes): + """ + Build a BlockNote block, inside the container the editor addresses it by. + + Every block lives in a `blockContainer` carrying its id and its colors: + that is the structure the editor writes, and the one the exports and the + search indexer read back. + """ + block = pycrdt.XmlElement( + kind, {"textAlignment": "left", **attributes}, [pycrdt.XmlText(text)] + ) + + return pycrdt.XmlElement( + "blockContainer", + {"id": str(uuid4()), "textColor": "default", "backgroundColor": "default"}, + [block], + ) + + +def create_section_blocks(writer): + """Return the blocks of one section: a title, some prose, sometimes a list.""" + blocks = [create_block("heading", writer.sentence(nb_words=4).rstrip("."), level=2)] + blocks += [ + create_block("paragraph", writer.paragraph(nb_sentences=random.randint(3, 8))) + for _ in range(random.randint(1, 3)) + ] + + if random_true_with_probability(0.4): + kind = random.choice(["bulletListItem", "numberedListItem"]) + blocks += [ + create_block(kind, writer.sentence(nb_words=random.randint(4, 10))) + for _ in range(random.randint(2, 5)) + ] + + if random_true_with_probability(0.2): + blocks.append(create_block("quote", writer.sentence(nb_words=12))) + + return blocks + + +def get_ydoc_for_document(title): + """ + Return the raw Yjs update of a document that reads like a real one. + + Faker writes the prose and the sections, in the structure BlockNote stores, + so a demo document is something to render, to export and to index rather + than the one line it used to be. A single language per document: the corpus + is multilingual, the documents are not. + """ + writer = fake[random.choice(fake.locales)] + + blocks = [create_block("heading", title, level=1)] + for _ in range(random.randint(2, 5)): + blocks.extend(create_section_blocks(writer)) + ydoc = pycrdt.Doc() - paragraph = pycrdt.XmlElement("p", {}, [pycrdt.XmlText(text)]) - fragment = pycrdt.XmlFragment([paragraph]) - ydoc["document-store"] = fragment - update = ydoc.get_update() - return base64.b64encode(update).decode("utf-8") + ydoc["document-store"] = pycrdt.XmlFragment( + [pycrdt.XmlElement("blockGroup", {}, blocks)] + ) + + return ydoc.get_update() + + +def seed_contents(stdout, contents): + """ + Seed the content of the demo documents in the collaboration server. + + It owns the content of the documents, so a demo document without content + there is an empty document — the object storage Django used to write it to + is not read by anything anymore. + """ + service = YHubService() + + with ThreadPoolExecutor(max_workers=SEED_CONCURRENCY) as pool: + seeds = { + pool.submit(service.create_ydoc, document, update): document + for document, update in contents + } + for seed in as_completed(seeds): + try: + seed.result() + except YHubError as err: + # nothing to fall back on: the demo would build a corpus of + # empty documents and look like it worked + raise CommandError( + f"Could not seed the content of document {seeds[seed].id}: {err}. " + "Is the collaboration server running?" + ) from err + stdout.write(".", ending="") class BulkQueue: @@ -150,6 +238,7 @@ def create_demo(stdout): users_ids = list(models.User.objects.values_list("id", flat=True)) with Timeit(stdout, "Creating documents"): + contents = [] for i in range(defaults.NB_OBJECTS["docs"]): # pylint: disable=protected-access key = models.Document._int2str(i) # noqa: SLF001 @@ -165,11 +254,16 @@ def create_demo(stdout): if random_true_with_probability(0.5) else random.choice(models.LinkReachChoices.values), ) - document.save_content(get_ydoc_for_text(f"Content for {title:s}")) + contents.append((document, get_ydoc_for_document(title))) queue.push(document) queue.flush() + # after the flush: a room seeded for a document the database ended up + # without would be content nothing points to + with Timeit(stdout, "Seeding document contents"): + seed_contents(stdout, contents) + with Timeit(stdout, "Creating docs accesses"): docs_ids = list(models.Document.objects.values_list("id", flat=True)) for doc_id in docs_ids: diff --git a/src/backend/demo/tests/test_commands_create_demo.py b/src/backend/demo/tests/test_commands_create_demo.py index 5223330136..8c2e585660 100644 --- a/src/backend/demo/tests/test_commands_create_demo.py +++ b/src/backend/demo/tests/test_commands_create_demo.py @@ -3,15 +3,30 @@ from unittest import mock from django.core.management import call_command +from django.core.management.base import CommandError from django.test import override_settings import pytest from core import models +from core.services.yhub_services import ServiceUnavailableError, YHubService +from core.utils.yjs import yjs_to_text, yjs_to_xml pytestmark = pytest.mark.django_db +@pytest.fixture(name="collaboration_server", autouse=True) +def collaboration_server_fixture(): + """ + Take the content of the demo documents, as the collaboration server does. + + It owns the content now, so building the demo corpus calls it once per + document. + """ + with mock.patch.object(YHubService, "create_ydoc") as mock_create_ydoc: + yield mock_create_ydoc + + @mock.patch( "demo.defaults.NB_OBJECTS", { @@ -21,7 +36,7 @@ }, ) @override_settings(DEBUG=True) -def test_commands_create_demo(): +def test_commands_create_demo(collaboration_server): """The create_demo management command should create objects as expected.""" call_command("create_demo") @@ -29,6 +44,27 @@ def test_commands_create_demo(): assert models.Document.objects.count() >= 10 assert models.DocumentAccess.objects.count() > 10 + # every document was seeded with its content in the collaboration server, + # and nothing was written to the object storage + assert collaboration_server.call_count == 10 + seeded = {call.args[0].id for call in collaboration_server.call_args_list} + assert seeded == set(models.Document.objects.values_list("id", flat=True)) + for call in collaboration_server.call_args_list: + document, update = call.args + assert document.content is None + + # the structure BlockNote stores, so the editor opens a real document + xml = yjs_to_xml(update) + assert xml.startswith(" 3 + # a title, as the number BlockNote reads a heading level as + assert " len(document.title) + # assert dev users have doc accesses user = models.User.objects.get(email="impress@impress.world") assert models.DocumentAccess.objects.filter(user=user).exists() @@ -38,3 +74,21 @@ def test_commands_create_demo(): assert models.DocumentAccess.objects.filter(user=user).exists() user = models.User.objects.get(email="user.test@chromium.test") assert models.DocumentAccess.objects.filter(user=user).exists() + + +@mock.patch( + "demo.defaults.NB_OBJECTS", + {"users": 2, "docs": 2, "max_users_per_document": 1}, +) +@override_settings(DEBUG=True) +def test_commands_create_demo_without_collaboration_server(collaboration_server): + """ + A demo of empty documents is not a demo: the command should say what is wrong. + + Nothing else holds the content, so a failure to seed it cannot be shrugged + off as it could when Django still wrote it to its object storage. + """ + collaboration_server.side_effect = ServiceUnavailableError("yhub is unreachable") + + with pytest.raises(CommandError, match="Is the collaboration server running?"): + call_command("create_demo") From da220545075ce24bf895e05cfca784e13c468441 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Mon, 10 Aug 2026 15:07:14 +0200 Subject: [PATCH 44/59] =?UTF-8?q?=E2=99=BB=EF=B8=8F(backend)=20remove=20us?= =?UTF-8?q?age=20of=20s3=20for=20document.content=20in=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tehe DocumentFactory was always creating a content and this content was saved on S3. This leads to the creation of huge amount of content in the S3 storage but not necesseraly used in the tests. In order to keep the refactor to remove the usage of content from document.content but from Yhub service, this content is no more generated. It is kept for part of the code not yet refactor like the versionning feature. --- src/backend/core/factories.py | 13 +++- src/backend/core/tests/commands/test_index.py | 14 +++- src/backend/core/tests/conftest.py | 29 +++++--- .../documents/test_api_document_versions.py | 68 ++++++++++++------- .../documents/test_api_documents_duplicate.py | 39 +++++------ .../test_api_documents_formatted_content.py | 26 ++++--- .../core/tests/test_models_documents.py | 4 +- .../tests/test_services_search_indexers.py | 42 ++++++++---- 8 files changed, 149 insertions(+), 86 deletions(-) diff --git a/src/backend/core/factories.py b/src/backend/core/factories.py index eeefa8f4b7..d918c073f0 100644 --- a/src/backend/core/factories.py +++ b/src/backend/core/factories.py @@ -2,6 +2,8 @@ Core application factories """ +import base64 + from django.conf import settings from django.contrib.auth.hashers import make_password @@ -28,6 +30,12 @@ "dGV4dENvbG9yAXcHZGVmYXVsdCgA9e7y1Q4eD2JhY2tncm91bmRDb2xvcgF3B2RlZmF1bHQA" ) +# The same document as the raw Yjs update the collaboration server serves, which +# is what a test faking `YHubService.get_ydoc` answers with (see the +# `yhub_content` fixture). The base64 above is the legacy object storage format, +# only the tests still about that storage have a use for it. +YDOC_HELLO_WORLD_UPDATE = base64.b64decode(YDOC_HELLO_WORLD_BASE64) + class UserFactory(factory.django.DjangoModelFactory): """A factory to random users for testing purposes.""" @@ -83,7 +91,10 @@ class Meta: title = factory.Sequence(lambda n: f"document{n}") excerpt = factory.Sequence(lambda n: f"excerpt{n}") - content = YDOC_HELLO_WORLD_BASE64 + # No content: the collaboration server holds it, and a document built here + # is one it knows nothing of. A test needing a document with content fakes + # what the collaboration server serves for it (`YHubService.get_ydoc`), and + # only the ones about the legacy object storage itself pass `content=`. creator = factory.SubFactory(UserFactory) deleted_at = None link_reach = factory.fuzzy.FuzzyChoice( diff --git a/src/backend/core/tests/commands/test_index.py b/src/backend/core/tests/commands/test_index.py index 1b4a42e6c8..48be84b4dd 100644 --- a/src/backend/core/tests/commands/test_index.py +++ b/src/backend/core/tests/commands/test_index.py @@ -11,7 +11,9 @@ import pytest from core import factories +from core.factories import YDOC_HELLO_WORLD_UPDATE from core.services.search_indexers import FindDocumentIndexer +from core.services.yhub_services import YHubService from core.utils.yjs import base64_yjs_to_text # what the fake collaboration server of the indexer_settings fixture serves @@ -28,7 +30,7 @@ def test_index(): with transaction.atomic(): doc = factories.DocumentFactory() - empty_doc = factories.DocumentFactory(title=None, content="") + empty_doc = factories.DocumentFactory(title=None) no_title_doc = factories.DocumentFactory(title=None) factories.UserDocumentAccessFactory(document=doc, user=user) @@ -41,7 +43,15 @@ def test_index(): str(no_title_doc.path): {"users": [user.sub]}, } - with mock.patch.object(FindDocumentIndexer, "push") as mock_push: + # the empty document is the one the collaboration server holds no content + # for, and it has no title either: nothing to index + def get_ydoc(document): + return None if document.pk == empty_doc.pk else YDOC_HELLO_WORLD_UPDATE + + with ( + mock.patch.object(FindDocumentIndexer, "push") as mock_push, + mock.patch.object(YHubService, "get_ydoc", side_effect=get_ydoc), + ): call_command("index") push_call_args = [call.args[0] for call in mock_push.call_args_list] diff --git a/src/backend/core/tests/conftest.py b/src/backend/core/tests/conftest.py index 9e1ada633d..b39af92a7a 100644 --- a/src/backend/core/tests/conftest.py +++ b/src/backend/core/tests/conftest.py @@ -32,15 +32,32 @@ def mock_user_teams(): yield mock_teams +@pytest.fixture(name="yhub_content") +def yhub_content_fixture(): + """ + Serve the content of every document, as the collaboration server does. + + It owns the content: a document built by the factories has none in the + database, and what it holds is whatever this fake answers for it. The mock + is yielded, so a test can serve another document (`return_value`), none at + all (`return_value = None`) or a different one per document + (`side_effect`). + """ + with mock.patch.object( + YHubService, "get_ydoc", return_value=factories.YDOC_HELLO_WORLD_UPDATE + ) as mock_get_ydoc: + yield mock_get_ydoc + + @pytest.fixture(name="indexer_settings") def indexer_settings_fixture(settings): """ Setup valid settings for the document indexer. Clear the indexer cache. The indexer reads the content of a document from the collaboration server, - which is faked here: it serves what the factories wrote in the database, so - a document built with `content=""` is one the collaboration server holds no - content for. + which is faked here: it holds the same content for every document, and a + test wanting one without content answers `None` for it (see the + `yhub_content` fixture, this is the same fake). """ # pylint: disable-next=import-outside-toplevel @@ -56,12 +73,8 @@ def indexer_settings_fixture(settings): settings.SEARCH_URL = "http://localhost:8081/api/v1.0/documents/search/" settings.SEARCH_INDEXER_COUNTDOWN = 1 - def get_ydoc(_service, document): - """Answer the raw update the collaboration server would serve.""" - return base64.b64decode(document.content) if document.content else None - with mock.patch.object( - YHubService, "get_ydoc", autospec=True, side_effect=get_ydoc + YHubService, "get_ydoc", return_value=factories.YDOC_HELLO_WORLD_UPDATE ): yield settings diff --git a/src/backend/core/tests/documents/test_api_document_versions.py b/src/backend/core/tests/documents/test_api_document_versions.py index 83b8c7f587..7e5105bb5a 100644 --- a/src/backend/core/tests/documents/test_api_document_versions.py +++ b/src/backend/core/tests/documents/test_api_document_versions.py @@ -9,11 +9,23 @@ from rest_framework.test import APIClient from core import factories, models +from core.factories import YDOC_HELLO_WORLD_BASE64 from core.tests.conftest import TEAM, USER, VIA pytestmark = pytest.mark.django_db +def create_document(**kwargs): + """ + Create a document holding content in the legacy object storage. + + Versions are the versions of that object, so these tests are the ones still + about it: the factories give a document no content anymore, the + collaboration server holds it. + """ + return factories.DocumentFactory(content=YDOC_HELLO_WORLD_BASE64, **kwargs) + + @pytest.mark.parametrize("reach", models.LinkReachChoices.values) @pytest.mark.parametrize("role", models.LinkRoleChoices.values) def test_api_document_versions_list_anonymous(role, reach): @@ -21,7 +33,7 @@ def test_api_document_versions_list_anonymous(role, reach): Anonymous users should not be allowed to list document versions for a document whatever the reach and role. """ - document = factories.DocumentFactory(link_role=role, link_reach=reach) + document = create_document(link_role=role, link_reach=reach) # Accesses and traces for other users should not interfere factories.UserDocumentAccessFactory(document=document) @@ -44,7 +56,7 @@ def test_api_document_versions_list_authenticated_unrelated(reach): client = APIClient() client.force_login(user) - document = factories.DocumentFactory(link_reach=reach) + document = create_document(link_reach=reach) factories.UserDocumentAccessFactory.create_batch(3, document=document) # The versions of another document to which the user is related should not be listed either @@ -70,7 +82,7 @@ def test_api_document_versions_list_authenticated_related_success(via, mock_user client = APIClient() client.force_login(user) - document = factories.DocumentFactory() + document = create_document() if via == USER: models.DocumentAccess.objects.create( document=document, @@ -125,7 +137,7 @@ def test_api_document_versions_list_authenticated_related_pagination( client = APIClient() client.force_login(user) - document = factories.DocumentFactory() + document = create_document() for i in range(3): document.content = f"before {i:d}" document.save() @@ -199,9 +211,9 @@ def test_api_document_versions_list_authenticated_related_pagination_parent( client = APIClient() client.force_login(user) - grand_parent = factories.DocumentFactory() - parent = factories.DocumentFactory(parent=grand_parent) - document = factories.DocumentFactory(parent=parent) + grand_parent = create_document() + parent = create_document(parent=grand_parent) + document = create_document(parent=parent) for i in range(3): document.content = f"before {i:d}" document.save() @@ -270,7 +282,7 @@ def test_api_document_versions_list_exceeds_max_page_size(): client = APIClient() client.force_login(user) - document = factories.DocumentFactory(users=[user]) + document = create_document(users=[user]) document.content = "version 2" document.save() @@ -288,7 +300,7 @@ def test_api_document_versions_retrieve_anonymous(reach): Anonymous users should not be allowed to find specific versions for a document with restricted or authenticated link reach. """ - document = factories.DocumentFactory(link_reach=reach) + document = create_document(link_reach=reach) document.content = "new content" document.save() @@ -314,7 +326,7 @@ def test_api_document_versions_retrieve_authenticated_unrelated(reach): client = APIClient() client.force_login(user) - document = factories.DocumentFactory(link_reach=reach) + document = create_document(link_reach=reach) document.content = "new content" document.save() @@ -340,7 +352,7 @@ def test_api_document_versions_retrieve_authenticated_related(via, mock_user_tea client = APIClient() client.force_login(user) - document = factories.DocumentFactory() + document = create_document() document.content = "new content" document.save() @@ -406,9 +418,9 @@ def test_api_document_versions_retrieve_authenticated_related_parent( client = APIClient() client.force_login(user) - grand_parent = factories.DocumentFactory() - parent = factories.DocumentFactory(parent=grand_parent) - document = factories.DocumentFactory(parent=parent) + grand_parent = create_document() + parent = create_document(parent=grand_parent) + document = create_document(parent=parent) document.content = "new content" document.save() @@ -462,7 +474,7 @@ def test_api_document_versions_retrieve_authenticated_related_parent( def test_api_document_versions_create_anonymous(): """Anonymous users should not be allowed to create document versions.""" - document = factories.DocumentFactory() + document = create_document() response = APIClient().post( f"/api/v1.0/documents/{document.id!s}/versions/", @@ -484,7 +496,7 @@ def test_api_document_versions_create_authenticated_unrelated(): client = APIClient() client.force_login(user) - document = factories.DocumentFactory() + document = create_document() response = client.post( f"/api/v1.0/documents/{document.id!s}/versions/", @@ -506,7 +518,7 @@ def test_api_document_versions_create_authenticated_related(via, mock_user_teams client = APIClient() client.force_login(user) - document = factories.DocumentFactory() + document = create_document() if via == USER: factories.UserDocumentAccessFactory(document=document, user=user) elif via == TEAM: @@ -524,8 +536,10 @@ def test_api_document_versions_create_authenticated_related(via, mock_user_teams def test_api_document_versions_update_anonymous(): """Anonymous users should not be allowed to update a document version.""" - access = factories.UserDocumentAccessFactory() - document = access.document + document = create_document() + factories.UserDocumentAccessFactory(document=document) + # a second version of the object: the first one is the latest, which the + # listing excludes document.content = "new content" document.save() @@ -550,8 +564,10 @@ def test_api_document_versions_update_authenticated_unrelated(): client = APIClient() client.force_login(user) - access = factories.UserDocumentAccessFactory() - document = access.document + document = create_document() + factories.UserDocumentAccessFactory(document=document) + # a second version of the object: the first one is the latest, which the + # listing excludes document.content = "new content" document.save() @@ -559,7 +575,7 @@ def test_api_document_versions_update_authenticated_unrelated(): version_id = document.get_versions_slice()["versions"][0]["version_id"] response = client.put( - f"/api/v1.0/documents/{access.document_id!s}/versions/{version_id:s}/", + f"/api/v1.0/documents/{document.id!s}/versions/{version_id:s}/", {"foo": "bar"}, format="json", ) @@ -577,7 +593,7 @@ def test_api_document_versions_update_authenticated_related(via, mock_user_teams client = APIClient() client.force_login(user) - document = factories.DocumentFactory() + document = create_document() if via == USER: factories.UserDocumentAccessFactory(document=document, user=user) @@ -630,7 +646,7 @@ def test_api_document_versions_delete_authenticated(reach): client = APIClient() client.force_login(user) - document = factories.DocumentFactory(link_reach=reach) + document = create_document(link_reach=reach) document.content = "new content" document.save() @@ -655,7 +671,7 @@ def test_api_document_versions_delete_reader_or_editor(via, role, mock_user_team client = APIClient() client.force_login(user) - document = factories.DocumentFactory() + document = create_document() if via == USER: factories.UserDocumentAccessFactory(document=document, user=user, role=role) elif via == TEAM: @@ -692,7 +708,7 @@ def test_api_document_versions_delete_administrator_or_owner(via, mock_user_team client = APIClient() client.force_login(user) - document = factories.DocumentFactory() + document = create_document() role = random.choice(["administrator", "owner"]) if via == USER: factories.UserDocumentAccessFactory(document=document, user=user, role=role) diff --git a/src/backend/core/tests/documents/test_api_documents_duplicate.py b/src/backend/core/tests/documents/test_api_documents_duplicate.py index 95a9827792..7ce23dfda2 100644 --- a/src/backend/core/tests/documents/test_api_documents_duplicate.py +++ b/src/backend/core/tests/documents/test_api_documents_duplicate.py @@ -2,7 +2,6 @@ Test file uploads API endpoint for users in impress's core app. """ -import base64 import uuid from io import BytesIO from unittest import mock @@ -19,6 +18,7 @@ from rest_framework.test import APIClient from core import factories, models +from core.factories import YDOC_HELLO_WORLD_UPDATE from core.services.yhub_services import ( ServiceUnavailableError as YHubServiceUnavailableError, ) @@ -31,15 +31,19 @@ def mock_yhub_fixture(): """ The content of a document is held by the collaboration server. - It stands for a server holding the very content the factories gave the - documents, which is what an editor connected to it would have saved. + It stands for a server holding content for every document, which is what an + editor connected to it would have saved; the database holds none of it. A + test caring about the content of a given document declares it in + `mock_yhub.contents`, keyed by document id. """ + contents = {} def get_ydoc(document): - return base64.b64decode(document.content) if document.content else None + return contents.get(document.id, YDOC_HELLO_WORLD_UPDATE) with mock.patch("core.api.viewsets.YHubService") as mock_service: mock_service.return_value.get_ydoc.side_effect = get_ydoc + mock_service.contents = contents yield mock_service @@ -119,17 +123,16 @@ def test_api_documents_duplicate_success(index, mock_yhub): ) ydoc["document-store"] = fragment update = ydoc.get_update() - base64_content = base64.b64encode(update).decode("utf-8") # Create documents document = factories.DocumentFactory( id=document_ids[index], - content=base64_content, link_reach="restricted", users=[user, factories.UserFactory()], title="document with an image", attachments=[key for key, _ in image_refs], ) + mock_yhub.contents[document.id] = update factories.DocumentFactory(id=document_ids[(index + 1) % 3]) # Don't create document for third ID to check that it doesn't impact access to attachments @@ -144,7 +147,7 @@ def test_api_documents_duplicate_success(index, mock_yhub): # the content is copied through the collaboration server assert duplicated_document.content is None mock_yhub.return_value.create_ydoc.assert_called_once_with( - duplicated_document, base64.b64decode(document.content) + duplicated_document, update ) assert duplicated_document.creator == user assert duplicated_document.link_reach == "restricted" @@ -247,7 +250,7 @@ def test_api_documents_duplicate_with_accesses_admin(role, mock_yhub): # the content is copied through the collaboration server assert duplicated_document.content is None mock_yhub.return_value.create_ydoc.assert_called_once_with( - duplicated_document, base64.b64decode(document.content) + duplicated_document, YDOC_HELLO_WORLD_UPDATE ) assert duplicated_document.link_reach == document.link_reach assert duplicated_document.link_role == document.link_role @@ -306,7 +309,7 @@ def test_api_documents_duplicate_with_accesses_non_admin(role, mock_yhub): # the content is copied through the collaboration server assert duplicated_document.content is None mock_yhub.return_value.create_ydoc.assert_called_once_with( - duplicated_document, base64.b64decode(document.content) + duplicated_document, YDOC_HELLO_WORLD_UPDATE ) assert duplicated_document.link_reach == document.link_reach assert duplicated_document.link_role == document.link_role @@ -358,7 +361,7 @@ def test_api_documents_duplicate_non_root_document(role, mock_yhub): # the content is copied through the collaboration server assert duplicated_document.content is None mock_yhub.return_value.create_ydoc.assert_called_once_with( - duplicated_document, base64.b64decode(child.content) + duplicated_document, YDOC_HELLO_WORLD_UPDATE ) assert duplicated_document.link_reach == child.link_reach assert duplicated_document.link_role == child.link_role @@ -576,16 +579,15 @@ def test_api_documents_duplicate_with_descendants_and_attachments(mock_yhub): ] ) ydoc["document-store"] = fragment - update = ydoc.get_update() - root_content = base64.b64encode(update).decode("utf-8") + root_update = ydoc.get_update() root = factories.DocumentFactory( id=root_id, users=[(user, "owner")], title="Root with Image", - content=root_content, attachments=[image_key_root], ) + mock_yhub.contents[root.id] = root_update # Create child with different attachment ydoc_child = pycrdt.Doc() @@ -595,17 +597,16 @@ def test_api_documents_duplicate_with_descendants_and_attachments(mock_yhub): ] ) ydoc_child["document-store"] = fragment_child - update_child = ydoc_child.get_update() - child_content = base64.b64encode(update_child).decode("utf-8") + child_update = ydoc_child.get_update() # child - factories.DocumentFactory( + child = factories.DocumentFactory( id=child_id, parent=root, title="Child with Image", - content=child_content, attachments=[image_key_child], ) + mock_yhub.contents[child.id] = child_update # Duplicate with descendants with mock.patch("core.api.viewsets.posthog_capture") as mock_capture: @@ -638,8 +639,8 @@ def test_api_documents_duplicate_with_descendants_and_attachments(mock_yhub): assert duplicated_root.content is None assert dup_child.content is None assert mock_yhub.return_value.create_ydoc.call_args_list == [ - mock.call(duplicated_root, base64.b64decode(root_content)), - mock.call(dup_child, base64.b64decode(child_content)), + mock.call(duplicated_root, root_update), + mock.call(dup_child, child_update), ] diff --git a/src/backend/core/tests/documents/test_api_documents_formatted_content.py b/src/backend/core/tests/documents/test_api_documents_formatted_content.py index 7a800a10fb..d57074f00f 100644 --- a/src/backend/core/tests/documents/test_api_documents_formatted_content.py +++ b/src/backend/core/tests/documents/test_api_documents_formatted_content.py @@ -2,7 +2,6 @@ Tests for Documents API endpoint in impress's core app: convert """ -import base64 from unittest.mock import patch import pytest @@ -11,6 +10,7 @@ from rest_framework.test import APIClient from core import factories +from core.factories import YDOC_HELLO_WORLD_UPDATE from core.services.yhub_services import ( ServiceUnavailableError as YHubServiceUnavailableError, ) @@ -23,15 +23,12 @@ def mock_yhub_fixture(): """ The content of a document is held by the collaboration server. - It stands for a server holding the very content the factories gave the - documents, which is what an editor connected to it would have saved. + It stands for a server holding content for every document, which is what an + editor connected to it would have saved. The documents themselves hold none + in the database, nothing writes it there anymore. """ - - def get_ydoc(document): - return base64.b64decode(document.content) if document.content else None - with patch("core.api.viewsets.YHubService") as mock_service: - mock_service.return_value.get_ydoc.side_effect = get_ydoc + mock_service.return_value.get_ydoc.return_value = YDOC_HELLO_WORLD_UPDATE yield mock_service @@ -58,7 +55,7 @@ def test_api_documents_formatted_content_public(mock_content, reach, role): assert data["title"] == document.title assert data["content"] == {"some": "data"} mock_content.assert_called_once_with( - base64.b64decode(document.content), + YDOC_HELLO_WORLD_UPDATE, "application/vnd.yjs.doc", "application/json", ) @@ -117,7 +114,7 @@ def test_api_documents_formatted_content_not_public( assert data["title"] == document.title assert data["content"] == {"some": "data"} mock_content.assert_called_once_with( - base64.b64decode(document.content), + YDOC_HELLO_WORLD_UPDATE, "application/vnd.yjs.doc", "application/json", ) @@ -147,7 +144,7 @@ def test_api_documents_formatted_content_format(mock_content, content_format, ac assert data["title"] == document.title assert data["content"] == {"some": "data"} mock_content.assert_called_once_with( - base64.b64decode(document.content), "application/vnd.yjs.doc", accept + YDOC_HELLO_WORLD_UPDATE, "application/vnd.yjs.doc", accept ) @@ -188,9 +185,11 @@ def test_api_documents_formatted_content_nonexistent_document(mock_request): @patch("core.services.converter_services.YdocConverter._request") -def test_api_documents_formatted_content_empty_document(mock_request): +def test_api_documents_formatted_content_empty_document(mock_request, mock_yhub): """Test that accessing an empty document returns empty content.""" - document = factories.DocumentFactory(link_reach="public", content="") + document = factories.DocumentFactory(link_reach="public") + # an empty document is one the collaboration server holds nothing for + mock_yhub.return_value.get_ydoc.return_value = None response = APIClient().get( f"/api/v1.0/documents/{document.id!s}/formatted-content/" @@ -212,7 +211,6 @@ def test_api_documents_formatted_content_from_collaboration_server( document = factories.DocumentFactory(link_reach="public") mock_content.return_value = {"some": "data"} # what the collaboration server holds, edited since Django last saw it - mock_yhub.return_value.get_ydoc.side_effect = None mock_yhub.return_value.get_ydoc.return_value = b"\x01\x02edited update" response = APIClient().get( diff --git a/src/backend/core/tests/test_models_documents.py b/src/backend/core/tests/test_models_documents.py index 407e239da7..570c440848 100644 --- a/src/backend/core/tests/test_models_documents.py +++ b/src/backend/core/tests/test_models_documents.py @@ -938,7 +938,7 @@ def test_models_documents_get_versions_slice_pagination(settings): settings.DOCUMENT_VERSIONS_PAGE_SIZE = 4 # Create a document with 7 versions - document = factories.DocumentFactory() + document = factories.DocumentFactory(content=factories.YDOC_HELLO_WORLD_BASE64) for i in range(6): document.content = f"bar{i:d}" document.save() @@ -997,7 +997,7 @@ def test_models_documents_get_versions_slice_min_datetime(): def test_models_documents_version_duplicate(): """A new version should be created in object storage only if the content has changed.""" - document = factories.DocumentFactory() + document = factories.DocumentFactory(content=factories.YDOC_HELLO_WORLD_BASE64) file_key = str(document.pk) response = default_storage.connection.meta.client.list_object_versions( diff --git a/src/backend/core/tests/test_services_search_indexers.py b/src/backend/core/tests/test_services_search_indexers.py index 2dcc972d1b..324b5d374e 100644 --- a/src/backend/core/tests/test_services_search_indexers.py +++ b/src/backend/core/tests/test_services_search_indexers.py @@ -1,6 +1,5 @@ """Tests for Documents search indexers""" -from base64 import b64decode from functools import partial from json import dumps as json_dumps from unittest.mock import patch @@ -231,7 +230,7 @@ def test_services_search_indexers_serialize_document_deleted(): @pytest.mark.usefixtures("indexer_settings") def test_services_search_indexers_serialize_document_empty(): """Empty documents returns empty content in the serialized json.""" - document = factories.DocumentFactory(content="", title=None) + document = factories.DocumentFactory(title=None) indexer = FindDocumentIndexer() result = indexer.serialize_document(document, "", {}) @@ -366,14 +365,14 @@ def test_services_search_indexers_skip_documents_the_content_of_which_is_unreada """ unreadable, readable = factories.DocumentFactory.create_batch(2) - def get_ydoc(_service, document): + def get_ydoc(document): if document.pk == unreadable.pk: raise ServiceUnavailableError("yhub is unreachable") - return b64decode(document.content) + return factories.YDOC_HELLO_WORLD_UPDATE - # a plain replacement: the indexer_settings fixture already serves the - # content of the documents, this test needs one of them to fail - with patch.object(YHubService, "get_ydoc", get_ydoc): + # the indexer_settings fixture serves content for every document, this + # test needs one of them to fail + with patch.object(YHubService, "get_ydoc", side_effect=get_ydoc): assert FindDocumentIndexer().index() == 1 results = {doc["id"] for doc in mock_push.call_args[0][0]} @@ -388,11 +387,18 @@ def test_services_search_indexers_ignore_empty_documents(mock_push): and only the access data relevant to each batch should be used. """ document = factories.DocumentFactory() - factories.DocumentFactory(content="", title="") + empty = factories.DocumentFactory(title="") empty_title = factories.DocumentFactory(title="") - empty_content = factories.DocumentFactory(content="") + empty_content = factories.DocumentFactory() - assert FindDocumentIndexer().index() == 3 + # a document with no content is one the collaboration server holds none for + def get_ydoc(doc): + if doc.pk in (empty.pk, empty_content.pk): + return None + return factories.YDOC_HELLO_WORLD_UPDATE + + with patch.object(YHubService, "get_ydoc", side_effect=get_ydoc): + assert FindDocumentIndexer().index() == 3 assert mock_push.call_count == 1 @@ -417,10 +423,18 @@ def test_services_search_indexers_skip_empty_batches(mock_push, indexer_settings document = factories.DocumentFactory() - # Only empty docs - factories.DocumentFactory.create_batch(5, content="", title="") - - assert FindDocumentIndexer().index() == 1 + # Only empty docs: no title, and no content in the collaboration server + empty = factories.DocumentFactory.create_batch(5, title="") + empty_ids = {doc.pk for doc in empty} + + with patch.object( + YHubService, + "get_ydoc", + side_effect=lambda doc: ( + None if doc.pk in empty_ids else factories.YDOC_HELLO_WORLD_UPDATE + ), + ): + assert FindDocumentIndexer().index() == 1 assert mock_push.call_count == 1 results = [doc["id"] for doc in mock_push.call_args[0][0]] From a832ac2b82fd1cb8396070040c3961898abc5065 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Mon, 10 Aug 2026 15:12:22 +0200 Subject: [PATCH 45/59] =?UTF-8?q?=E2=9C=85(backend)=20correctly=20reload?= =?UTF-8?q?=20urls=20in=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After removing most of the usage of S3 in the tests, these ones are faster and make some flakyness more relevant. For example, in tests related to the external api we have to reload the urls based on the settings. We now have some race conditions where tests collapsed and urls are not correctly reloaded. --- src/backend/core/tests/conftest.py | 21 +++++++++- .../test_external_api_documents.py | 8 +++- src/backend/core/tests/utils/urls.py | 39 +++++++++++++++++-- 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/backend/core/tests/conftest.py b/src/backend/core/tests/conftest.py index b39af92a7a..45e0a4dbb9 100644 --- a/src/backend/core/tests/conftest.py +++ b/src/backend/core/tests/conftest.py @@ -10,7 +10,7 @@ from core import factories from core.services.yhub_services import YHubService -from core.tests.utils.urls import reload_urls +from core.tests.utils.urls import reload_urls, restore_urls USER = "user" TEAM = "team" @@ -23,6 +23,25 @@ def clear_cache(): cache.clear() +@pytest.fixture(autouse=True) +def restore_urlconf(): + """ + Put the URLs back after a test that reloaded them. + + Reloading is how a test makes the resource server routes appear or checks + that they are absent, but the URLconf belongs to the process: without this, + a test asserting a 404 on `/external_api/` and one asserting a 401 pass or + fail depending on which ran first in their worker. + + Autouse and asking for nothing, so it is set up before the `settings` + fixture and torn down after it: the reload then sees the settings of the + project, not the ones of the test. + """ + yield + + restore_urls() + + @pytest.fixture def mock_user_teams(): """Mock for the "teams" property on the User model.""" diff --git a/src/backend/core/tests/external_api/test_external_api_documents.py b/src/backend/core/tests/external_api/test_external_api_documents.py index b4bb8f1eb7..183b092ed5 100644 --- a/src/backend/core/tests/external_api/test_external_api_documents.py +++ b/src/backend/core/tests/external_api/test_external_api_documents.py @@ -581,11 +581,17 @@ def test_external_api_documents_trashbin_not_allowed( assert response.status_code == 403 -def test_external_api_documents_create_for_owner_not_allowed(): +def test_external_api_documents_create_for_owner_not_allowed( + resource_server_backend_conf, +): """ Authenticated users SHOULD NOT be allowed to call create documents on behalf of other users. This API endpoint is reserved for server-to-server calls. + + The route only exists when the resource server is enabled, hence the + fixture: the endpoint answering 401 is what this asserts, not the + `/external_api/` prefix being routed at all. """ user = factories.UserFactory() diff --git a/src/backend/core/tests/utils/urls.py b/src/backend/core/tests/utils/urls.py index 78455de1ee..2da23f7ee8 100644 --- a/src/backend/core/tests/utils/urls.py +++ b/src/backend/core/tests/utils/urls.py @@ -5,12 +5,21 @@ from django.urls import clear_url_caches -def reload_urls(): +class _URLConf: """ - Reload the URLs. Since the URLs are loaded based on a - settings value, we need to reload them to make the - URL settings based condition effective. + Whether a test reloaded the URLs of this process. + + The URLconf is module-level state: a reload outlives the test that did it + and every test running after it in the same worker sees its routes — which + ones share a worker changes from one run to the next. `restore_urls` puts + the default back, and this flag keeps it to the tests that need it. """ + + reloaded = False + + +def _reload(): + """Reload the URL modules and drop the resolver caches.""" import core.urls # pylint:disable=import-outside-toplevel # noqa: PLC0415 import impress.urls # pylint:disable=import-outside-toplevel # noqa: PLC0415 @@ -18,3 +27,25 @@ def reload_urls(): importlib.reload(core.urls) importlib.reload(impress.urls) clear_url_caches() + + +def reload_urls(): + """ + Reload the URLs. Since the URLs are loaded based on a + settings value, we need to reload them to make the + URL settings based condition effective. + """ + _URLConf.reloaded = True + _reload() + + +def restore_urls(): + """ + Reload the URLs of a test that changed them, so the next one starts clean. + + Called once the settings of the test are restored, so the routes are the + ones the settings of the project declare. + """ + if _URLConf.reloaded: + _URLConf.reloaded = False + _reload() From a6fdc6d514045ab681e0cdbc73aa653df4560b5e Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Mon, 10 Aug 2026 16:32:21 +0200 Subject: [PATCH 46/59] =?UTF-8?q?=E2=9C=A8(backend)=20add=20a=20`migrate?= =?UTF-8?q?=5Fdocuments`=20command?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit command replaying the legacy content of the documents into the collaboration server, one call to its migrate endpoint per document. Resumable and safe to re-run: what became of every document is recorded (`impress_document_migration`), a server that is unwell is retried with a backoff and a document it refuses is left for a later run (`--retry-failed`). Bounded by `--concurrency`, `--rate` and `--limit`, most recently edited documents first --- CHANGELOG.md | 7 + .../management/commands/migrate_documents.py | 287 ++++++++++++++++++ .../core/migrations/0033_documentmigration.py | 92 ++++++ src/backend/core/models.py | 63 ++++ src/backend/core/services/yhub_services.py | 36 ++- .../tests/commands/test_migrate_documents.py | 169 +++++++++++ .../tests/test_integration_yhub_migration.py | 13 +- .../core/tests/test_services_yhub_services.py | 38 +++ src/backend/impress/settings.py | 8 + src/yhub-server/README.md | 8 +- src/yhub-server/server.js | 39 +-- 11 files changed, 726 insertions(+), 34 deletions(-) create mode 100644 src/backend/core/management/commands/migrate_documents.py create mode 100644 src/backend/core/migrations/0033_documentmigration.py create mode 100644 src/backend/core/tests/commands/test_migrate_documents.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cd01346b2..335a51741b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,13 @@ and this project adheres to `503` instead of denying access like a permission failure, so clients retry instead of giving up. The built-in endpoints can also answer JSON on `Accept: application/json` +- ✨(backend) add a `migrate_documents` command replaying the legacy content of + the documents into the collaboration server, one call to its migrate endpoint + per document. Resumable and safe to re-run: what became of every document is + recorded (`impress_document_migration`), a server that is unwell is retried + with a backoff and a document it refuses is left for a later run + (`--retry-failed`). Bounded by `--concurrency`, `--rate` and `--limit`, most + recently edited documents first - ✨(collaboration) notify the backend when the worker persists new content for a document, so the lists ordered by `updated_at` follow the edits made on the collaboration server. The backend serves it on diff --git a/src/backend/core/management/commands/migrate_documents.py b/src/backend/core/management/commands/migrate_documents.py new file mode 100644 index 0000000000..0e7d965bf7 --- /dev/null +++ b/src/backend/core/management/commands/migrate_documents.py @@ -0,0 +1,287 @@ +""" +Replay the legacy content of the documents into the collaboration server. + +The content of a document used to be a file in our object storage, one version +per save; it now lives in the collaboration server, which is able to read those +versions back and rebuild the history from them, document by document. This +command is what hands it the corpus. + +It is meant to be run again: every document it finishes is recorded, and the +collaboration server answers "already" for anything it has already migrated, so +a run that is interrupted, rate limited or killed simply picks up where it +stopped. Nothing is destroyed, on either side. +""" + +import logging +import time +from concurrent.futures import ThreadPoolExecutor + +from django.core.management.base import BaseCommand +from django.utils import timezone + +from core import models +from core.services.yhub_services import APIError, YHubError, YHubService + +logger = logging.getLogger("impress.commands.migrate_documents") + +# Reported often enough to see a run is alive, rarely enough to keep the logs +# of a corpus of hundreds of thousands of documents readable. +PROGRESS_EVERY = 500 + +# Results are written in batches: a smaller one costs a query per document, a +# larger one loses more work when the command is killed — and losing it only +# means handing those documents over again, which answers "already". +WRITE_BATCH = 200 + +# The collaboration server says whether it is worth insisting: a 5xx or a 429 +# is a server that is unwell or busy, anything else in the 4xx range is about +# this document and will fail the same way forever. +RETRY_STATUSES = frozenset({429}) + + +class Command(BaseCommand): + """Migrate the legacy content of the documents into the collaboration server.""" + + help = __doc__ + + def add_arguments(self, parser): + """Define the arguments of the command.""" + parser.add_argument( + "--concurrency", + type=int, + default=2, + help=( + "Documents migrated at the same time. The replay runs on the " + "main thread of the collaboration server, so this is bounded " + "by its cpu: raise it against a pool that serves nothing else." + ), + ) + parser.add_argument( + "--rate", + type=float, + default=0, + help="Documents per second not to exceed (0: as fast as possible)", + ) + parser.add_argument( + "--limit", + type=int, + default=0, + help="Stop after this many documents (0: all of them)", + ) + parser.add_argument( + "--created-before", + type=str, + default=None, + help=( + "Only migrate the documents created before this date " + "(ISO 8601). Documents created after the collaboration server " + "became the source of truth have no legacy content." + ), + ) + parser.add_argument( + "--retry-failed", + action="store_true", + default=False, + help="Hand over the documents a previous run could not migrate", + ) + parser.add_argument( + "--retries", + type=int, + default=3, + help="Attempts per document when the collaboration server is unwell", + ) + parser.add_argument( + "--dry-run", + action="store_true", + default=False, + help="Count what would be migrated, call nothing", + ) + + def handle(self, *args, **options): + """Hand the documents to the collaboration server, and record what it says.""" + queryset = self.get_queryset(options) + total = queryset.count() + + if options["dry_run"]: + self.stdout.write(f"{total:d} documents to migrate") + return + + self.stdout.write( + f"Migrating {total:d} documents, {options['concurrency']:d} at a time" + ) + started = time.monotonic() + counts = self.migrate(queryset, options) + elapsed = time.monotonic() - started + + done = sum(counts.values()) + rate = done / elapsed if elapsed else 0 + self.stdout.write( + f"Migrated {done:d} documents in {elapsed:.0f}s ({rate:.1f}/s): " + + ", ".join( + f"{status}={count:d}" for status, count in sorted(counts.items()) + ) + ) + if counts.get(models.DocumentMigrationStatus.FAILED): + self.stdout.write( + self.style.WARNING( + "Some documents could not be migrated, they are recorded as " + "failed: run again with --retry-failed once the cause is fixed." + ) + ) + + def get_queryset(self, options): + """ + Return the documents left to migrate, the ones that matter most first. + + A document is opened before it is missed: the ones edited recently are + the ones users are about to read, and until a document is migrated the + collaboration server only seeds its latest state, without its history. + """ + queryset = models.Document.objects.all() + + if options["created_before"]: + queryset = queryset.filter(created_at__lt=options["created_before"]) + + done = set(models.DocumentMigrationStatus.values) + if options["retry_failed"]: + done.discard(models.DocumentMigrationStatus.FAILED) + + queryset = queryset.exclude(migration__status__in=done) + + if options["limit"]: + queryset = queryset.order_by("-updated_at")[: options["limit"]] + # a sliced queryset cannot be iterated with a server-side cursor + return models.Document.objects.filter( + pk__in=queryset.values("pk") + ).order_by("-updated_at") + + return queryset.order_by("-updated_at") + + def migrate(self, queryset, options): + """Run the migration, writing what happened as the answers come in.""" + counts = {} + results = [] + done = 0 + started = time.monotonic() + + with ThreadPoolExecutor(max_workers=options["concurrency"]) as pool: + # imap-like: the documents are read from the database as the pool + # frees up, so a corpus of any size is never held in memory + documents = queryset.only("pk").iterator(chunk_size=WRITE_BATCH) + migrations = pool.map( + lambda document: self.migrate_document(document, options["retries"]), + self.paced(documents, options["rate"]), + ) + + for migration in migrations: + results.append(migration) + counts[migration.status] = counts.get(migration.status, 0) + 1 + done += 1 + + if len(results) >= WRITE_BATCH: + self.save(results) + results = [] + if done % PROGRESS_EVERY == 0: + rate = done / (time.monotonic() - started) + self.stdout.write(f" {done:d} documents ({rate:.1f}/s)") + + self.save(results) + + return counts + + @staticmethod + def paced(documents, rate): + """Yield the documents no faster than `rate` per second.""" + if not rate: + yield from documents + return + + interval = 1 / rate + next_at = time.monotonic() + for document in documents: + now = time.monotonic() + if next_at > now: + time.sleep(next_at - now) + next_at = max(next_at + interval, now) + yield document + + def migrate_document(self, document, retries): + """ + Hand one document over, and return what the collaboration server said. + + Runs in a worker thread and touches no database: the results are + written by the main thread, so the pool needs no connection of its own. + """ + service = YHubService() + + for attempt in range(1, retries + 1): + try: + result = service.migrate(document) + except YHubError as err: + if attempt < retries and self.is_retryable(err): + # the server is unwell or busy, not this document + time.sleep(2**attempt) + continue + + logger.warning("document %s was not migrated: %s", document.pk, err) + return models.DocumentMigration( + document_id=document.pk, + status=models.DocumentMigrationStatus.FAILED, + error=str(err)[:500], + updated_at=timezone.now(), + ) + + return models.DocumentMigration( + document_id=document.pk, + status=result.get("status", models.DocumentMigrationStatus.MIGRATED), + versions=result.get("versions", 0), + applied=result.get("applied", 0), + skipped=result.get("skipped", 0), + dropped=result.get("dropped", 0), + duration_ms=result.get("durationMs", 0), + updated_at=timezone.now(), + ) + + raise AssertionError( + "unreachable: the loop returns or raises" + ) # pragma: no cover + + @staticmethod + def is_retryable(err): + """ + Say whether handing the same document over again could go better. + + The collaboration server answers a 5xx or a 429 when it is unwell or + busy, and any other 4xx about the document itself — which will not fix + itself. Not reaching it at all is worth another try. + """ + if not isinstance(err, APIError): + return True + + return ( + err.status_code is None + or err.status_code >= 500 + or err.status_code in RETRY_STATUSES + ) + + @staticmethod + def save(migrations): + """Record what became of these documents, replacing what a previous run said.""" + if not migrations: + return + + models.DocumentMigration.objects.bulk_create( + migrations, + update_conflicts=True, + update_fields=[ + "status", + "versions", + "applied", + "skipped", + "dropped", + "duration_ms", + "error", + "updated_at", + ], + unique_fields=["document"], + ) diff --git a/src/backend/core/migrations/0033_documentmigration.py b/src/backend/core/migrations/0033_documentmigration.py new file mode 100644 index 0000000000..9e7adcce93 --- /dev/null +++ b/src/backend/core/migrations/0033_documentmigration.py @@ -0,0 +1,92 @@ +# Generated by Django 5.2.14 on 2026-08-10 13:36 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0032_remove_linktrace_is_masked"), + ] + + operations = [ + migrations.CreateModel( + name="DocumentMigration", + fields=[ + ( + "document", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + related_name="migration", + serialize=False, + to="core.document", + ), + ), + ( + "status", + models.CharField( + choices=[ + ("ok", "Migrated"), + ("already", "Already migrated"), + ("empty", "Nothing in the object storage"), + ("nothing", "No readable version"), + ("failed", "Failed"), + ], + max_length=10, + verbose_name="status", + ), + ), + ( + "versions", + models.PositiveIntegerField(default=0, verbose_name="versions"), + ), + ( + "applied", + models.PositiveIntegerField( + default=0, + help_text="versions that added content, one activity entry each", + verbose_name="applied", + ), + ), + ( + "skipped", + models.PositiveIntegerField( + default=0, + help_text="versions that could not be read", + verbose_name="skipped", + ), + ), + ( + "dropped", + models.PositiveIntegerField( + default=0, + help_text="versions older than the ones the server replays", + verbose_name="dropped", + ), + ), + ( + "duration_ms", + models.PositiveIntegerField(default=0, verbose_name="duration"), + ), + ( + "error", + models.TextField(blank=True, default="", verbose_name="error"), + ), + ( + "updated_at", + models.DateTimeField(auto_now=True, verbose_name="updated on"), + ), + ], + options={ + "verbose_name": "Document migration", + "verbose_name_plural": "Document migrations", + "db_table": "impress_document_migration", + "indexes": [ + models.Index( + fields=["status"], name="impress_doc_status_7d8208_idx" + ) + ], + }, + ), + ] diff --git a/src/backend/core/models.py b/src/backend/core/models.py index 1b890f6190..34e54c9cc0 100644 --- a/src/backend/core/models.py +++ b/src/backend/core/models.py @@ -2133,3 +2133,66 @@ def get_abilities(self, user): "partial_update": is_admin_or_owner, "retrieve": is_admin_or_owner, } + + +class DocumentMigrationStatus(models.TextChoices): + """What became of a document handed to the collaboration server to migrate.""" + + MIGRATED = "ok", _("Migrated") + ALREADY = "already", _("Already migrated") + EMPTY = "empty", _("Nothing in the object storage") + NOTHING = "nothing", _("No readable version") + FAILED = "failed", _("Failed") + + +class DocumentMigration(models.Model): + """ + What the collaboration server did with the legacy content of a document. + + The ledger of the backfill: the collaboration server keeps its own set of + the documents it migrated, but only of those it actually wrote history for. + A document it found nothing for is not in it and would be handed over again + on every run, and a valkey configured to evict would lose the set entirely. + This table is what the command reads to know what is left to do, and what + an operator reads to know how it went. + """ + + document = models.OneToOneField( + Document, + on_delete=models.CASCADE, + related_name="migration", + primary_key=True, + ) + status = models.CharField( + max_length=10, + choices=DocumentMigrationStatus.choices, + verbose_name=_("status"), + ) + versions = models.PositiveIntegerField(default=0, verbose_name=_("versions")) + applied = models.PositiveIntegerField( + default=0, + verbose_name=_("applied"), + help_text=_("versions that added content, one activity entry each"), + ) + skipped = models.PositiveIntegerField( + default=0, + verbose_name=_("skipped"), + help_text=_("versions that could not be read"), + ) + dropped = models.PositiveIntegerField( + default=0, + verbose_name=_("dropped"), + help_text=_("versions older than the ones the server replays"), + ) + duration_ms = models.PositiveIntegerField(default=0, verbose_name=_("duration")) + error = models.TextField(blank=True, default="", verbose_name=_("error")) + updated_at = models.DateTimeField(auto_now=True, verbose_name=_("updated on")) + + class Meta: + db_table = "impress_document_migration" + verbose_name = _("Document migration") + verbose_name_plural = _("Document migrations") + indexes = [models.Index(fields=["status"])] + + def __str__(self): + return f"{self.document_id!s}: {self.status:s}" diff --git a/src/backend/core/services/yhub_services.py b/src/backend/core/services/yhub_services.py index c67758f10c..8ce04fa27c 100644 --- a/src/backend/core/services/yhub_services.py +++ b/src/backend/core/services/yhub_services.py @@ -183,12 +183,14 @@ def build_user_header(user_id): """ return {"X-User-Id": str(user_id)} if user_id else {} - def request(self, method, url, data=None, headers=None): + # pylint: disable-next=too-many-arguments + def request(self, method, url, *, data=None, headers=None, timeout=None): """ Send an authenticated request to the yhub API, asking it for JSON. Return the raw response, it is up to the caller to decode its body: the - endpoints do not all answer with the same payload. + endpoints do not all answer with the same payload. An endpoint doing + more than answering a document passes its own timeout. """ try: response = requests.request( @@ -201,7 +203,7 @@ def request(self, method, url, data=None, headers=None): "Accept": "application/json", **(headers or {}), }, - timeout=self.timeout, + timeout=timeout or self.timeout, ) except requests.RequestException as err: logger.exception("yhub service error: url=%s", url) @@ -285,6 +287,34 @@ def create_ydoc(self, document, update): }, ) + def migrate(self, document, force=False): + """ + Replay the legacy version history of a document into the collaboration server. + + The content of the documents used to live in our object storage, one + version of `{id}/file` per save. yhub reads them all back and rebuilds + the history with the timestamps of those versions, which is what makes + its activity line up with the versions we report. + + Answers what became of the document: `ok` when this call wrote its + history, `already` when a previous one did, `empty` when there is + nothing in the object storage (a document born in yhub) and `nothing` + when none of its versions could be read. All four are terminal, only a + failure to reach yhub raises. + + Forcing a document that is already migrated attributes its content a + second time: it is for a document whose yhub state was wiped, never for + a retry. + """ + url = self.build_url("migrate", document) + response = self.request( + "post", + f"{url}?force=true" if force else url, + timeout=settings.YHUB_MIGRATION_TIMEOUT, + ) + + return self.json_body(response) + def reset_connections(self, document, user_id=None): """ Re-check the access of the clients connected to a document. diff --git a/src/backend/core/tests/commands/test_migrate_documents.py b/src/backend/core/tests/commands/test_migrate_documents.py new file mode 100644 index 0000000000..de580689e3 --- /dev/null +++ b/src/backend/core/tests/commands/test_migrate_documents.py @@ -0,0 +1,169 @@ +"""Unit tests for the `migrate_documents` command.""" + +from io import StringIO +from unittest import mock + +from django.core.management import call_command + +import pytest + +from core import factories, models +from core.services.yhub_services import APIError, ServiceUnavailableError, YHubService + +pytestmark = pytest.mark.django_db + + +def migrated(status="ok", **stats): + """What the collaboration server answers for a document it migrated.""" + return {"status": status, "migrated": status == "ok", **stats} + + +@pytest.fixture(name="collaboration_server", autouse=True) +def collaboration_server_fixture(): + """Answer every document as migrated, unless a test says otherwise.""" + with mock.patch.object( + YHubService, "migrate", return_value=migrated() + ) as mock_migrate: + yield mock_migrate + + +def run_command(**options): + """Run the command and return what it wrote.""" + stdout = StringIO() + call_command("migrate_documents", stdout=stdout, **options) + + return stdout.getvalue() + + +def test_commands_migrate_documents(collaboration_server): + """Every document should be handed to the collaboration server, once.""" + documents = factories.DocumentFactory.create_batch(3) + collaboration_server.return_value = migrated( + versions=4, applied=3, skipped=1, dropped=2, durationMs=42 + ) + + output = run_command() + + assert collaboration_server.call_count == 3 + handed = {call.args[0].pk for call in collaboration_server.call_args_list} + assert handed == {document.pk for document in documents} + + assert models.DocumentMigration.objects.count() == 3 + migration = models.DocumentMigration.objects.first() + assert migration.status == models.DocumentMigrationStatus.MIGRATED + assert (migration.versions, migration.applied) == (4, 3) + assert (migration.skipped, migration.dropped) == (1, 2) + assert migration.duration_ms == 42 + assert "ok=3" in output + + +@pytest.mark.parametrize("status", ["ok", "already", "empty", "nothing"]) +def test_commands_migrate_documents_records_every_outcome(collaboration_server, status): + """The four answers of the collaboration server are all terminal.""" + document = factories.DocumentFactory() + collaboration_server.return_value = migrated(status) + + run_command() + + assert models.DocumentMigration.objects.get(document=document).status == status + + # none of them is handed over again + run_command() + + assert collaboration_server.call_count == 1 + + +def test_commands_migrate_documents_failure_is_recorded_and_retried( + collaboration_server, +): + """A document that could not be migrated should be left for another run.""" + document = factories.DocumentFactory() + collaboration_server.side_effect = APIError("yhub is confused", status_code=400) + + output = run_command(retries=1) + + migration = models.DocumentMigration.objects.get(document=document) + assert migration.status == models.DocumentMigrationStatus.FAILED + assert "yhub is confused" in migration.error + assert "failed=1" in output + + # left alone by a plain run, handed over again when asked for + run_command() + assert collaboration_server.call_count == 1 + + collaboration_server.side_effect = None + collaboration_server.return_value = migrated() + run_command(retry_failed=True) + + assert collaboration_server.call_count == 2 + assert ( + models.DocumentMigration.objects.get(document=document).status + == models.DocumentMigrationStatus.MIGRATED + ) + + +def test_commands_migrate_documents_retries_a_server_that_is_unwell( + collaboration_server, +): + """A 5xx is about the server, the same document is worth handing over again.""" + document = factories.DocumentFactory() + collaboration_server.side_effect = [ + ServiceUnavailableError("connection reset"), + migrated(), + ] + + with mock.patch("time.sleep"): # no backoff wait in tests + run_command(retries=2) + + assert collaboration_server.call_count == 2 + assert ( + models.DocumentMigration.objects.get(document=document).status + == models.DocumentMigrationStatus.MIGRATED + ) + + +def test_commands_migrate_documents_does_not_retry_a_refused_document( + collaboration_server, +): + """A 4xx is about the document, insisting would only waste the server.""" + factories.DocumentFactory() + collaboration_server.side_effect = APIError("Room name is invalid", status_code=400) + + with mock.patch("time.sleep"): + run_command(retries=3) + + assert collaboration_server.call_count == 1 + + +def test_commands_migrate_documents_limit(collaboration_server): + """The most recently edited documents should be migrated first.""" + factories.DocumentFactory.create_batch(3) + recent = factories.DocumentFactory() + + run_command(limit=1) + + assert collaboration_server.call_count == 1 + assert collaboration_server.call_args[0][0].pk == recent.pk + + +def test_commands_migrate_documents_created_before(collaboration_server): + """A document created after the cutover has no legacy content to migrate.""" + old = factories.DocumentFactory() + models.Document.objects.filter(pk=old.pk).update(created_at="2020-01-01T00:00:00Z") + factories.DocumentFactory() + + run_command(created_before="2021-01-01T00:00:00Z") + + assert collaboration_server.call_count == 1 + assert collaboration_server.call_args[0][0].pk == old.pk + + +def test_commands_migrate_documents_dry_run(collaboration_server): + """A dry run should count the documents and call nothing.""" + factories.DocumentFactory.create_batch(2) + + output = run_command(dry_run=True) + + assert "2 documents to migrate" in output + collaboration_server.assert_not_called() + assert not models.DocumentMigration.objects.exists() diff --git a/src/backend/core/tests/test_integration_yhub_migration.py b/src/backend/core/tests/test_integration_yhub_migration.py index 36f06c053f..89e0f83414 100644 --- a/src/backend/core/tests/test_integration_yhub_migration.py +++ b/src/backend/core/tests/test_integration_yhub_migration.py @@ -31,10 +31,8 @@ import pytest import requests -from core.services.jwt_services import JWTService +from core.services.jwt_services import Audiences, JWTService -# yhub verifies that its own name is the audience of the token (server.js) -YHUB_AUDIENCE = "yhub" # the org yhub is configured with; documents live under /docs/{docid} YHUB_ORG = "docs" TIMEOUT = 10 @@ -80,7 +78,8 @@ def admin_headers_fixture(settings): # pylint: disable=redefined-outer-name """ if not settings.JWT_PRIVATE_KEY: pytest.skip("JWT_PRIVATE_KEY is not configured") - token = JWTService().get_admin_token({"aud": YHUB_AUDIENCE}) + # yhub verifies that its own name is the audience of the token (server.js) + token = JWTService().get_admin_token(Audiences.YHUB) return {"Authorization": f"Bearer {token}"} @@ -330,7 +329,11 @@ def test_integration_yhub_full_migration_is_idempotent(admin_headers): again = _migrate(docid, admin_headers) assert again.status_code == 200 - assert again.json() == {"message": "Already migrated", "migrated": False} + assert again.json() == { + "status": "already", + "message": "Already migrated", + "migrated": False, + } forced = _migrate(docid, admin_headers, force="true") assert forced.json()["migrated"] is True diff --git a/src/backend/core/tests/test_services_yhub_services.py b/src/backend/core/tests/test_services_yhub_services.py index ffb90cc412..9379188c37 100644 --- a/src/backend/core/tests/test_services_yhub_services.py +++ b/src/backend/core/tests/test_services_yhub_services.py @@ -204,6 +204,44 @@ def test_create_ydoc_already_exists(mock_request): assert excinfo.value.status_code == 409 +@patch("requests.request") +def test_migrate(mock_request): + """Should ask yhub to replay the legacy history, and answer what it did.""" + mock_request.return_value.ok = True + mock_request.return_value.json.return_value = { + "status": "ok", + "message": "Migration completed", + "migrated": True, + "versions": 12, + "applied": 9, + "durationMs": 1234, + } + + result = YHubService().migrate(DOCUMENT) + + assert result["status"] == "ok" + assert result["applied"] == 9 + args, kwargs = mock_request.call_args + assert args == ( + "post", + f"http://yhub:3002/collaboration/migrate/v1/docs/{DOCUMENT.id!s}", + ) + # reading every version of a document takes longer than any other call + assert kwargs["timeout"] == 600 + + +@patch("requests.request") +def test_migrate_forced(mock_request): + """Forcing a document that is already migrated should be asked for explicitly.""" + mock_request.return_value.ok = True + mock_request.return_value.json.return_value = {"status": "ok"} + + YHubService().migrate(DOCUMENT, force=True) + + args, _kwargs = mock_request.call_args + assert args[1].endswith("?force=true") + + @patch("requests.request") def test_reset_connections(mock_request): """Should ask yhub to re-check every connection of the document.""" diff --git a/src/backend/impress/settings.py b/src/backend/impress/settings.py index 724331036a..f1b3fb1807 100755 --- a/src/backend/impress/settings.py +++ b/src/backend/impress/settings.py @@ -540,6 +540,14 @@ class Base(Configuration): environ_name="YHUB_API_TIMEOUT", environ_prefix=None, ) + # Replaying the legacy history of a document reads every one of its S3 + # versions, so it is the one call that can take minutes. Timing it out does + # not stop the collaboration server, it only loses the answer. + YHUB_MIGRATION_TIMEOUT = values.IntegerValue( + default=600, + environ_name="YHUB_MIGRATION_TIMEOUT", + environ_prefix=None, + ) # JWT # RSA private key (PEM) used to sign the tokens issued by diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index a0653e42ef..e9b15a4ef5 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -292,8 +292,12 @@ Guarantees: - **More than 500 versions**: only the newest 500 are replayed and the rest fold into the first replayed version, reported as `dropped`. -Response (200): `{ migrated, versions, applied, skipped, dropped, bytes, -durationMs }`. +Response (200): `{ status, message, migrated, versions, applied, skipped, +dropped, bytes, durationMs }`. `status` is the machine-readable outcome a +backfill driver records — `ok`, `already`, `empty` (no legacy object, a +brand-new document) or `nothing` (versions exist, none readable) — all of them +done, which is why they share one 2xx. `migrated` says whether this very call +wrote the history. Caveats: diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index 3c952b3871..14d65f2180 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -429,31 +429,22 @@ const api = [ const { status, ...stats } = await fullMigrate(req.yhub, req.room, { force: req.query.force === 'true', }); - if (status === 'already') { - return jsonResponse(200, { - message: 'Already migrated', - migrated: false, - }); - } - if (status === 'empty') { - // brand-new documents never had a legacy object; nothing to replay - // and nothing wrong — a backfill driver treats this as done - return jsonResponse(200, { - message: 'No legacy document in s3', - migrated: false, - ...stats, - }); - } - if (status === 'nothing') { - return jsonResponse(200, { - message: 'No usable content in the legacy versions', - migrated: false, - ...stats, - }); - } + // `status` is what a backfill driver records per document: 'ok', + // 'already', 'empty' (no legacy object — a brand-new document) or + // 'nothing' (versions exist, none readable). All four are done, hence + // one 2xx; `message` says the same thing to a human, `migrated` + // whether this call is the one that wrote the history. + const messages = { + already: 'Already migrated', + empty: 'No legacy document in s3', + nothing: 'No usable content in the legacy versions', + ok: 'Migration completed', + }; + return jsonResponse(200, { - message: 'Migration completed', - migrated: true, + status, + message: messages[status], + migrated: status === 'ok', ...stats, }); }, From a3dec0883fcb3d2e43cf0eedcd4d314e085613be Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Wed, 12 Aug 2026 11:57:33 +0200 Subject: [PATCH 47/59] =?UTF-8?q?=E2=9C=A8(yhub)=20add=20a=20restore=20end?= =?UTF-8?q?point?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We will use the delete endpoint available in the yhub server with the soft delete feature in the backend application, but we also need a restore endpoint and this endpoint is not available in the yhub server. This commit adds a new custom endpoint implementing the restore action. --- src/yhub-server/README.md | 37 ++++++++++++++++++++++++++++-- src/yhub-server/server.js | 47 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index e9b15a4ef5..1a7c9ee191 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -42,6 +42,10 @@ It is not a fork of yhub — it is a thin wrapper: - exposes `POST /collaboration/migrate/v1/{org}/{docid}`, which replays a document's **full** legacy version history out of the S3 media bucket (see "Full migration" below) — admin JWT only, like `reset-connections`, +- exposes `POST /collaboration/restore-ydoc/v1/{org}/{docid}`, which undoes the + deletion of a document — admin JWT only, like `reset-connections`. Deleting + one needs nothing custom, the built-in `DELETE .../ydoc/` does it (see + "Deletion" below); restoring has no built-in route, - notifies the Django backend on `POST /api/v1.0/documents/{id}/content-updated/` whenever the worker persists new content for a document, so that lists ordered by `updated_at` @@ -61,8 +65,9 @@ the websocket and the built-in document APIs (`ydoc`, `rollback`, `prune`, `changeset`, `activity`) are all guarded by the same cookie-based document authorization and are meant to be reachable by browsers, as is `/collaboration/jwks/v1`, which carries public keys and nothing else. The one -exception is `/collaboration/reset-connections/` and `/collaboration/migrate/`, which are -backend-internal and should not be routed through the public ingress. +exception is `/collaboration/reset-connections/`, `/collaboration/migrate/` and +`/collaboration/restore-ydoc/`, which are backend-internal and should not be +routed through the public ingress. ## Container image @@ -113,6 +118,34 @@ the counterpart of `make migrate` for the Django database. `make bootstrap` already includes it, so a fresh checkout needs nothing extra; an upgrade is `make migrate-yhub` and restart the service. +## Deletion + +The content of a document lives here, so deleting one in Docs has to be said +here too — otherwise the clients already connected keep editing it and the +content outlives the document. The backend does that from +`sync_service_deletions_in_cascade`, which walks the deleted subtree and tells +this server what became of each of its documents. + +Deleting is `DELETE /collaboration/ydoc/v1/{org}/{docid}`, built into yhub +0.6.0. It is a **soft** deletion: the deletion is recorded, the clients editing +the document are disconnected (websocket close code 4404) and every route +answers 404 for it (`{"code": "doc-deleted"}`, which a document that was never +written does not — that one answers an empty document), but its content is left +untouched. Deleting twice keeps the date of the first deletion. + +Restoring is the custom `POST /collaboration/restore-ydoc/v1/{org}/{docid}` +above: yhub 0.6.0 has no built-in route for it. The content was never touched, +so the document comes back with its whole history. Restoring one that is not +deleted answers 200 and changes nothing, which is what lets the backend restore +a subtree without asking what became of each document in it. + +Erasing the content for good is a third operation (`YHub.deleteDoc(room, { +hard: true })`), reachable from inside this process only — yhub deliberately +keeps it off the REST API. Nothing here calls it: Docs never erases a document +either, a soft-deleted one simply stops being restorable after +`TRASHBIN_CUTOFF_DAYS`. Note that a hard deletion is final for that room — the +docid can never be written again, and `restore-ydoc` answers 409 for it. + ## Soft migration (`SOFT_MIGRATION=true`) Documents were historically stored by the Django backend in the S3 media diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index 14d65f2180..34560d20aa 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -556,6 +556,53 @@ const api = [ }, }, }), + // POST /collaboration/restore-ydoc/v1/{org}/{docid} — undo the deletion of a + // document, putting back what `DELETE .../ydoc/` took away. + // + // Deleting has a built-in route, restoring does not: yhub 0.6.0 exposes + // `restoreDoc` to the process embedding it and nothing else. Backend-internal + // like reset-connections and migrate, gated to the admin token by the + // 'restore' purpose — a document leaves the trashbin because the backend + // says so, never because an editor asked. + createApiEndpoint('restore-ydoc', { + accessPurpose: 'restore', + post: { + handler: async (req) => { + if (req.org !== ORG) { + return jsonResponse(400, { error: 'Unknown org' }); + } + if (!UUID4.test(req.docid)) { + return jsonResponse(400, { error: 'Room name is invalid' }); + } + if (req.branch !== 'main') { + // as in create-ydoc: the admin token is not fenced to main by + // getAccessType, and a deletion is recorded per branch + return jsonResponse(400, { error: 'Unknown branch' }); + } + // read the deletion before undoing it: `restoreDoc` throws a plain + // Error for a document whose content was erased, and that is a + // conflict to report as one — catching around the call would turn + // every failure alike, a database outage included, into the same answer + const tombstone = await req.yhub.persistence.retrieveTombstone(req.room); + if (tombstone == null) { + // not an error: the backend restores a whole subtree, of which only + // the part that was deleted with it has anything to put back + return jsonResponse(200, { + message: 'Document is not deleted', + restored: false, + }); + } + if (tombstone.hard || tombstone.purgedAt != null) { + return jsonResponse(409, { error: 'Document content was erased' }); + } + await req.yhub.restoreDoc(req.room); + return jsonResponse(200, { + message: 'Document restored', + restored: true, + }); + }, + }, + }), ]; // Django orders the document lists by `updated_at` and no edit goes through it From 37ec23dc3e3262cfed7e9c969a58319698db7e5d Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Wed, 12 Aug 2026 12:13:35 +0200 Subject: [PATCH 48/59] =?UTF-8?q?=E2=9C=A8(backend)=20wired=20soft=20delet?= =?UTF-8?q?ion=20with=20yhub=20server?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit yhub is the source of truth, when a user delete a document, it should also be deleted in the yhub server. We call the yhub server in the perform_destroy action but also the restore endpoint of yhub when a document is restored. --- CHANGELOG.md | 13 ++ src/backend/core/api/viewsets.py | 17 +++ src/backend/core/services/yhub_services.py | 35 +++++- src/backend/core/tasks/documents.py | 59 +++++++++ .../documents/test_api_documents_delete.py | 29 +++++ .../documents/test_api_documents_restore.py | 33 +++++ .../core/tests/test_services_yhub_services.py | 48 +++++++ .../core/tests/test_tasks_documents.py | 118 ++++++++++++++++++ 8 files changed, 351 insertions(+), 1 deletion(-) create mode 100644 src/backend/core/tasks/documents.py create mode 100644 src/backend/core/tests/test_tasks_documents.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 335a51741b..c1ea4ed3c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,19 @@ and this project adheres to ### Added +- ✨(collaboration) delete a document on the collaboration server when it is + deleted in Docs, and restore it when it comes back out of the trashbin. The + content lives there now, so until it was told, the clients already editing a + deleted document went on editing it and its content outlived it. Both go + through `sync_service_deletions_in_cascade`, which walks the deleted subtree + and reports what each of its documents is now — a restored document only + brings back the part of its subtree that was deleted with it. Deleting uses + yhub's built-in `DELETE .../ydoc/` (a soft deletion: connected clients are + disconnected with the close code 4404 and every route answers 404, the + content is left untouched), restoring the new backend-internal + `POST /collaboration/restore-ydoc/v1/{org}/{docid}`, since yhub 0.6.0 has no + built-in route for it. Erasing the content for good stays out of reach, as it + is in Docs: a document that is no longer restorable is not erased either - ⬆️(collaboration) upgrade yhub to 0.6.0, which needs a schema change: a `yhub_ydoc_tombstones_v1` table (it adds document deletion) and four `*_is_reference` markers on `yhub_ydoc_v1`. Neither is optional — every diff --git a/src/backend/core/api/viewsets.py b/src/backend/core/api/viewsets.py index 88be692425..b8c78c0ba0 100644 --- a/src/backend/core/api/viewsets.py +++ b/src/backend/core/api/viewsets.py @@ -8,6 +8,7 @@ import socket import uuid from collections import defaultdict +from functools import partial from urllib.parse import unquote, urlencode, urlparse from django.conf import settings @@ -69,6 +70,7 @@ ) from core.services.yhub_services import YHubError, YHubService from core.tasks.access import reset_service_connections_in_cascade +from core.tasks.documents import sync_service_deletions_in_cascade from core.tasks.mail import send_ask_for_access_mail from core.tasks.search import trigger_batch_document_indexer from core.utils.analytics import PosthogEventName, posthog_capture @@ -820,6 +822,14 @@ def perform_destroy(self, instance): """Override to implement a soft delete instead of dumping the record in database.""" instance.soft_delete() + # the collaboration server holds the content: until it is told, it goes + # on serving the document to the clients editing it. On commit, because + # the task reads back what was just written to know what to report — it + # would find the document alive and restore it instead + transaction.on_commit( + partial(sync_service_deletions_in_cascade.delay, str(instance.id)) + ) + posthog_capture( PosthogEventName.DOC_DELETED, self.request.user, {}, document=instance ) @@ -1102,6 +1112,13 @@ def restore(self, request, *args, **kwargs): except RuntimeError as err: raise drf.exceptions.ValidationError({"detail": str(err)}) from err + # the counterpart of the deletion: the same walk puts back the content + # of the documents that came back with this one, and it reads the + # restored state back, hence on commit as well + transaction.on_commit( + partial(sync_service_deletions_in_cascade.delay, str(document.id)) + ) + return drf_response.Response( {"detail": "Document has been successfully restored."}, status=status.HTTP_200_OK, diff --git a/src/backend/core/services/yhub_services.py b/src/backend/core/services/yhub_services.py index 8ce04fa27c..214decbd1c 100644 --- a/src/backend/core/services/yhub_services.py +++ b/src/backend/core/services/yhub_services.py @@ -9,7 +9,8 @@ room is addressed as `/{prefix}/{endpoint}/{version}/{org}/{docid}`, where `org` is the yhub organization Docs runs under and `docid` the document id. The built-in endpoints are `ydoc` (get the state of a document, patch it with a Yjs -update), `rollback`, `prune`, `changeset` and `activity`, all at `v1`. yhub also +update, delete it), `rollback`, `prune`, `changeset` and `activity`, all at +`v1`. yhub also accepts a `branch` query parameter, but our auth plugin only ever grants access to the `main` branch, so this service never sends it. @@ -287,6 +288,38 @@ def create_ydoc(self, document, update): }, ) + def delete_ydoc(self, document): + """ + Delete a document on the collaboration server. + + This is what stops the clients editing a deleted document: they are + disconnected, and the collaboration server answers 404 for it from then + on. The deletion is a soft one, its content is left untouched and + `restore_ydoc` brings the document back whole. Erasing the content for + good is a separate, irreversible operation that yhub deliberately does + not expose over its REST API. + + Idempotent, and never refused: deleting a document twice keeps the date + of the first deletion, and a document the collaboration server holds + nothing for is recorded as deleted all the same. + """ + return self.request("delete", self.build_url("ydoc", document)) + + def restore_ydoc(self, document): + """ + Undo the deletion of a document on the collaboration server. + + Its content was never touched, so it comes back with its whole history. + A document that is not deleted is left alone rather than refused, which + is what lets a restored subtree be reported without asking what became + of each of its documents. + + yhub answers 409 for a document whose content was erased — there is + nothing left to bring back — reported as an `APIError` carrying the + status. + """ + return self.request("post", self.build_url("restore-ydoc", document)) + def migrate(self, document, force=False): """ Replay the legacy version history of a document into the collaboration server. diff --git a/src/backend/core/tasks/documents.py b/src/backend/core/tasks/documents.py new file mode 100644 index 0000000000..0b42976715 --- /dev/null +++ b/src/backend/core/tasks/documents.py @@ -0,0 +1,59 @@ +"""Tasks dedicated to the documents themselves.""" + +from logging import getLogger + +from core import models +from core.services.yhub_services import YHubError, YHubService + +from impress.celery_app import app + +logger = getLogger(__name__) + + +@app.task +def sync_service_deletions_in_cascade(document_id): + """ + Report the deletion of a document and of its descendants to the + collaboration server. + + The content of a document lives there, not here: until it is told, it keeps + serving a deleted document to the clients already editing it, and its + content outlives the document. The endpoint is document scoped, hence the + walk down the tree — deleting a document deletes the subtree under it. + + Restoring goes through the very same walk. A restored document brings back + only the part of its subtree that was deleted with it, the documents deleted + on their own stay deleted, so what each document of the subtree needs is + read from what it is now rather than from what was just done to it. Running + this twice therefore changes nothing, and running it late still lands on the + right answer. + + A document failing is logged and does not stop the ones after it; the + collaboration server keeps serving it until something says so again. + """ + try: + document = models.Document.objects.get(pk=document_id) + except models.Document.DoesNotExist: + logger.error("Document %s does not exists anymore", document_id) + return + + documents = models.Document.objects.filter( + path__startswith=document.path, depth__gte=document.depth + ).order_by("path") + + service = YHubService() + for doc in documents: + # a descendant carries the deletion of its ancestors, never its own + # `deleted_at`, unless it was deleted on its own beforehand + deleted = doc.deleted_at is not None or doc.ancestors_deleted_at is not None + try: + if deleted: + service.delete_ydoc(doc) + else: + service.restore_ydoc(doc) + except YHubError: + logger.exception( + "impossible to %s document %s on the collaboration server", + "delete" if deleted else "restore", + doc.id, + ) diff --git a/src/backend/core/tests/documents/test_api_documents_delete.py b/src/backend/core/tests/documents/test_api_documents_delete.py index f89503eb71..6672b13760 100644 --- a/src/backend/core/tests/documents/test_api_documents_delete.py +++ b/src/backend/core/tests/documents/test_api_documents_delete.py @@ -148,3 +148,32 @@ def test_api_documents_delete_authenticated_owner(via, mock_user_teams): {}, document=document, ) + + +def test_api_documents_delete_reports_the_deletion_to_the_collaboration_server( + django_capture_on_commit_callbacks, +): + """ + Deleting a document should tell the collaboration server, which holds its + content and would otherwise go on serving it to the clients editing it. + """ + user = factories.UserFactory() + document = factories.DocumentFactory(users=[(user, "owner")]) + child = factories.DocumentFactory(parent=document) + + client = APIClient() + client.force_login(user) + + # the report is made once the deletion is committed: the task reads it back + with ( + mock.patch("core.tasks.documents.YHubService") as mock_service, + django_capture_on_commit_callbacks(execute=True), + ): + response = client.delete(f"/api/v1.0/documents/{document.id!s}/") + + assert response.status_code == 204 + # the subtree goes with it + assert mock_service.return_value.delete_ydoc.call_args_list == [ + mock.call(document), + mock.call(child), + ] diff --git a/src/backend/core/tests/documents/test_api_documents_restore.py b/src/backend/core/tests/documents/test_api_documents_restore.py index a1343d7c7b..850a0d3098 100644 --- a/src/backend/core/tests/documents/test_api_documents_restore.py +++ b/src/backend/core/tests/documents/test_api_documents_restore.py @@ -3,6 +3,7 @@ """ from datetime import timedelta +from unittest import mock from django.utils import timezone @@ -145,3 +146,35 @@ def test_api_documents_restore_authenticated_owner_not_deleted(): document.refresh_from_db() assert document.deleted_at is None assert document.ancestors_deleted_at is None + + +def test_api_documents_restore_reports_the_restoration_to_the_collaboration_server( + django_capture_on_commit_callbacks, +): + """ + Restoring a document should tell the collaboration server, which answers + 404 for it as long as it believes it deleted. + """ + user = factories.UserFactory() + client = APIClient() + client.force_login(user) + + document = factories.DocumentFactory() + child = factories.DocumentFactory(parent=document) + factories.UserDocumentAccessFactory(document=document, user=user, role="owner") + document.soft_delete() + + # the report is made once the restoration is committed: the task reads it back + with ( + mock.patch("core.tasks.documents.YHubService") as mock_service, + django_capture_on_commit_callbacks(execute=True), + ): + response = client.post(f"/api/v1.0/documents/{document.id!s}/restore/") + + assert response.status_code == 200 + # the subtree comes back with it + assert mock_service.return_value.restore_ydoc.call_args_list == [ + mock.call(document), + mock.call(child), + ] + mock_service.return_value.delete_ydoc.assert_not_called() diff --git a/src/backend/core/tests/test_services_yhub_services.py b/src/backend/core/tests/test_services_yhub_services.py index 9379188c37..e39219252e 100644 --- a/src/backend/core/tests/test_services_yhub_services.py +++ b/src/backend/core/tests/test_services_yhub_services.py @@ -242,6 +242,54 @@ def test_migrate_forced(mock_request): assert args[1].endswith("?force=true") +@patch("requests.request") +def test_delete_ydoc(mock_request): + """Should ask yhub to delete the document, on its built-in endpoint.""" + mock_request.return_value.ok = True + + response = YHubService().delete_ydoc(DOCUMENT) + + assert response is mock_request.return_value + args, _kwargs = mock_request.call_args + assert args == ( + "delete", + f"http://yhub:3002/collaboration/ydoc/v1/docs/{DOCUMENT.id!s}", + ) + + +@patch("requests.request") +def test_restore_ydoc(mock_request): + """Should ask yhub to bring the document back, on our own endpoint.""" + mock_request.return_value.ok = True + + response = YHubService().restore_ydoc(DOCUMENT) + + assert response is mock_request.return_value + args, _kwargs = mock_request.call_args + # yhub has a built-in route to delete a document but none to restore one + assert args == ( + "post", + f"http://yhub:3002/collaboration/restore-ydoc/v1/docs/{DOCUMENT.id!s}", + ) + + +@patch("requests.request") +def test_restore_ydoc_erased_content(mock_request): + """A document whose content was erased should report the conflict it is.""" + mock_request.return_value.ok = False + mock_request.return_value.status_code = 409 + mock_request.return_value.text = '{"error": "Document content was erased"}' + mock_request.return_value.json.return_value = { + "error": "Document content was erased" + } + + with pytest.raises(APIError) as excinfo: + YHubService().restore_ydoc(DOCUMENT) + + assert excinfo.value.status_code == 409 + assert "Document content was erased" in str(excinfo.value) + + @patch("requests.request") def test_reset_connections(mock_request): """Should ask yhub to re-check every connection of the document.""" diff --git a/src/backend/core/tests/test_tasks_documents.py b/src/backend/core/tests/test_tasks_documents.py new file mode 100644 index 0000000000..47840b2127 --- /dev/null +++ b/src/backend/core/tests/test_tasks_documents.py @@ -0,0 +1,118 @@ +""" +Tests for the `sync_service_deletions_in_cascade` Celery task in the +core.tasks.documents module. +""" + +from unittest import mock + +import pytest + +from core import factories +from core.services.yhub_services import ServiceUnavailableError +from core.tasks.documents import sync_service_deletions_in_cascade + +pytestmark = pytest.mark.django_db + + +@mock.patch("core.tasks.documents.YHubService") +def test_sync_service_deletions_deletes_the_document(mock_service): + """A deleted document should be deleted on the collaboration server.""" + document = factories.DocumentFactory() + document.soft_delete() + + sync_service_deletions_in_cascade(str(document.id)) + + mock_service.return_value.delete_ydoc.assert_called_once_with(document) + mock_service.return_value.restore_ydoc.assert_not_called() + + +@mock.patch("core.tasks.documents.YHubService") +def test_sync_service_deletions_in_cascade(mock_service): + """ + Deleting a document deletes the subtree under it, so the whole subtree + should be deleted, the document itself included and its ancestors left out. + """ + parent = factories.DocumentFactory() + document = factories.DocumentFactory(parent=parent) + child = factories.DocumentFactory(parent=document) + grand_child = factories.DocumentFactory(parent=child) + factories.DocumentFactory() # a document of another tree + document.soft_delete() + + sync_service_deletions_in_cascade(str(document.id)) + + assert mock_service.return_value.delete_ydoc.call_args_list == [ + mock.call(document), + mock.call(child), + mock.call(grand_child), + ] + + +@mock.patch("core.tasks.documents.YHubService") +def test_sync_service_deletions_restores_the_document(mock_service): + """A document that is back should be restored on the collaboration server.""" + document = factories.DocumentFactory() + child = factories.DocumentFactory(parent=document) + document.soft_delete() + document.restore() + + sync_service_deletions_in_cascade(str(document.id)) + + assert mock_service.return_value.restore_ydoc.call_args_list == [ + mock.call(document), + mock.call(child), + ] + mock_service.return_value.delete_ydoc.assert_not_called() + + +@mock.patch("core.tasks.documents.YHubService") +def test_sync_service_deletions_restore_leaves_out_what_stays_deleted(mock_service): + """ + A document deleted on its own before its ancestor was stays deleted when + the ancestor comes back, and so should its content. + """ + document = factories.DocumentFactory() + child = factories.DocumentFactory(parent=document) + grand_child = factories.DocumentFactory(parent=child) + child.soft_delete() + document.soft_delete() + document.restore() + + sync_service_deletions_in_cascade(str(document.id)) + + # the subtree of the child was deleted on its own and is still deleted + assert mock_service.return_value.restore_ydoc.call_args_list == [ + mock.call(document) + ] + assert mock_service.return_value.delete_ydoc.call_args_list == [ + mock.call(child), + mock.call(grand_child), + ] + + +@mock.patch("core.tasks.documents.YHubService") +def test_sync_service_deletions_unknown_document(mock_service): + """A document deleted for good in the meantime should not reach the service.""" + sync_service_deletions_in_cascade("d43ea3c5-b8ee-4a4a-9c60-2ad7a1d9e6cf") + + mock_service.return_value.delete_ydoc.assert_not_called() + mock_service.return_value.restore_ydoc.assert_not_called() + + +@mock.patch("core.tasks.documents.YHubService") +def test_sync_service_deletions_keeps_going_on_failure(mock_service): + """A document failing should not deprive the ones after it of their deletion.""" + document = factories.DocumentFactory() + child = factories.DocumentFactory(parent=document) + document.soft_delete() + mock_service.return_value.delete_ydoc.side_effect = [ + ServiceUnavailableError("yhub is down"), + None, + ] + + sync_service_deletions_in_cascade(str(document.id)) + + assert mock_service.return_value.delete_ydoc.call_args_list == [ + mock.call(document), + mock.call(child), + ] From 99046673c15068f48ee489f3260ae60802976b0d Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Wed, 12 Aug 2026 14:21:32 +0200 Subject: [PATCH 49/59] =?UTF-8?q?=F0=9F=90=9B(frontend)=20stop=20reconnect?= =?UTF-8?q?ing=20to=20the=20websocket=20based=20on=20the=20status=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The yhub server returns custom status code when the websocket is not accessible, like 4401 when an access is removed and 4404 when a document is deleted. the websocket client now use these custom status code to stop reconnection forever. --- CHANGELOG.md | 13 ++ .../docs/doc-editor/hook/useCollaboration.tsx | 32 ++++- .../__tests__/useProviderStore.test.tsx | 126 ++++++++++++++++++ .../stores/useProviderStore.tsx | 56 +++++++- 4 files changed, 220 insertions(+), 7 deletions(-) create mode 100644 src/frontend/apps/impress/src/features/docs/doc-management/stores/__tests__/useProviderStore.test.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index c1ea4ed3c5..81fc2161b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,19 @@ and this project adheres to ## [Unreleased] +### Fixed + +- 🐛(frontend) stop reconnecting to the collaboration server when it has refused + the connection for good. Close codes 4400-4499 are a refusal, not a lost + socket — the document was deleted (4404) or the access of this connection + changed (4401) — and retrying only asked the same question again, twice a + minute, for as long as the tab stayed open. The editor now stops and refetches + the document instead: it reconnects when the document is still there (an + access upgraded from reader to editor is a refusal too, and has to reconnect + to carry its new rights) and stays closed when it is not, where the page + already tells the user what happened. Everything else — a dropped socket, a + restart, an unreachable server — keeps its retry loop untouched + ### Added - ✨(collaboration) delete a document on the collaboration server when it is diff --git a/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useCollaboration.tsx b/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useCollaboration.tsx index eb26a0cb98..aa574204d0 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useCollaboration.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-editor/hook/useCollaboration.tsx @@ -25,6 +25,8 @@ export const useCollaboration = (room: string) => { isReady, hasLostConnection, resetLostConnection, + isPermanentlyClosed, + reconnect, pauseForInactivity, resumeFromInactivity, } = useProviderStore(); @@ -45,9 +47,6 @@ export const useCollaboration = (room: string) => { * When the provider detects a lost connection, we invalidate the document query to trigger a refetch. * Because it can be because the user has access to the document that are modified * (e.g., permissions changed, document deleted, user removed) - * TODO(yhub): this invalidation used to ride on the server-side kick - * (reset-connections); without a kick API a permission change no longer - * triggers a refetch until the connection drops for another reason. */ useEffect(() => { if (hasLostConnection && room) { @@ -58,6 +57,33 @@ export const useCollaboration = (room: string) => { } }, [hasLostConnection, room, queryClient, resetLostConnection]); + /** + * The collaboration server refused the connection for good and the retry loop + * stopped, so nothing will ask again on its own: this refetch is what asks. + * + * A refusal says the answer changed, not what it changed to. The document may + * be gone, our access to it revoked, or merely upgraded from reader to editor + * — the last one has to reconnect to carry the new rights. So the connection + * comes back only when the document does, and stays closed otherwise, where + * the query error puts the page in charge of telling the user why. + */ + useEffect(() => { + if (!isPermanentlyClosed || !room) { + return; + } + + void queryClient + .invalidateQueries({ queryKey: [KEY_DOC, { id: room }] }) + .then(() => { + if ( + queryClient.getQueryState([KEY_DOC, { id: room }])?.status === + 'success' + ) { + reconnect(); + } + }); + }, [isPermanentlyClosed, room, queryClient, reconnect]); + /** * We add a broadcast task to reset the query cache * when the document visibility changes. diff --git a/src/frontend/apps/impress/src/features/docs/doc-management/stores/__tests__/useProviderStore.test.tsx b/src/frontend/apps/impress/src/features/docs/doc-management/stores/__tests__/useProviderStore.test.tsx new file mode 100644 index 0000000000..f8b2e6980f --- /dev/null +++ b/src/frontend/apps/impress/src/features/docs/doc-management/stores/__tests__/useProviderStore.test.tsx @@ -0,0 +1,126 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { useProviderStore } from '../useProviderStore'; + +/** + * A stand-in for y-websocket's provider, faithful on the two points these + * tests are about: the listeners it lets us register, and `shouldConnect`, + * which is what its retry loop reads before opening a socket again. + */ +class FakeProvider { + public shouldConnect = true; + public connect = vi.fn(() => { + this.shouldConnect = true; + }); + public disconnect = vi.fn(() => { + this.shouldConnect = false; + }); + public destroy = vi.fn(); + public awareness = { destroy: vi.fn() }; + public doc = { destroy: vi.fn() }; + + private listeners: Record void)[]> = {}; + + on(event: string, listener: (...args: unknown[]) => void) { + (this.listeners[event] ??= []).push(listener); + } + + emit(event: string, ...args: unknown[]) { + this.listeners[event]?.forEach((listener) => listener(...args)); + } +} + +let provider: FakeProvider; + +vi.mock('y-websocket', () => ({ + // a function expression, not an arrow: the store builds it with `new` + WebsocketProvider: vi.fn(function () { + return provider; + }), +})); + +const closeWith = (code: number) => + provider.emit('connection-close', { code }, provider); + +describe('useProviderStore', () => { + beforeEach(() => { + vi.useFakeTimers(); + provider = new FakeProvider(); + // the store is a module-level singleton: put it back to its defaults, or + // a test reads what the one before it left behind + useProviderStore.getState().destroyProvider(); + useProviderStore.getState().createProvider('ws://localhost', 'doc-id'); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('keeps reconnecting when the connection is merely lost', () => { + closeWith(1006); + vi.runAllTimers(); + + // y-websocket has scheduled its next attempt and nothing stops it + expect(provider.shouldConnect).toBe(true); + expect(useProviderStore.getState().isPermanentlyClosed).toBe(false); + // the document is refetched: the connection may have dropped because the + // access to it changed + expect(useProviderStore.getState().hasLostConnection).toBe(true); + }); + + it.each([ + ['a deleted document', 4404], + ['a revoked access', 4401], + ])('stops reconnecting on %s', (_label, code) => { + closeWith(code); + + // immediately, before the reconnection y-websocket has just scheduled + expect(provider.shouldConnect).toBe(false); + + vi.runAllTimers(); + + // the backend is asked what became of the document, through this rather + // than through `hasLostConnection`: it decides whether to come back + expect(useProviderStore.getState().isPermanentlyClosed).toBe(true); + expect(useProviderStore.getState().hasLostConnection).toBe(false); + expect(useProviderStore.getState().isConnected).toBe(false); + }); + + it('keeps reconnecting on a transient error of the collaboration server', () => { + // 4500-4599 is its transient range, 1013 is "try again later" + closeWith(4503); + vi.runAllTimers(); + + expect(provider.shouldConnect).toBe(true); + expect(useProviderStore.getState().isPermanentlyClosed).toBe(false); + }); + + it('does not report a close it triggered itself as permanent', () => { + // `destroy()` and `disconnect()` emit the event with no close event + provider.emit('connection-close', null, provider); + vi.runAllTimers(); + + expect(useProviderStore.getState().isPermanentlyClosed).toBe(false); + }); + + it('reopens the connection when the document is still there', () => { + closeWith(4404); + vi.runAllTimers(); + + useProviderStore.getState().reconnect(); + + expect(provider.connect).toHaveBeenCalled(); + expect(useProviderStore.getState().isPermanentlyClosed).toBe(false); + }); + + it('leaves a connection refused for good closed when the tab becomes active', () => { + closeWith(4404); + vi.runAllTimers(); + + useProviderStore.getState().pauseForInactivity(); + useProviderStore.getState().resumeFromInactivity(); + + expect(provider.connect).not.toHaveBeenCalled(); + expect(useProviderStore.getState().isPermanentlyClosed).toBe(true); + }); +}); diff --git a/src/frontend/apps/impress/src/features/docs/doc-management/stores/useProviderStore.tsx b/src/frontend/apps/impress/src/features/docs/doc-management/stores/useProviderStore.tsx index cba10a6388..b5ee44bcae 100644 --- a/src/frontend/apps/impress/src/features/docs/doc-management/stores/useProviderStore.tsx +++ b/src/frontend/apps/impress/src/features/docs/doc-management/stores/useProviderStore.tsx @@ -20,7 +20,9 @@ export interface UseCollaborationStore { isSynced: boolean; hasLostConnection: boolean; isPausedForInactivity: boolean; + isPermanentlyClosed: boolean; resetLostConnection: () => void; + reconnect: () => void; } const defaultValues = { @@ -30,6 +32,7 @@ const defaultValues = { isSynced: false, hasLostConnection: false, isPausedForInactivity: false, + isPermanentlyClosed: false, }; /** @@ -40,6 +43,21 @@ const defaultValues = { */ const RECONNECT_JITTER_MAX_MS = 3000; +/** + * Close codes 4400-4499 are the collaboration server refusing this connection + * rather than losing it: its access changed (4401) or the document was deleted + * (4404). It has answered, and reconnecting on a timer only asks the same + * question again — twice a minute, for as long as the tab stays open, for a + * document that may never come back. Everything else (a dropped socket, a + * server restart, an upgrade that failed) is transient and keeps its retry + * loop. + * + * Refused is not the same as gone: an access upgraded from reader to editor is + * a refusal too, and the connection has to be made again to carry the new + * rights. Asking the backend is what settles it, in `useCollaboration`. + */ +const isPermanentCloseCode = (code: number) => code >= 4400 && code <= 4499; + let lostConnectionTimeout: ReturnType | undefined; export const useProviderStore = create((set, get) => ({ @@ -80,7 +98,8 @@ export const useProviderStore = create((set, get) => ({ // Fires on every close AND every failed connection attempt // (an auth failure surfaces as an upgrade-level 401, close code 1006). - provider.on('connection-close', () => { + // The event is null when the socket was closed from here. + provider.on('connection-close', (event) => { // Skip when the disconnect was triggered by inactivity: // reconnection only happens once the user becomes active again. if (get().isPausedForInactivity) { @@ -93,14 +112,30 @@ export const useProviderStore = create((set, get) => ({ clearTimeout(lostConnectionTimeout); // Jitter spreading: Math.random() generates a random delay to avoid // all clients invalidating their queries at the same time + const jitter = Math.random() * RECONNECT_JITTER_MAX_MS; + + if (event && isPermanentCloseCode(event.code)) { + /** + * Stop the retry loop. Assigning `shouldConnect` rather than calling + * `disconnect()`: this runs inside y-websocket's own close handling, + * and `disconnect()` closes the socket that is already closing, which + * re-enters this listener. The reconnection it has just scheduled reads + * the flag back when it fires, and gives up. + */ + provider.shouldConnect = false; + lostConnectionTimeout = setTimeout( + () => set({ isPermanentlyClosed: true }), + jitter, + ); + return; + } + lostConnectionTimeout = setTimeout( () => set({ hasLostConnection: true }), - Math.random() * RECONNECT_JITTER_MAX_MS, + jitter, ); }); - // TODO(yhub): re-add kick handling when yhub exposes a kick API (was onClose code 1000). - set({ provider, }); @@ -139,7 +174,20 @@ export const useProviderStore = create((set, get) => ({ } clearTimeout(lostConnectionTimeout); set({ isPausedForInactivity: false }); + // a connection that was refused for good is only reopened by `reconnect`, + // once the backend has been asked again — becoming active is not an answer + if (get().isPermanentlyClosed) { + return; + } get().provider?.connect(); }, resetLostConnection: () => set({ hasLostConnection: false }), + /** + * Open the connection again after it was refused for good, once the backend + * has confirmed the document is still there to open. + */ + reconnect: () => { + set({ isPermanentlyClosed: false }); + get().provider?.connect(); + }, })); From 2aff9da569e4b2ece5fa436d1ed7b4d233b3adeb Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Wed, 12 Aug 2026 15:05:55 +0200 Subject: [PATCH 50/59] =?UTF-8?q?=E2=9C=A8(collaboration)=20erase=20conten?= =?UTF-8?q?t=20in=20yhub=20from=20clean=5Fdocument=20command?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The clean_document command makes a reset of a document deleting its content and all the attachments linked to this subdocument and its children. The hard delete api in yhub make the room, so the document id, not usable at all and this is not not what we want. We added a new custom api in yhub to manage this case, the document is hard deleted and then the Tombstone to make the room reusable again. --- CHANGELOG.md | 12 +++ .../management/commands/clean_document.py | 57 +++++++++-- src/backend/core/services/yhub_services.py | 19 ++++ .../tests/commands/test_clean_document.py | 60 ++++++++++++ .../core/tests/test_services_yhub_services.py | 18 ++++ src/yhub-server/README.md | 50 ++++++++-- src/yhub-server/server.js | 97 ++++++++++++++++++- 7 files changed, 295 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81fc2161b5..c152a45a50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,18 @@ and this project adheres to ### Added +- ✨(collaboration) erase the content of a document on the collaboration server + when `clean_document` resets it. The command cleared the database and the + object storage, but the content lives on the collaboration server now: it kept + serving the document the reset was supposed to erase, and the manual + remediation was to run SQL against yhub's own database. It goes through the + new backend-internal `POST /collaboration/reset-ydoc/v1/{org}/{docid}`, which + hard-deletes the room and then drops the deletion record so it stays usable — + the document keeps its id and goes on being edited, which is why neither of + yhub's deletions fits on its own. The documents it could not erase are named + on stderr: their content is still served, and the reset is not done until they + are dealt with. Note that erasing the room does not erase the copy each editor + holds — reset a document when nobody is editing it - ✨(collaboration) delete a document on the collaboration server when it is deleted in Docs, and restore it when it comes back out of the trashbin. The content lives there now, so until it was told, the clients already editing a diff --git a/src/backend/core/management/commands/clean_document.py b/src/backend/core/management/commands/clean_document.py index 5e041a80c8..525868b4d6 100644 --- a/src/backend/core/management/commands/clean_document.py +++ b/src/backend/core/management/commands/clean_document.py @@ -1,13 +1,5 @@ """Clean a document by resetting it (keeping its title) and deleting all descendants.""" -# TODO(yhub): this sandbox reset no longer erases the document content. It purges -# the S3 versions, but yhub durably retains the Yjs document in its own Postgres -# and re-serves it on the next websocket connect (CRDT merge with the empty -# seed resurrects the purged content). yhub has no delete API; until it grows -# one, the interim remediation is to run, against yhub's stores: -# DELETE FROM yhub_ydoc_v1 WHERE org='docs' AND docid=''; -# and drop the `yhub:room:docs::*` redis keys. - import logging from django.conf import settings @@ -28,6 +20,7 @@ LinkTrace, Thread, ) +from core.services.yhub_services import YHubError, YHubService logger = logging.getLogger("impress.commands.clean_document") @@ -150,8 +143,56 @@ def handle(self, *args, **options): logger.warning("Failed to delete S3 attachment %s", key) self.stdout.write(f"Deleted {len(all_attachment_keys)} attachment(s) from S3.") + + # After the object storage, never before: what is erased here can be + # seeded back from a legacy object that is still in the bucket, and the + # first read of the document is all it takes. + self._erase_collaboration_content(all_documents) + self.stdout.write("Done.") + def _erase_collaboration_content(self, documents): + """ + Erase the content of the documents on the collaboration server. + + That is where the content lives: without this the reset only clears the + database and the object storage, and the next editor to connect is + served the document that was supposed to be gone. + + The room is emptied and left usable rather than deleted — the root + document keeps its id and goes on being edited. Its descendants are + deleted for good by then and would not mind either way. + + The editors are disconnected, but each of them holds a copy of the + document: one that reconnects with it syncs the content back into the + emptied room. Run this when nobody is editing, and have anyone who was + reload the page. + """ + service = YHubService() + failed = [] + + for doc in documents: + try: + service.reset_ydoc(doc) + except YHubError: + logger.warning( + "Failed to erase the collaboration content of document %s", doc.id + ) + failed.append(doc.id) + + erased = len(documents) - len(failed) + self.stdout.write(f"Erased collaboration content for {erased} document(s).") + + if failed: + # loud, and by id: the content of these documents is still being + # served, so the reset is not done until they are dealt with + self.stderr.write( + "Collaboration content NOT erased for " + f"{len(failed)} document(s): {', '.join(str(id) for id in failed)}. " + "Their content is still served by the collaboration server, " + "run the command again." + ) + def _clean_root_relations(self, document): """ Delete the relations attached to the root document: accesses (except diff --git a/src/backend/core/services/yhub_services.py b/src/backend/core/services/yhub_services.py index 214decbd1c..327fa55f68 100644 --- a/src/backend/core/services/yhub_services.py +++ b/src/backend/core/services/yhub_services.py @@ -320,6 +320,25 @@ def restore_ydoc(self, document): """ return self.request("post", self.build_url("restore-ydoc", document)) + def reset_ydoc(self, document): + """ + Erase the content of a document on the collaboration server. + + The document itself stays: its room is emptied and left usable, as if + it had never been written. This is what resetting a document means once + the content lives there — deleting the room would answer 404 for a + document that goes on existing. + + Irreversible, and it is meant to be: the editors are disconnected and + the content is gone from the collaboration server for good, history + included. Only the backend can ask for it. + """ + return self.request( + "post", + self.build_url("reset-ydoc", document), + headers=self.build_user_header(self.user_id), + ) + def migrate(self, document, force=False): """ Replay the legacy version history of a document into the collaboration server. diff --git a/src/backend/core/tests/commands/test_clean_document.py b/src/backend/core/tests/commands/test_clean_document.py index 2f84684166..bcc554b4de 100644 --- a/src/backend/core/tests/commands/test_clean_document.py +++ b/src/backend/core/tests/commands/test_clean_document.py @@ -11,10 +11,23 @@ from core import choices, factories, models from core.choices import LinkReachChoices, LinkRoleChoices +from core.services.yhub_services import ServiceUnavailableError pytestmark = pytest.mark.django_db +@pytest.fixture(autouse=True, name="mock_yhub") +def mock_yhub_fixture(): + """ + Stand in for the collaboration server, which holds the content the command + erases. Autouse: every run of the command reaches it. + """ + with mock.patch( + "core.management.commands.clean_document.YHubService" + ) as mock_service: + yield mock_service.return_value + + def purged_keys(mock_storage): """ Return the set of object keys whose versions were purged from S3, i.e. the @@ -378,3 +391,50 @@ def test_clean_document_with_options(settings): child.file_key, grandchild.file_key, } + + +def test_clean_document_erases_the_collaboration_content(settings, mock_yhub): + """ + The content lives on the collaboration server, so resetting a document + means erasing it there too — for the root and for the descendants the + command deletes. + """ + settings.DEBUG = True + + root = factories.DocumentFactory(title="Root") + child = factories.DocumentFactory(parent=root) + grandchild = factories.DocumentFactory(parent=child) + + with mock.patch("core.management.commands.clean_document.default_storage"): + call_command("clean_document", str(root.id)) + + assert mock_yhub.reset_ydoc.call_args_list == [ + mock.call(root), + mock.call(child), + mock.call(grandchild), + ] + + +def test_clean_document_reports_the_documents_it_could_not_erase( + settings, mock_yhub, capsys +): + """ + A document the collaboration server would not erase is named, and does not + deprive the ones after it of their erasure: its content is still served, so + the reset is not done. + """ + settings.DEBUG = True + + root = factories.DocumentFactory(title="Root") + child = factories.DocumentFactory(parent=root) + mock_yhub.reset_ydoc.side_effect = [ServiceUnavailableError("yhub is down"), None] + + with mock.patch("core.management.commands.clean_document.default_storage"): + call_command("clean_document", str(root.id)) + + assert mock_yhub.reset_ydoc.call_args_list == [mock.call(root), mock.call(child)] + + captured = capsys.readouterr() + assert "Erased collaboration content for 1 document(s)." in captured.out + assert str(root.id) in captured.err + assert str(child.id) not in captured.err diff --git a/src/backend/core/tests/test_services_yhub_services.py b/src/backend/core/tests/test_services_yhub_services.py index e39219252e..8d780677e8 100644 --- a/src/backend/core/tests/test_services_yhub_services.py +++ b/src/backend/core/tests/test_services_yhub_services.py @@ -273,6 +273,24 @@ def test_restore_ydoc(mock_request): ) +@patch("requests.request") +def test_reset_ydoc(mock_request): + """Should ask yhub to erase the content of the document.""" + mock_request.return_value.ok = True + user = UserFactory.build() + + response = YHubService(user=user).reset_ydoc(DOCUMENT) + + assert response is mock_request.return_value + args, kwargs = mock_request.call_args + assert args == ( + "post", + f"http://yhub:3002/collaboration/reset-ydoc/v1/docs/{DOCUMENT.id!s}", + ) + # who erased the content, for the record yhub keeps of the deletion it does + assert kwargs["headers"]["X-User-Id"] == str(user.pk) + + @patch("requests.request") def test_restore_ydoc_erased_content(mock_request): """A document whose content was erased should report the conflict it is.""" diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index 1a7c9ee191..3b3e7d6075 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -46,6 +46,9 @@ It is not a fork of yhub — it is a thin wrapper: deletion of a document — admin JWT only, like `reset-connections`. Deleting one needs nothing custom, the built-in `DELETE .../ydoc/` does it (see "Deletion" below); restoring has no built-in route, +- exposes `POST /collaboration/reset-ydoc/v1/{org}/{docid}`, which erases the + content of a document and leaves its room usable — admin JWT only, and + irreversible (see "Deletion" below), - notifies the Django backend on `POST /api/v1.0/documents/{id}/content-updated/` whenever the worker persists new content for a document, so that lists ordered by `updated_at` @@ -65,9 +68,9 @@ the websocket and the built-in document APIs (`ydoc`, `rollback`, `prune`, `changeset`, `activity`) are all guarded by the same cookie-based document authorization and are meant to be reachable by browsers, as is `/collaboration/jwks/v1`, which carries public keys and nothing else. The one -exception is `/collaboration/reset-connections/`, `/collaboration/migrate/` and -`/collaboration/restore-ydoc/`, which are backend-internal and should not be -routed through the public ingress. +exception is `/collaboration/reset-connections/`, `/collaboration/migrate/`, +`/collaboration/restore-ydoc/` and `/collaboration/reset-ydoc/`, which are +backend-internal and should not be routed through the public ingress. ## Container image @@ -141,11 +144,46 @@ a subtree without asking what became of each document in it. Erasing the content for good is a third operation (`YHub.deleteDoc(room, { hard: true })`), reachable from inside this process only — yhub deliberately -keeps it off the REST API. Nothing here calls it: Docs never erases a document -either, a soft-deleted one simply stops being restorable after -`TRASHBIN_CUTOFF_DAYS`. Note that a hard deletion is final for that room — the +keeps it off the REST API. It is not what deleting a document in Docs does: a +soft-deleted one simply stops being restorable after `TRASHBIN_CUTOFF_DAYS`, +and its content is kept. Note that a hard deletion is final for that room — the docid can never be written again, and `restore-ydoc` answers 409 for it. +### Resetting (`POST /collaboration/reset-ydoc/v1/{org}/{docid}`) + +One caller does erase content: the backend's `clean_document` command, which +resets the onboarding sandbox. It empties a document rather than deleting it — +the Django document keeps its id and goes on being edited — so neither deletion +fits: a soft one answers 404 for a document that still exists, and a hard one is +final for the room. + +This endpoint hard-deletes and then drops the deletion record, which is what +leaves the room writable again. That order matters: the record is also the +barrier that refuses every write while the erasure runs, so a compaction that +was already merging cannot put the content back. Compaction is disabled for the +room around the whole sequence, and the content is read back afterwards — if it +reappeared, the erasure runs once more, and the endpoint answers 500 rather than +report an erasure it did not achieve. + +Irreversible, admin JWT only, and backend-internal. + +**Erasing a room does not erase the copies of it.** The editors are disconnected +(close code 4404), but a Yjs client holds the whole document in memory: one that +reconnects with its copy syncs it back into the empty room, and the content is +returned. The room accepting writes again is what makes this a reset rather than +a deletion, so the room itself cannot refuse them. + +Connected clients could be dealt with, and deliberately are not: broadcasting an +update that deletes everything, before the kick, empties them for good — a Yjs +client with garbage collection on (what an editor runs, Docs refuses `gc=false` +connections to users) drops the deleted content rather than keeping it as +history, so it has nothing left to push back. What that does not cover is a +client that was offline or backgrounded at that moment, which comes back with +its copy intact either way. + +So: reset a document when nobody is editing it, and have anyone who was reload +the page. + ## Soft migration (`SOFT_MIGRATION=true`) Documents were historically stored by the Django backend in the S3 media diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index 34560d20aa..f2cfc9a391 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -61,6 +61,9 @@ const UUID4 = // an empty Yjs update (what `Y.encodeStateAsUpdate(new Y.Doc())` encodes to) — // hardcoded so we don't import @y/y for two bytes const EMPTY_YDOC = new Uint8Array([0, 0]); +// yhub's "no effective content" convention: an empty update encodes to 2 bytes, +// and anything up to 3 is read as an empty document +const EMPTY_UPDATE_MAX_BYTES = 3; // uws buffers the whole body before the handler sees it, so this cap does not // bound upload memory — it bounds what a single create hands to a compute // worker and writes to the valkey stream as one message. Creates carry one @@ -69,6 +72,7 @@ const EMPTY_YDOC = new Uint8Array([0, 0]); const MAX_CREATE_BYTES = 10 * 1024 * 1024; const touchLog = logger.child({ module: 'updated-at-notifier' }); +const resetLog = logger.child({ module: 'reset-ydoc' }); const BACKEND_NOTIFY_TIMEOUT_MS = 5000; // Audience of the tokens the backend accepts from us. It must match the one @@ -354,6 +358,36 @@ const jsonResponse = (status, body) => headers: { 'content-type': 'application/json' }, }); +// Does the room hold anything? Covers persisted rows and the messages still on +// the stream, which is what makes it an answer about the content rather than +// about the storage. +const hasContent = async (yhub, room) => { + const { gcDoc } = await yhub.getDoc( + room, + { gc: true, nongc: false }, + { gcOnMerge: false }, + ); + return gcDoc != null && gcDoc.byteLength > EMPTY_UPDATE_MAX_BYTES; +}; + +// Erase every trace of a room's content and leave it writable again. +// +// The erasure is yhub's hard deletion: it clears the stream, disconnects the +// editors and drops every row and asset, irreversibly. Its tombstone is also +// the barrier that stops a compaction still in flight from writing the content +// back — every `store` is refused while it is there, and the purge runs behind +// it — so the room is only made writable again, by dropping the tombstone, +// once there is nothing left to write back. +// +// Dropping the tombstone is what makes this a reset rather than a deletion: +// yhub has no such operation, a hard deletion is final for the room and even +// `restoreDoc` refuses it. Here the document id belongs to a Django document +// that goes on living, so the room has to be usable again. +const eraseContent = async (yhub, room, by) => { + await yhub.deleteDoc(room, { hard: true, by }); + await yhub.persistence.deleteTombstone(room); +}; + const api = [ // GET /collaboration/jwks/v1 — the public keys verifying the tokens we sign // to call the backend, in the JSON Web Key Set format (RFC 7517). Global @@ -493,9 +527,9 @@ const api = [ // ("HTTP/1.1 413 ") — legal, and callers switch on the code return jsonResponse(413, { error: 'Update too large' }); } - // <= 3 bytes is yhub's "no effective content" convention (an empty - // update encodes to 2 bytes) — reject before it reaches a worker - if (update.byteLength <= 3) { + // yhub's "no effective content" convention — reject before it reaches + // a worker + if (update.byteLength <= EMPTY_UPDATE_MAX_BYTES) { return jsonResponse(400, { error: 'Empty update' }); } // covers persisted state AND uncompacted stream messages. Not atomic @@ -509,7 +543,7 @@ const api = [ { gc: true, nongc: false }, { gcOnMerge: false }, ); - if (gcDoc != null && gcDoc.byteLength > 3) { + if (gcDoc != null && gcDoc.byteLength > EMPTY_UPDATE_MAX_BYTES) { return jsonResponse(409, { error: 'Document already exists' }); } // Only the backend admin token may attribute the content to another @@ -603,6 +637,61 @@ const api = [ }, }, }), + // POST /collaboration/reset-ydoc/v1/{org}/{docid} — erase the content of a + // document and leave the room usable, as if it had never been written. + // + // What the backend's `clean_document` command needs to reset the onboarding + // sandbox: the Django document keeps its id and goes on being edited, so + // deleting the room is not an option — a hard deletion is final and even a + // soft one would answer 404 for a document that still exists. Backend-internal + // and admin-only, like the deletions it is built on: this destroys content + // with no way back. + createApiEndpoint('reset-ydoc', { + accessPurpose: 'reset', + post: { + handler: async (req) => { + if (req.org !== ORG) { + return jsonResponse(400, { error: 'Unknown org' }); + } + if (!UUID4.test(req.docid)) { + return jsonResponse(400, { error: 'Room name is invalid' }); + } + if (req.branch !== 'main') { + return jsonResponse(400, { error: 'Unknown branch' }); + } + const by = req.headers['x-user-id'] || req.authInfo.userid; + // Nothing compacts this room while the erasure runs: this drops the + // task already waiting for it and refuses to enqueue another, which + // leaves one writer to race with — a task a worker had claimed before + // this call. The tombstone barrier covers it right up to the moment + // the room is made writable again, so it can only land after that, + // and the second pass below is what picks it up. + await req.yhub.stream.disableCompaction(req.room); + try { + await eraseContent(req.yhub, req.room, by); + if (await hasContent(req.yhub, req.room)) { + resetLog.warn( + { docid: req.docid }, + 'content came back while it was being erased, erasing again', + ); + await eraseContent(req.yhub, req.room, by); + if (await hasContent(req.yhub, req.room)) { + // saying it is erased when it is not is the one answer this + // endpoint must never give + return jsonResponse(500, { + error: 'Document content came back after being erased', + }); + } + } + } finally { + // even on failure: leaving compaction off would freeze the room for + // every later edit, a worse state than the one we came to fix + await req.yhub.stream.enableCompaction(req.room); + } + return jsonResponse(200, { message: 'Document content erased' }); + }, + }, + }), ]; // Django orders the document lists by `updated_at` and no edit goes through it From ec002aebfc6fe39f654e38463774ec84e3676db3 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Thu, 13 Aug 2026 11:45:49 +0200 Subject: [PATCH 51/59] =?UTF-8?q?=E2=9C=A8(helm)=20deploy=20new=20infra=20?= =?UTF-8?q?using=20helm?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new infra we have must be configured in the helm chart. This commit all the missing templates to deploy yhub, it also automate the creation of the private keys needed by all services. --- .dockerignore | 7 + .github/workflows/docker-hub.yml | 15 + .github/workflows/ghcr.yml | 47 ++ CHANGELOG.md | 44 +- bin/Tiltfile | 13 + compose.yml | 15 +- documentation/collaboration.md | 55 ++- .../examples/helm/impress.values.yaml | 24 +- documentation/format_conversion.md | 28 +- documentation/installation/kubernetes.md | 1 + src/helm/env.d/dev/values.impress.yaml.gotmpl | 53 ++- .../env.d/feature/values.impress.yaml.gotmpl | 36 +- src/helm/impress/README.md | 114 ++++- src/helm/impress/templates/_helpers.tpl | 77 +++- .../templates/backend_cronjob_list.yaml | 11 +- .../impress/templates/backend_deployment.yaml | 11 +- src/helm/impress/templates/backend_job.yml | 11 +- .../backend_job_createsuperuser.yaml | 11 +- .../templates/backend_job_migrate.yaml | 11 +- .../templates/celery_worker_deployment.yaml | 11 +- .../templates/ingress_collaboration_api.yaml | 16 +- .../templates/ingress_collaboration_ws.yaml | 8 +- src/helm/impress/templates/jwt_keys_job.yaml | 153 +++++++ src/helm/impress/templates/jwt_keys_rbac.yaml | 63 +++ .../impress/templates/yhub_deployment.yaml | 178 ++++++++ .../impress/templates/yhub_job_init_db.yaml | 148 +++++++ src/helm/impress/templates/yhub_svc.yaml | 22 + .../templates/yprovider_deployment.yaml | 2 +- .../yprovider_deployment_converter.yaml | 189 -------- .../templates/yprovider_svc_converter.yaml | 25 -- src/helm/impress/values.yaml | 409 ++++++++++++++---- src/yhub-server/Dockerfile | 24 +- src/yhub-server/README.md | 10 +- 33 files changed, 1472 insertions(+), 370 deletions(-) create mode 100644 src/helm/impress/templates/jwt_keys_job.yaml create mode 100644 src/helm/impress/templates/jwt_keys_rbac.yaml create mode 100644 src/helm/impress/templates/yhub_deployment.yaml create mode 100644 src/helm/impress/templates/yhub_job_init_db.yaml create mode 100644 src/helm/impress/templates/yhub_svc.yaml delete mode 100644 src/helm/impress/templates/yprovider_deployment_converter.yaml delete mode 100644 src/helm/impress/templates/yprovider_svc_converter.yaml diff --git a/.dockerignore b/.dockerignore index dc3ed9896d..e9235269a9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,6 +5,10 @@ __pycache__ **/*.pyc venv .venv +# the pattern above only matches at the root, and every image is built from +# there: without this one, the backend virtualenv travels to the daemon on +# every build +**/.venv # System-specific files .DS_Store @@ -34,4 +38,7 @@ db.sqlite3 # Frontend node_modules +# same as .venv above: nested ones are not matched by the pattern above, and no +# image copies them — every one of them runs its own install +**/node_modules **/.next diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index b0a5aed25b..984b738a1d 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -60,11 +60,26 @@ jobs: should_push: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'preview') }} docker_user: 1001:127 + build-and-push-yhub: + uses: ./.github/workflows/docker-publish.yml + permissions: + contents: read + secrets: inherit + with: + image_name: lasuite/impress-yhub + context: . + file: src/yhub-server/Dockerfile + target: yhub + should_push: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'preview') }} + # no docker_user: the image defaults to uid 1000, the `node` user the + # base image already declares in /etc/passwd + notify-argocd: needs: - build-and-push-backend - build-and-push-frontend - build-and-push-y-provider + - build-and-push-yhub runs-on: ubuntu-latest if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'preview') steps: diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml index ac2fb71d06..00040dd638 100644 --- a/.github/workflows/ghcr.yml +++ b/.github/workflows/ghcr.yml @@ -158,3 +158,50 @@ jobs: run: | docker system prune -af docker volume prune -f + + build-and-push-yhub: + runs-on: ubuntu-latest + if: github.event.repository.fork == true + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - name: Set up QEMU + uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4 + - name: Docker meta + id: meta + uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6 + with: + images: ${{ env.REGISTRY }}/${{ github.repository }}/yhub + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + - name: Login to GHCR + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7 + with: + context: . + file: ./src/yhub-server/Dockerfile + target: yhub + platforms: linux/amd64,linux/arm64 + build-args: DOCKER_USER=${{ env.DOCKER_USER }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + - name: Cleanup Docker after build + if: always() + run: | + docker system prune -af + docker volume prune -f diff --git a/CHANGELOG.md b/CHANGELOG.md index c152a45a50..74a57ce555 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -143,6 +143,24 @@ and this project adheres to on its own - 🔧(dev) generate the JWT signing key of the collaboration server when bootstrapping the dev stack, alongside the backend one +- ✨(helm) generate the JWT signing keys of the services on the cluster + (`jwtKeys.enabled`, off by default): a job generates the backend and the + collaboration server keys with `openssl`, hands them to a secret both mount + read-only, and sets the `*_FILE` variables pointing at them. No key is + templated into a manifest or kept in a values file, and the job is the only + thing granted a write: its role may create a secret and read whether that one + exists, nothing else, and the services never call the kubernetes API. + Idempotent — an existing secret is left alone, so it re-runs on every sync, + and rolling the keys is deleting the secret and letting the next run create + it again. `jwtKeys.existingSecret` points at keys of your own instead, and + skips both the job and its rights +- ✨(helm) deploy the collaboration server: the chart gains a `yhub` deployment, + its service, and the job running the `init-db` script that creates and + upgrades its schema — next to the backend migrate job, retrying while the + postgres server does not answer, since nothing in the chart creates it. + Configured under the `yhub` values key, where `REDIS` and `POSTGRES` are + required and have no default; the image is published as + `lasuite/impress-yhub` - ✨(backend) serve `documents/{id}/formatted-content/` from yhub - ✨(backend) duplicate a document through the collaboration server - ✨(backend) call YHubService to seed initial document content @@ -204,13 +222,31 @@ and this project adheres to - 💥(backend) remove the `documents/{id}/content/` endpoint - 💥(backend) remove the `documents/{id}/can-edit/` endpoint - 💥(y-provider) the published `lasuite/impress-y-provider` image becomes - converter-only and no longer serves `/collaboration/ws/`; deployments using - the existing helm values lose collaboration until the helm chart routes - collaboration to yhub (follow-up) + converter-only and no longer serves `/collaboration/ws/` +- 💥(helm) route `/collaboration/` to yhub instead of the y-provider: both + collaboration ingresses now point at the yhub service, and + `ingressCollaborationApi` serves the routes yhub exposes to browsers + (`ingressCollaborationApi.paths`, one ingress rule each) instead of the + single `/collaboration/api/` path — what is not listed stays in-cluster, so + `create-ydoc`, `reset-connections`, `migrate`, `restore-ydoc` and + `reset-ydoc` are not published. The `upstream-hash-by: $arg_room` annotation + is dropped: yhub replicas exchange updates through redis, so a room needs no + sticky upstream — and hashing on a query argument its urls do not carry would + pin every connection to a single pod +- 💥(helm) drop the `yProvider.converter` values, its deployment and its + service: the y-provider serves nothing but the conversion API since the + collaboration moved to yhub, so the `yProvider` release *is* the converter + and there is no second one to enable. Deployments that had it on lose the + `-converter` suffix on the url the backend calls — + `Y_PROVIDER_API_BASE_URL: http://impress-docs-y-provider:443/api/` — and + `yProvider.converter.*` values are now ignored, their `yProvider.*` + counterparts taking over - 🔧(collaboration) split the yhub image into a development and a production stage, like the other services: the dev stack now bind-mounts `src/yhub-server` and runs the server through nodemon, so editing a source - file restarts it instead of needing `make build-yhub` + file restarts it instead of needing `make build-yhub`. The production stage + gains the un-privileged user and the entrypoint the other images have, and + both are now built from the repository root like the rest of them ## [v5.4.1] - 2026-07-09 diff --git a/bin/Tiltfile b/bin/Tiltfile index edcea0a7a6..032e234284 100644 --- a/bin/Tiltfile +++ b/bin/Tiltfile @@ -42,8 +42,21 @@ docker_build( ] ) +docker_build( + 'localhost:5001/impress-yhub:latest', + context='..', + dockerfile='../src/yhub-server/Dockerfile', + only=['./src/yhub-server', './docker', './.dockerignore'], + target = 'yhub', + build_args={'DOCKER_USER': '1000:1000'}, + live_update=[ + sync('../src/yhub-server', '/app'), + ] +) + k8s_resource('impress-docs-backend-migrate', resource_deps=['dev-backend-postgres']) k8s_resource('impress-docs-backend-createsuperuser', resource_deps=['impress-docs-backend-migrate']) +k8s_resource('impress-docs-yhub-init-db', resource_deps=['dev-backend-postgres']) k8s_resource('dev-backend-keycloak', resource_deps=['dev-backend-keycloak-pg']) k8s_resource('impress-docs-backend', resource_deps=['impress-docs-backend-migrate', 'dev-backend-redis', 'dev-backend-keycloak', 'dev-backend-postgres', 'dev-backend-minio:statefulset']) k8s_yaml(local('cd ../src/helm && helmfile -n impress -e dev template .')) diff --git a/compose.yml b/compose.yml index 3c4b63239f..a53036eb70 100644 --- a/compose.yml +++ b/compose.yml @@ -235,8 +235,8 @@ services: yhub: user: ${DOCKER_USER:-1000} build: - context: ./src/yhub-server - dockerfile: Dockerfile + context: . + dockerfile: ./src/yhub-server/Dockerfile target: yhub-development image: impress:yhub-development environment: @@ -255,6 +255,11 @@ services: - env.d/development/common.local volumes: - ./data/jwt:/data/jwt:ro + # editing a source file restarts the server (nodemon), no rebuild + - ./src/yhub-server:/app + # node_modules is installed in the image, not in the source tree: keep + # the bind mount above from hiding it + - /app/node_modules restart: unless-stopped ports: - "3002:3002" @@ -267,12 +272,6 @@ services: # starting before minio would cache 401s for the first accessed docs minio: condition: service_healthy - volumes: - # editing a source file restarts the server (nodemon), no rebuild - - ./src/yhub-server:/app - # node_modules is installed in the image, not in the source tree: keep - # the bind mount above from hiding it - - /app/node_modules kc_postgresql: image: postgres:14.3 diff --git a/documentation/collaboration.md b/documentation/collaboration.md index 873a87a808..68e8f35e58 100644 --- a/documentation/collaboration.md +++ b/documentation/collaboration.md @@ -1,12 +1,65 @@ # Collaboration -By default with Docs, collaboration is enabled. To allow the collaboration between users, a connection to a websocket server is made (the y-provider service), you only have to configure the Django backend URL and the allowed origin in your y-provider service: +By default with Docs, collaboration is enabled. To allow the collaboration between users, a connection to a websocket server is made (the yhub service), you only have to configure the Django backend URL and the allowed origin in your yhub service: ```yaml COLLABORATION_BACKEND_BASE_URL: https://{yourdocsdomain.tld} COLLABORATION_SERVER_ORIGIN: https://{yourdocsdomain.tld} ``` +The collaboration server keeps the live state of a document in Redis and persists it to a PostgreSQL database of its own, so it needs both: + +```yaml +REDIS: redis://{redis-host}:6379/0 +POSTGRES: postgres://{user}:{password}@{postgres-host}:5432/yhub +``` + +Nothing creates that schema at startup: the server never runs DDL. Run the script yhub ships (`npm run init-db`, which the helm chart runs as a job) once before starting it, and again after every upgrade that adds a table. It creates the database when it is missing, it is idempotent, and until it has run every document read fails with `relation "..." does not exist`. + +The Django backend reads and writes document content there too, so point it at the service: + +```yaml +YHUB_API_BASE_URL: http://{yhub-service}:443 +``` + +Prefer the internal service url: the routes the backend calls are not meant to be reachable from the outside. Route `/collaboration/ws/` to the service publicly — that is the one the browsers open — plus the document routes (`/collaboration/ydoc/`, `rollback`, `prune`, `changeset`, `activity`) and `/collaboration/jwks/`, which carries public keys and nothing else. Keep `reset-connections`, `migrate`, `restore-ydoc`, `reset-ydoc` and `create-ydoc` in-cluster. + +Both directions are authenticated with short-lived RS256 JWTs rather than a shared secret, and each side verifies the other against the JWKS it publishes — so both need a signing key of their own, and neither needs a copy of the other's: + +```yaml +# Django +JWT_PRIVATE_KEY_FILE: /path/to/backend-private.pem +# yhub +YHUB_JWT_PRIVATE_KEY_FILE: /path/to/yhub-private.pem +``` + +They are ordinary PKCS#8 RSA keys (`openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048`), and rolling one needs no change on the other side. Without them the documents still open and edit, but the backend cannot create, delete or restore a document's content, and yhub cannot tell it that a document changed — its `updated_at` stops following the edits. + +### Generating them on the cluster + +The helm chart generates both for you, so that no key has to be created by hand, put in a values file or in a secret: + +```yaml +jwtKeys: + enabled: true +``` + +A job then creates the two keys, once, in a secret every service mounts read-only, and points the backend and yhub at them. It generates them with `openssl` in a pod-local volume and hands them to `kubectl create secret`, so they never touch a disk, a manifest or a values file. The secret is left alone when it is already there, so the job is safe to re-run — it runs on every sync — and rolling the keys is deleting the secret and letting the next run create it again. Both sides follow: they pick the verification key by its `kid` and fetch the set again when they meet one they do not know. + +The job is the only thing allowed near that secret: the chart gives it a service account whose role can `create` a secret and read whether that one exists, nothing more. The services never call the kubernetes API — they read a mounted file. The secret is not part of the release either, so uninstalling keeps the same identities; delete the secret to start over. + +Deployments already holding their keys in a secret of their own point the chart at it instead, and the job and its rights are not created at all: + +```yaml +jwtKeys: + enabled: true + existingSecret: my-jwt-keys # holding private.pem and yhub-private.pem +``` + +Setting `JWT_PRIVATE_KEY_FILE` or `YHUB_JWT_PRIVATE_KEY_FILE` yourself keeps priority over what the job provides, so a deployment holding its keys in a secret of its own can leave `jwtKeys` disabled and mount them where it wants. + +Several replicas can serve the same document: they exchange updates through Redis, so no sticky routing is needed on the websocket ingress. + ## What happens when connection to the websocket is not allowed? When multiple users access a Docs and the connection to the websocket is not allowed, then they will be in a situation where they can lose data. diff --git a/documentation/examples/helm/impress.values.yaml b/documentation/examples/helm/impress.values.yaml index a6d26dab53..6eb1b24169 100644 --- a/documentation/examples/helm/impress.values.yaml +++ b/documentation/examples/helm/impress.values.yaml @@ -67,7 +67,9 @@ backend: AWS_STORAGE_BUCKET_NAME: docs-media-storage STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage USER_RECONCILIATION_FORM_URL: https://docs.127.0.0.1.nip.io - Y_PROVIDER_API_BASE_URL: http://impress-y-provider:443/api/ + # the collaboration server, reached in-cluster + YHUB_API_BASE_URL: http://impress-docs-yhub:443 + Y_PROVIDER_API_BASE_URL: http://impress-docs-y-provider:443/api/ Y_PROVIDER_API_KEY: my-secret CACHES_KEY_PREFIX: "{{ now | unixEpoch }}" migrate: @@ -135,6 +137,26 @@ yProvider: COLLABORATION_LOGGING: true COLLABORATION_SERVER_ORIGIN: https://docs.127.0.0.1.nip.io +# The collaboration server: it serves everything under /collaboration/, the +# websocket included. It keeps the live state of a document in redis and +# persists it to a PostgreSQL database of its own, created by the init-db job +# the chart ships — give the user in POSTGRES the right to create it, or create +# the database yourself beforehand. +yhub: + replicas: 1 + + image: + repository: lasuite/impress-yhub + pullPolicy: Always + tag: "latest" + + envVars: + POSTGRES: postgres://dinum:pass@postgresql-dev-backend-postgres:5432/yhub + REDIS: redis://user:pass@redis-dev-backend-redis:6379/2 + REDIS_PREFIX: yhub + COLLABORATION_BACKEND_BASE_URL: https://docs.127.0.0.1.nip.io + COLLABORATION_SERVER_ORIGIN: https://docs.127.0.0.1.nip.io + ingress: enabled: true host: docs.127.0.0.1.nip.io diff --git a/documentation/format_conversion.md b/documentation/format_conversion.md index 828b82b1e4..2ea13be7c7 100644 --- a/documentation/format_conversion.md +++ b/documentation/format_conversion.md @@ -26,33 +26,17 @@ COLLABORATION_BACKEND_BASE_URL: http://{django-service}:8000 The JWKS url defaults to `{COLLABORATION_BACKEND_BASE_URL}/api/v1.0/jwks`; override it with `JWKS_URL` if Django is not reachable at that base url from the y-provider service. -### Splitting conversion service +### One service, not two anymore -The conversion service is present in the `y-provider` server. The same server used to manage websockets. You can split in one side the websocket server and in an other side the converter service. -This feature is only available in our helm chart, if you are deploying an other way you can take example of what is made to implement it. -The idea is to deploy twice the `y-provider` server, one dedicated for websockets and one dedicated to the conversion. +The `y-provider` server used to serve the websockets as well, which is why it could be deployed twice — one release for the collaboration, one for the conversion (`yProvider.converter`). The collaboration is served by [yhub](collaboration.md) now, so the conversion is all that is left: the `y-provider` service **is** the converter, and the `yProvider.converter` values are gone. -In the helm chart, you can use this value that will do the job for you: - -```yaml -yProvider: - converter: - enabled: true -``` - -Every parameter in the `yProvider` key can be overridden in the `yProvider.converter` key. - -Once enabled, you have to enable the `Y_PROVIDER_API_BASE_URL` with the url of the newly created service, it is the same as before with `-converter` at the end. -If before it was - -```yaml -Y_PROVIDER_API_BASE_URL: http://impress-docs-y-provider:443/api/ -``` - -now it is +A deployment coming from a chart older than this one has one thing to change, the url the backend calls, which loses its suffix: ```yaml +# before Y_PROVIDER_API_BASE_URL: http://impress-docs-y-provider-converter:443/api/ +# now +Y_PROVIDER_API_BASE_URL: http://impress-docs-y-provider:443/api/ ``` ## Docspec configuration diff --git a/documentation/installation/kubernetes.md b/documentation/installation/kubernetes.md index 41410e76b7..e87eb71996 100644 --- a/documentation/installation/kubernetes.md +++ b/documentation/installation/kubernetes.md @@ -226,6 +226,7 @@ impress-docs-backend-8494fb797d-8k8wt 1/1 Running 0 6m45s impress-docs-celery-worker-764b5dd98f-9qd6v 1/1 Running 0 6m45s impress-docs-frontend-5b69b65cc4-s8pps 1/1 Running 0 6m45s impress-docs-y-provider-5fc7ccd8cc-6ttrf 1/1 Running 0 6m45s +impress-docs-yhub-6d84f9b7c5-2xqzp 1/1 Running 0 6m45s keycloak-dev-backend-keycloak-0 1/1 Running 0 24m keycloak-dev-backend-keycloak-pg-0 1/1 Running 0 24m minio-dev-backend-minio-0 1/1 Running 0 8m24s diff --git a/src/helm/env.d/dev/values.impress.yaml.gotmpl b/src/helm/env.d/dev/values.impress.yaml.gotmpl index a4f06e27d9..df87cc03a7 100644 --- a/src/helm/env.d/dev/values.impress.yaml.gotmpl +++ b/src/helm/env.d/dev/values.impress.yaml.gotmpl @@ -71,7 +71,10 @@ backend: STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage DOCSPEC_API_URL: http://impress-docs-docspec:4000/conversion USER_RECONCILIATION_FORM_URL: https://docs.127.0.0.1.nip.io - Y_PROVIDER_API_BASE_URL: http://impress-docs-y-provider-converter:443/api/ + # the collaboration server, reached in-cluster: the backend reads and + # writes document content there, and fetches its JWKS from the same host + YHUB_API_BASE_URL: http://impress-docs-yhub:443 + Y_PROVIDER_API_BASE_URL: http://impress-docs-y-provider:443/api/ Y_PROVIDER_API_KEY: my-secret CACHES_KEY_PREFIX: "{{ now | unixEpoch }}" django: @@ -159,11 +162,6 @@ frontend: runAsNonRoot: false yProvider: - - converter: - enabled: true - replicas: 2 - replicas: 1 image: @@ -192,6 +190,47 @@ yProvider: - key: cacert.pem path: cacert.pem +# The keys the backend and the collaboration server sign the calls they make to +# each other with, generated on the cluster by a job into a secret both mount +# read-only. +jwtKeys: + enabled: true + +yhub: + replicas: 1 + + image: + repository: localhost:5001/impress-yhub + pullPolicy: Always + tag: "latest" + + envVars: + # its own logical database on the dev-backend postgres: the init-db job + # creates it, the backend never touches it + POSTGRES: postgres://dinum:pass@dev-backend-postgres:5432/yhub + # a redis database of its own too — the backend cache and celery live in /1 + REDIS: redis://user:pass@dev-backend-redis:6379/2 + REDIS_PREFIX: yhub + COLLABORATION_BACKEND_BASE_URL: https://docs.127.0.0.1.nip.io + COLLABORATION_SERVER_ORIGIN: https://docs.127.0.0.1.nip.io + NODE_EXTRA_CA_CERTS: /cert/cacert.pem + # YHUB_JWT_PRIVATE_KEY_FILE comes from the jwtKeys job below + + # Extra volume mounts to manage our local custom CA and avoid to set ssl_verify: false + extraVolumeMounts: + - name: certs + mountPath: /cert/cacert.pem + subPath: cacert.pem + + # Extra volumes to manage our local custom CA and avoid to set ssl_verify: false + extraVolumes: + - name: certs + configMap: + name: certifi + items: + - key: cacert.pem + path: cacert.pem + docSpec: enabled: true replicas: 1 @@ -218,7 +257,7 @@ ingressCollaborationWS: host: docs.127.0.0.1.nip.io ingressCollaborationApi: - enabled: true + enabled: false host: docs.127.0.0.1.nip.io ingressAdmin: diff --git a/src/helm/env.d/feature/values.impress.yaml.gotmpl b/src/helm/env.d/feature/values.impress.yaml.gotmpl index 2579b985db..e2ff44aca6 100644 --- a/src/helm/env.d/feature/values.impress.yaml.gotmpl +++ b/src/helm/env.d/feature/values.impress.yaml.gotmpl @@ -71,7 +71,10 @@ backend: STORAGES_STATICFILES_BACKEND: django.contrib.staticfiles.storage.StaticFilesStorage DOCSPEC_API_URL: http://impress-docs-docspec:4000/conversion USER_RECONCILIATION_FORM_URL: https://{{ .Values.feature }}-docs.{{ .Values.domain }} - Y_PROVIDER_API_BASE_URL: http://impress-docs-y-provider-converter:443/api/ + # the collaboration server, reached in-cluster: the backend reads and + # writes document content there, and fetches its JWKS from the same host + YHUB_API_BASE_URL: http://impress-docs-yhub:443 + Y_PROVIDER_API_BASE_URL: http://impress-docs-y-provider:443/api/ Y_PROVIDER_API_KEY: my-secret CACHES_KEY_PREFIX: "{{ now | unixEpoch }}" migrate: @@ -137,11 +140,6 @@ frontend: tag: *tag yProvider: - - converter: - enabled: true - replicas: 1 - replicas: 1 image: @@ -155,6 +153,30 @@ yProvider: COLLABORATION_SERVER_ORIGIN: https://{{ .Values.feature }}-docs.{{ .Values.domain }} NODE_OPTIONS: "--max-old-space-size=1024" +# The keys the backend and the collaboration server sign the calls they make to +# each other with, generated on the cluster by a job into a secret both mount +# read-only. +jwtKeys: + enabled: true + +yhub: + replicas: 1 + + image: + repository: lasuite/impress-yhub + pullPolicy: Always + tag: *tag + + envVars: + # its own logical database on the dev-backend postgres, created by the + # init-db job; redis /2, the backend cache and celery live in /1 + POSTGRES: postgres://dinum:pass@dev-backend-postgres:5432/yhub + REDIS: redis://user:pass@dev-backend-redis:6379/2 + REDIS_PREFIX: yhub + COLLABORATION_BACKEND_BASE_URL: https://{{ .Values.feature }}-docs.{{ .Values.domain }} + COLLABORATION_SERVER_ORIGIN: https://{{ .Values.feature }}-docs.{{ .Values.domain }} + NODE_OPTIONS: "--max-old-space-size=1024" + docSpec: enabled: true replicas: 1 @@ -182,7 +204,7 @@ ingressCollaborationWS: host: {{ .Values.feature }}-docs.{{ .Values.domain }} ingressCollaborationApi: - enabled: true + enabled: false host: {{ .Values.feature }}-docs.{{ .Values.domain }} ingressAdmin: diff --git a/src/helm/impress/README.md b/src/helm/impress/README.md index 5e8a97f56a..568e971eab 100644 --- a/src/helm/impress/README.md +++ b/src/helm/impress/README.md @@ -38,7 +38,6 @@ | `ingressCollaborationWS.annotations.nginx.ingress.kubernetes.io/enable-websocket` | | `true` | | `ingressCollaborationWS.annotations.nginx.ingress.kubernetes.io/proxy-read-timeout` | | `86400` | | `ingressCollaborationWS.annotations.nginx.ingress.kubernetes.io/proxy-send-timeout` | | `86400` | -| `ingressCollaborationWS.annotations.nginx.ingress.kubernetes.io/upstream-hash-by` | | `$arg_room` | | `ingressRedirects.enabled` | whether to enable the Ingress Redirects or not | `false` | | `ingressRedirects.className` | IngressClass to use for the Ingress Redirects | `nil` | | `ingressRedirects.host` | Host for the Ingress Redirects | `impress.example.com` | @@ -51,13 +50,13 @@ | `ingressCollaborationApi.className` | IngressClass to use for the Ingress | `nil` | | `ingressCollaborationApi.host` | Host for the Ingress | `impress.example.com` | | `ingressCollaborationApi.path` | Path to use for the Ingress | `/collaboration/api/` | +| `ingressCollaborationApi.paths` | Paths to route to the collaboration server, one rule each | `["/collaboration/ydoc/","/collaboration/jwks/"]` | | `ingressCollaborationApi.hosts` | Additional host to configure for the Ingress | `[]` | | `ingressCollaborationApi.tls.enabled` | Whether to enable TLS for the Ingress | `true` | | `ingressCollaborationApi.tls.secretName` | Secret name for TLS config | `nil` | | `ingressCollaborationApi.tls.additional[].secretName` | Secret name for additional TLS config | | | `ingressCollaborationApi.tls.additional[].hosts[]` | Hosts for additional TLS config | | | `ingressCollaborationApi.customBackends` | Add custom backends to ingress | `[]` | -| `ingressCollaborationApi.annotations.nginx.ingress.kubernetes.io/upstream-hash-by` | | `$arg_room` | | `ingressAdmin.enabled` | whether to enable the Ingress or not | `false` | | `ingressAdmin.className` | IngressClass to use for the Ingress | `nil` | | `ingressAdmin.host` | Host for the Ingress | `impress.example.com` | @@ -301,13 +300,122 @@ | `yProvider.pdb.enabled` | Enable pdb on yProvider | `true` | | `yProvider.serviceAccountName` | Optional service account name to use for yProvider pods | `nil` | +### JWT signing keys + +| Name | Description | Value | +| ------------------------------------------------------ | ------------------------------------------------------------------------------------ | -------------------- | +| `jwtKeys.enabled` | Generate the JWT signing keys of the services on the cluster | `false` | +| `jwtKeys.existingSecret` | Secret already holding the keys, generated in a secret of the chart's own when empty | `nil` | +| `jwtKeys.mountPath` | Path the keys are mounted at, in every service reading them | `/data/jwt` | +| `jwtKeys.backendKeyFilename` | Name of the key signing the tokens the backend issues | `private.pem` | +| `jwtKeys.yhubKeyFilename` | Name of the key signing the calls the collaboration server makes to the backend | `yhub-private.pem` | +| `jwtKeys.keySize` | Size, in bits, of the generated RSA keys | `2048` | +| `jwtKeys.rbac.create` | Create the service account and the role the job needs to create the secret | `true` | +| `jwtKeys.image.repository` | Repository to use to pull the image generating the keys | `alpine/openssl` | +| `jwtKeys.image.tag` | Tag of the image generating the keys | `3.5.7` | +| `jwtKeys.image.pullPolicy` | Pull policy of the image generating the keys | `IfNotPresent` | +| `jwtKeys.kubectlImage.repository` | Repository to use to pull the image handing the keys to the secret | `dtzar/helm-kubectl` | +| `jwtKeys.kubectlImage.tag` | Tag of the image handing the keys to the secret | `3.16.2` | +| `jwtKeys.kubectlImage.pullPolicy` | Pull policy of the image handing the keys to the secret | `IfNotPresent` | +| `jwtKeys.job.podSecurityContext` | Pod security context of the generating job | `{}` | +| `jwtKeys.job.securityContext.allowPrivilegeEscalation` | Whether to allow privilege escalation for the job containers | `false` | +| `jwtKeys.job.securityContext.capabilities.drop` | List of capabilities to drop for the job containers | `["ALL"]` | +| `jwtKeys.job.securityContext.runAsNonRoot` | Whether to run the job containers as a non-root user | `true` | +| `jwtKeys.job.securityContext.runAsUser` | User the job containers run as, their images declaring none | `1000` | +| `jwtKeys.job.securityContext.runAsGroup` | Group the job containers run as | `1000` | +| `jwtKeys.job.securityContext.seccompProfile.type` | Seccomp profile type for the job containers | `RuntimeDefault` | +| `jwtKeys.job.restartPolicy` | Restart policy of the generating job | `Never` | +| `jwtKeys.job.backoffLimit` | Numbers of generating job retries | `2` | +| `jwtKeys.job.ttlSecondsAfterFinished` | Period to wait before removing the generating job | `30` | +| `jwtKeys.job.generateCommand` | Override the command generating the keys | `[]` | +| `jwtKeys.job.publishCommand` | Override the command creating the secret from the generated keys | `[]` | +| `jwtKeys.job.annotations` | Annotations to add to the generating job | `{}` | +| `jwtKeys.job.podAnnotations` | Annotations to add to the generating job Pod | `{}` | +| `jwtKeys.job.resources` | Resource requirements for the job containers | `{}` | +| `jwtKeys.job.nodeSelector` | Node selector for the generating job Pod | `{}` | +| `jwtKeys.job.tolerations` | Tolerations for the generating job Pod | `[]` | +| `jwtKeys.job.affinity` | Affinity for the generating job Pod | `{}` | +| `jwtKeys.job.serviceAccountName` | Service account of the generating job Pod, the one created above when empty | `nil` | + +### yhub + +| Name | Description | Value | +| -------------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | ------------------------ | +| `yhub.enabled` | Enable the yhub collaboration server, its service and its init-db job | `true` | +| `yhub.image.repository` | Repository to use to pull the yhub container image | `lasuite/impress-yhub` | +| `yhub.image.tag` | yhub container tag | `latest` | +| `yhub.image.pullPolicy` | yhub container image pull policy | `IfNotPresent` | +| `yhub.command` | Override the yhub container command | `[]` | +| `yhub.args` | Override the yhub container args | `[]` | +| `yhub.replicas` | Amount of yhub replicas | `3` | +| `yhub.shareProcessNamespace` | Enable share process namespace between containers | `false` | +| `yhub.sidecars` | Add sidecars containers to yhub deployment | `[]` | +| `yhub.terminationGracePeriodSeconds` | Grace period given to a yhub pod to drain before it is killed | `60` | +| `yhub.securityContext.allowPrivilegeEscalation` | Whether to allow privilege escalation for the yhub container | `false` | +| `yhub.securityContext.capabilities.drop` | List of capabilities to drop for the yhub container | `["ALL"]` | +| `yhub.securityContext.runAsNonRoot` | Whether to run the yhub container as a non-root user | `true` | +| `yhub.securityContext.runAsUser` | User the yhub container runs as | `1000` | +| `yhub.securityContext.runAsGroup` | Group the yhub container runs as | `1000` | +| `yhub.securityContext.seccompProfile.type` | Seccomp profile type for the yhub container | `RuntimeDefault` | +| `yhub.envVars` | Configure yhub container environment variables | `undefined` | +| `yhub.envVars.REDIS` | Required, redis/valkey url holding the live document state (e.g. redis://valkey:6379/0) | | +| `yhub.envVars.POSTGRES` | Required, url of the yhub database, created by the init-db job (e.g. postgres://user:pass@postgres:5432/yhub) | | +| `yhub.envVars.REDIS_PREFIX` | Namespace of the redis keys, when the instance is shared (default: yhub) | | +| `yhub.envVars.COLLABORATION_BACKEND_BASE_URL` | Base url of the Docs backend, which yhub asks about users and document access rights | | +| `yhub.envVars.COLLABORATION_SERVER_ORIGIN` | Comma separated list of the origins allowed to open a websocket | | +| `yhub.envVars.YHUB_JWT_PRIVATE_KEY_FILE` | Path to the RSA private key (PEM) yhub signs its calls to the backend with, mounted from a secret | | +| `yhub.envVars.SOFT_MIGRATION` | Set to "true" to seed rooms from the legacy Django/S3 document store on first access | | +| `yhub.envVars.BY_VALUE` | Example environment variable by setting value directly | | +| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name` | Name of a ConfigMap when configuring env vars from a ConfigMap | | +| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.key` | Key within a ConfigMap when configuring env vars from a ConfigMap | | +| `yhub.envVars.FROM_SECRET.secretKeyRef.name` | Name of a Secret when configuring env vars from a Secret | | +| `yhub.envVars.FROM_SECRET.secretKeyRef.key` | Key within a Secret when configuring env vars from a Secret | | +| `yhub.podAnnotations` | Annotations to add to the yhub Pod | `{}` | +| `yhub.dpAnnotations` | Annotations to add to the yhub Deployment | `{}` | +| `yhub.initDbJobAnnotations` | Annotations for the yhub init-db job | `{}` | +| `yhub.jobs.ttlSecondsAfterFinished` | Period to wait before removing the init-db job | `30` | +| `yhub.jobs.backoffLimit` | Numbers of init-db job retries | `2` | +| `yhub.initDb.enabled` | Run the job creating and upgrading the yhub schema | `true` | +| `yhub.initDb.command` | Override the command creating and upgrading the yhub schema | `[]` | +| `yhub.initDb.retries` | How many times the schema script is retried while the postgres server does not answer | `60` | +| `yhub.initDb.retryDelaySeconds` | Seconds between two attempts | `5` | +| `yhub.initDb.restartPolicy` | Restart policy of the init-db job | `Never` | +| `yhub.service.type` | yhub Service type | `ClusterIP` | +| `yhub.service.port` | yhub Service listening port | `443` | +| `yhub.service.targetPort` | yhub container listening port | `3002` | +| `yhub.service.annotations` | Annotations to add to the yhub Service | `{}` | +| `yhub.probes.liveness.path` | Configure path for yhub HTTP liveness probe | `/collaboration/jwks/v1` | +| `yhub.probes.liveness.initialDelaySeconds` | Configure initial delay for yhub liveness probe | `10` | +| `yhub.probes.readiness.path` | Configure path for yhub HTTP readiness probe | `/collaboration/jwks/v1` | +| `yhub.probes.readiness.initialDelaySeconds` | Configure initial delay for yhub readiness probe | `5` | +| `yhub.probes.liveness.targetPort` | Configure port for yhub HTTP liveness probe | | +| `yhub.probes.liveness.timeoutSeconds` | Configure timeout for yhub liveness probe | | +| `yhub.probes.readiness.targetPort` | Configure port for yhub HTTP readiness probe | | +| `yhub.probes.readiness.timeoutSeconds` | Configure timeout for yhub readiness probe | | +| `yhub.probes.startup.path` | Configure path for yhub HTTP startup probe | | +| `yhub.probes.startup.targetPort` | Configure port for yhub HTTP startup probe | | +| `yhub.probes.startup.initialDelaySeconds` | Configure initial delay for yhub startup probe | | +| `yhub.probes.startup.timeoutSeconds` | Configure timeout for yhub startup probe | | +| `yhub.resources` | Resource requirements for the yhub container | `{}` | +| `yhub.nodeSelector` | Node selector for the yhub Pod | `{}` | +| `yhub.tolerations` | Tolerations for the yhub Pod | `[]` | +| `yhub.affinity` | Affinity for the yhub Pod | `{}` | +| `yhub.persistence` | Additional volumes to create and mount on the yhub. Used for debugging purposes | `{}` | +| `yhub.persistence.volume-name.size` | Size of the additional volume | | +| `yhub.persistence.volume-name.type` | Type of the additional volume, persistentVolumeClaim or emptyDir | | +| `yhub.persistence.volume-name.mountPath` | Path where the volume should be mounted to | | +| `yhub.extraVolumeMounts` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` | +| `yhub.extraVolumes` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` | +| `yhub.pdb.enabled` | Enable pdb on yhub | `true` | +| `yhub.serviceAccountName` | Optional service account name to use for yhub pods | `nil` | + ### docSpec | Name | Description | Value | | -------------------------------------------------- | --------------------------------------------------------------- | ----------------------- | | `docSpec.enabled` | Enable docSpec deployment | `false` | | `docSpec.image.repository` | Repository to use to pull docSpec container image | `ghcr.io/docspecio/api` | -| `docSpec.image.tag` | docSpec container tag | `2.6.3` | +| `docSpec.image.tag` | docSpec container tag | `3.0.1` | | `docSpec.image.pullPolicy` | docSpec container image pull policy | `IfNotPresent` | | `docSpec.command` | Override the docSpec container command | `[]` | | `docSpec.args` | Override the docSpec container args | `[]` | diff --git a/src/helm/impress/templates/_helpers.tpl b/src/helm/impress/templates/_helpers.tpl index c4fe19048b..8f84245712 100644 --- a/src/helm/impress/templates/_helpers.tpl +++ b/src/helm/impress/templates/_helpers.tpl @@ -187,22 +187,87 @@ Requires top level scope {{- end }} {{/* -Full name for the yProvider converter +Full name for the docSpec Requires top level scope */}} -{{- define "impress.yProvider.converter.fullname" -}} -{{ include "impress.yProvider.fullname" . }}-converter +{{- define "impress.docSpec.fullname" -}} +{{ include "impress.fullname" . }}-docspec {{- end }} +{{/* +Full name for the yhub collaboration server + +Requires top level scope +*/}} +{{- define "impress.yhub.fullname" -}} +{{ include "impress.fullname" . }}-yhub +{{- end }} {{/* -Full name for the docSpec +JWT signing keys — the RSA keys the services sign the calls they make to each +other with. The jwt-keys job generates them once into a secret every service +mounts read-only, so no key is ever templated into a manifest, written in a +values file, or kept anywhere the services themselves can write. Requires top level scope */}} -{{- define "impress.docSpec.fullname" -}} -{{ include "impress.fullname" . }}-docspec +{{- define "impress.jwtKeys.secretName" -}} +{{- .Values.jwtKeys.existingSecret | default (printf "%s-jwt-keys" (include "impress.fullname" .)) -}} +{{- end }} + +{{- define "impress.jwtKeys.serviceAccountName" -}} +{{- .Values.jwtKeys.job.serviceAccountName | default (printf "%s-jwt-keys" (include "impress.fullname" .)) -}} +{{- end }} + +{{- define "impress.jwtKeys.backendPath" -}} +{{ .Values.jwtKeys.mountPath }}/{{ .Values.jwtKeys.backendKeyFilename }} +{{- end }} + +{{- define "impress.jwtKeys.yhubPath" -}} +{{ .Values.jwtKeys.mountPath }}/{{ .Values.jwtKeys.yhubKeyFilename }} +{{- end }} + +{{/* +The volume holding the keys. A pod referencing a secret that does not exist yet +stays in ContainerCreating and mounts it as soon as the job creates it, so +nothing else is needed to order the two. + +Requires top level scope +*/}} +{{- define "impress.jwtKeys.volume" -}} +- name: jwt-keys + secret: + secretName: {{ include "impress.jwtKeys.secretName" . }} + # read-only for everyone, as the files the job generates are + defaultMode: 0444 +{{- end }} + +{{- define "impress.jwtKeys.volumeMount" -}} +- name: jwt-keys + mountPath: {{ .Values.jwtKeys.mountPath }} + readOnly: true +{{- end }} + +{{/* +`*_FILE` environment variables pointing at the keys, added only when the +deployment did not set them by hand — configuring a key of your own stays +possible, and wins. + +Requires top level scope +*/}} +{{- define "impress.jwtKeys.backendEnv" -}} +{{- if not (hasKey (.Values.backend.envVars | default dict) "JWT_PRIVATE_KEY_FILE") }} +- name: "JWT_PRIVATE_KEY_FILE" + value: {{ include "impress.jwtKeys.backendPath" . | quote }} +{{- end }} +{{- end }} + +{{- define "impress.jwtKeys.yhubEnv" -}} +{{- if not (hasKey (.Values.yhub.envVars | default dict) "YHUB_JWT_PRIVATE_KEY_FILE") }} +- name: "YHUB_JWT_PRIVATE_KEY_FILE" + value: {{ include "impress.jwtKeys.yhubPath" . | quote }} +{{- end }} {{- end }} diff --git a/src/helm/impress/templates/backend_cronjob_list.yaml b/src/helm/impress/templates/backend_cronjob_list.yaml index 10708c0598..dce8f72056 100644 --- a/src/helm/impress/templates/backend_cronjob_list.yaml +++ b/src/helm/impress/templates/backend_cronjob_list.yaml @@ -38,9 +38,12 @@ items: imagePullPolicy: {{ ($.Values.backend.image | default dict).pullPolicy | default $.Values.image.pullPolicy }} args: {{- toYaml .command | nindent 22 }} - {{- if $envVars}} + {{- if or $envVars $.Values.jwtKeys.enabled }} env: {{- $envVars | indent 22 }} + {{- if $.Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.backendEnv" $ | nindent 22 }} + {{- end }} {{- end }} {{- if $.Values.backend.envFrom }} envFrom: @@ -55,6 +58,9 @@ items: {{- toYaml . | nindent 22 }} {{- end }} volumeMounts: + {{- if $.Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.volumeMount" $ | nindent 20 }} + {{- end }} {{- range $index, $value := $.Values.mountFiles }} - name: "files-{{ $index }}" mountPath: {{ $value.path }} @@ -72,6 +78,9 @@ items: {{- end }} restartPolicy: {{ .restartPolicy | default "Never" }} volumes: + {{- if $.Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.volume" $ | nindent 16 }} + {{- end }} {{- range $index, $value := $.Values.mountFiles }} - name: "files-{{ $index }}" configMap: diff --git a/src/helm/impress/templates/backend_deployment.yaml b/src/helm/impress/templates/backend_deployment.yaml index ec5961cc88..41657ee2a4 100644 --- a/src/helm/impress/templates/backend_deployment.yaml +++ b/src/helm/impress/templates/backend_deployment.yaml @@ -49,9 +49,12 @@ spec: args: {{- toYaml . | nindent 12 }} {{- end }} - {{- if $envVars}} + {{- if or $envVars .Values.jwtKeys.enabled }} env: {{- $envVars | indent 12 }} + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.backendEnv" . | nindent 12 }} + {{- end }} {{- end }} {{- $envFrom := concat (.Values.backend.envFrom | default list) ((.Values.backend.django | default dict).envFrom | default list) }} {{- if $envFrom }} @@ -83,6 +86,9 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} volumeMounts: + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.volumeMount" . | nindent 12 }} + {{- end }} {{- range $index, $value := .Values.mountFiles }} - name: "files-{{ $index }}" mountPath: {{ $value.path }} @@ -116,6 +122,9 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.volume" . | nindent 8 }} + {{- end }} {{- range $index, $value := .Values.mountFiles }} - name: "files-{{ $index }}" configMap: diff --git a/src/helm/impress/templates/backend_job.yml b/src/helm/impress/templates/backend_job.yml index 397f4cfbe1..59f4ad1c03 100644 --- a/src/helm/impress/templates/backend_job.yml +++ b/src/helm/impress/templates/backend_job.yml @@ -44,9 +44,12 @@ spec: args: {{- toYaml . | nindent 12 }} {{- end }} - {{- if $envVars}} + {{- if or $envVars .Values.jwtKeys.enabled }} env: {{- $envVars | indent 12 }} + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.backendEnv" . | nindent 12 }} + {{- end }} {{- end }} {{- if .Values.backend.envFrom }} envFrom: @@ -61,6 +64,9 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} volumeMounts: + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.volumeMount" . | nindent 12 }} + {{- end }} {{- range $index, $value := .Values.mountFiles }} - name: "files-{{ $index }}" mountPath: {{ $value.path }} @@ -90,6 +96,9 @@ spec: {{- end }} restartPolicy: {{ .Values.backend.job.restartPolicy }} volumes: + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.volume" . | nindent 8 }} + {{- end }} {{- range $index, $value := .Values.mountFiles }} - name: "files-{{ $index }}" configMap: diff --git a/src/helm/impress/templates/backend_job_createsuperuser.yaml b/src/helm/impress/templates/backend_job_createsuperuser.yaml index 76c230ce08..d34624e50c 100644 --- a/src/helm/impress/templates/backend_job_createsuperuser.yaml +++ b/src/helm/impress/templates/backend_job_createsuperuser.yaml @@ -48,9 +48,12 @@ spec: args: {{- toYaml . | nindent 12 }} {{- end }} - {{- if $envVars}} + {{- if or $envVars .Values.jwtKeys.enabled }} env: {{- $envVars | indent 12 }} + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.backendEnv" . | nindent 12 }} + {{- end }} {{- end }} {{- if .Values.backend.envFrom }} envFrom: @@ -65,6 +68,9 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} volumeMounts: + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.volumeMount" . | nindent 12 }} + {{- end }} {{- range $index, $value := .Values.mountFiles }} - name: "files-{{ $index }}" mountPath: {{ $value.path }} @@ -94,6 +100,9 @@ spec: {{- end }} restartPolicy: {{ .Values.backend.createsuperuser.restartPolicy }} volumes: + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.volume" . | nindent 8 }} + {{- end }} {{- range $index, $value := .Values.mountFiles }} - name: "files-{{ $index }}" configMap: diff --git a/src/helm/impress/templates/backend_job_migrate.yaml b/src/helm/impress/templates/backend_job_migrate.yaml index 28ce98978e..24cf064587 100644 --- a/src/helm/impress/templates/backend_job_migrate.yaml +++ b/src/helm/impress/templates/backend_job_migrate.yaml @@ -48,9 +48,12 @@ spec: args: {{- toYaml . | nindent 12 }} {{- end }} - {{- if $envVars}} + {{- if or $envVars .Values.jwtKeys.enabled }} env: {{- $envVars | indent 12 }} + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.backendEnv" . | nindent 12 }} + {{- end }} {{- end }} {{- if .Values.backend.envFrom }} envFrom: @@ -65,6 +68,9 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} volumeMounts: + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.volumeMount" . | nindent 12 }} + {{- end }} {{- range $index, $value := .Values.mountFiles }} - name: "files-{{ $index }}" mountPath: {{ $value.path }} @@ -94,6 +100,9 @@ spec: {{- end }} restartPolicy: {{ .Values.backend.migrate.restartPolicy }} volumes: + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.volume" . | nindent 8 }} + {{- end }} {{- range $index, $value := .Values.mountFiles }} - name: "files-{{ $index }}" configMap: diff --git a/src/helm/impress/templates/celery_worker_deployment.yaml b/src/helm/impress/templates/celery_worker_deployment.yaml index 7c9a5c831d..93854459fd 100644 --- a/src/helm/impress/templates/celery_worker_deployment.yaml +++ b/src/helm/impress/templates/celery_worker_deployment.yaml @@ -49,9 +49,12 @@ spec: args: {{- toYaml . | nindent 12 }} {{- end }} - {{- if $envVars}} + {{- if or $envVars .Values.jwtKeys.enabled }} env: {{- $envVars | indent 12 }} + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.backendEnv" . | nindent 12 }} + {{- end }} {{- end }} {{- $envFrom := concat (.Values.backend.envFrom | default list) (.Values.backend.celery.envFrom | default list) }} {{- if $envFrom }} @@ -83,6 +86,9 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} volumeMounts: + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.volumeMount" . | nindent 12 }} + {{- end }} {{- range $index, $value := .Values.mountFiles }} - name: "files-{{ $index }}" mountPath: {{ $value.path }} @@ -116,6 +122,9 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.volume" . | nindent 8 }} + {{- end }} {{- range $index, $value := .Values.mountFiles }} - name: "files-{{ $index }}" configMap: diff --git a/src/helm/impress/templates/ingress_collaboration_api.yaml b/src/helm/impress/templates/ingress_collaboration_api.yaml index 30d6327915..8da0493bb8 100644 --- a/src/helm/impress/templates/ingress_collaboration_api.yaml +++ b/src/helm/impress/templates/ingress_collaboration_api.yaml @@ -46,20 +46,26 @@ spec: - host: {{ .Values.ingressCollaborationApi.host | quote }} http: paths: - - path: {{ .Values.ingressCollaborationApi.path | quote }} + {{- /* one rule per route: what is not listed here is not reachable + from the outside, which is how the backend-internal routes + (reset-connections, migrate, restore-ydoc, reset-ydoc) stay + in-cluster */}} + {{- range .Values.ingressCollaborationApi.paths | default (list .Values.ingressCollaborationApi.path) }} + - path: {{ . | quote }} {{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }} pathType: ImplementationSpecific {{- end }} backend: {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} service: - name: {{ include "impress.yProvider.fullname" . }} + name: {{ include "impress.yhub.fullname" $ }} port: - number: {{ .Values.yProvider.service.port }} + number: {{ $.Values.yhub.service.port }} {{- else }} - serviceName: {{ include "impress.yProvider.fullname" . }} - servicePort: {{ .Values.yProvider.service.port }} + serviceName: {{ include "impress.yhub.fullname" $ }} + servicePort: {{ $.Values.yhub.service.port }} {{- end }} + {{- end }} {{- with .Values.ingressCollaborationApi.customBackends }} {{- toYaml . | nindent 10 }} {{- end }} diff --git a/src/helm/impress/templates/ingress_collaboration_ws.yaml b/src/helm/impress/templates/ingress_collaboration_ws.yaml index 887f74dd71..bac92ced37 100644 --- a/src/helm/impress/templates/ingress_collaboration_ws.yaml +++ b/src/helm/impress/templates/ingress_collaboration_ws.yaml @@ -53,12 +53,12 @@ spec: backend: {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} service: - name: {{ include "impress.yProvider.fullname" . }} + name: {{ include "impress.yhub.fullname" . }} port: - number: {{ .Values.yProvider.service.port }} + number: {{ .Values.yhub.service.port }} {{- else }} - serviceName: {{ include "impress.yProvider.fullname" . }} - servicePort: {{ .Values.yProvider.service.port }} + serviceName: {{ include "impress.yhub.fullname" . }} + servicePort: {{ .Values.yhub.service.port }} {{- end }} {{- with .Values.ingressCollaborationWS.customBackends }} {{- toYaml . | nindent 10 }} diff --git a/src/helm/impress/templates/jwt_keys_job.yaml b/src/helm/impress/templates/jwt_keys_job.yaml new file mode 100644 index 0000000000..296e26525e --- /dev/null +++ b/src/helm/impress/templates/jwt_keys_job.yaml @@ -0,0 +1,153 @@ +{{- if and .Values.jwtKeys.enabled (not .Values.jwtKeys.existingSecret) -}} +{{- $fullName := include "impress.fullname" . -}} +{{- $component := "jwt-keys" -}} +{{- $secretName := include "impress.jwtKeys.secretName" . -}} +# Generates the RSA keys the services sign the calls they make to each other +# with: one for the backend (the tokens Django issues to the collaboration +# server and the converter), one for the collaboration server (the calls it +# makes back to Django). Only the private halves are written — each service +# publishes the public half of its own key on its JWKS endpoint, where the +# other one reads it, so no key is ever copied from one side to the other. +# +# Two steps, two images: openssl writes the keys in a volume the pod throws +# away, then kubectl hands them to the secret the services mount. They only +# ever exist in that pod and in the secret. +# +# Idempotent, and deliberately so: the secret is left alone when it is already +# there, which is what makes it safe to re-run on every sync. Rolling the keys +# is deleting the secret and letting the next run create it again — both +# services follow, they pick the verification key by its "kid" and re-fetch the +# set when they meet one they do not know. +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $fullName }}-jwt-keys + namespace: {{ .Release.Namespace | quote }} + annotations: + argocd.argoproj.io/sync-options: Replace=true,Force=true + # after the service account it runs as, before anything mounting the secret + argocd.argoproj.io/sync-wave: "-2" + {{- with .Values.jwtKeys.job.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "impress.common.labels" (list . $component) | nindent 4 }} +spec: + ttlSecondsAfterFinished: {{ .Values.jwtKeys.job.ttlSecondsAfterFinished }} + backoffLimit: {{ .Values.jwtKeys.job.backoffLimit }} + template: + metadata: + {{- with .Values.jwtKeys.job.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "impress.common.selectorLabels" (list . $component) | nindent 8 }} + spec: + {{- if $.Values.image.credentials }} + imagePullSecrets: + - name: {{ include "impress.secret.dockerconfigjson.name" (dict "fullname" (include "impress.fullname" .) "imageCredentials" $.Values.image.credentials) }} + {{- end}} + serviceAccountName: {{ include "impress.jwtKeys.serviceAccountName" . }} + {{- with .Values.jwtKeys.job.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + restartPolicy: {{ .Values.jwtKeys.job.restartPolicy }} + initContainers: + - name: generate + image: "{{ .Values.jwtKeys.image.repository }}:{{ .Values.jwtKeys.image.tag }}" + imagePullPolicy: {{ .Values.jwtKeys.image.pullPolicy }} + {{- if .Values.jwtKeys.job.generateCommand }} + command: + {{- toYaml .Values.jwtKeys.job.generateCommand | nindent 12 }} + {{- else }} + # `command` rather than `args`: the image entrypoint is openssl + # itself. The keys are the ones `bin/generate-jwt-private-key.sh` + # writes for the compose stack, same command — PKCS#8 RSA, the format + # both the backend and the collaboration server read. + command: + - /bin/sh + - -c + - | + set -eu + + for name in {{ .Values.jwtKeys.backendKeyFilename | quote }} {{ .Values.jwtKeys.yhubKeyFilename | quote }}; do + openssl genpkey -algorithm RSA \ + -pkeyopt rsa_keygen_bits:{{ .Values.jwtKeys.keySize }} \ + -out "/keys/$name" + echo "$name generated" + done + {{- end }} + {{- with .Values.jwtKeys.job.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.jwtKeys.job.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: keys + mountPath: /keys + containers: + - name: publish + image: "{{ .Values.jwtKeys.kubectlImage.repository }}:{{ .Values.jwtKeys.kubectlImage.tag }}" + imagePullPolicy: {{ .Values.jwtKeys.kubectlImage.pullPolicy }} + {{- if .Values.jwtKeys.job.publishCommand }} + command: + {{- toYaml .Values.jwtKeys.job.publishCommand | nindent 12 }} + {{- else }} + command: + - /bin/sh + - -c + - | + set -eu + + # the keys generated above are dropped on the floor when the + # secret is already there: they are new ones, and replacing the + # live pair is a decision, never a side effect of a sync + if kubectl get secret {{ $secretName | quote }} >/dev/null 2>&1; then + echo "secret {{ $secretName }} already exists, keeping the keys it holds" + exit 0 + fi + + kubectl create secret generic {{ $secretName | quote }} \ + --from-file={{ .Values.jwtKeys.backendKeyFilename }}=/keys/{{ .Values.jwtKeys.backendKeyFilename }} \ + --from-file={{ .Values.jwtKeys.yhubKeyFilename }}=/keys/{{ .Values.jwtKeys.yhubKeyFilename }} + echo "secret {{ $secretName }} created" + {{- end }} + {{- with .Values.jwtKeys.job.env }} + env: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.jwtKeys.job.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.jwtKeys.job.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: keys + mountPath: /keys + readOnly: true + {{- with .Values.jwtKeys.job.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.jwtKeys.job.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.jwtKeys.job.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + # the keys live here for the lifetime of this pod and nowhere else + - name: keys + emptyDir: + medium: Memory +{{- end }} diff --git a/src/helm/impress/templates/jwt_keys_rbac.yaml b/src/helm/impress/templates/jwt_keys_rbac.yaml new file mode 100644 index 0000000000..397b55f4cf --- /dev/null +++ b/src/helm/impress/templates/jwt_keys_rbac.yaml @@ -0,0 +1,63 @@ +{{- if and .Values.jwtKeys.enabled .Values.jwtKeys.rbac.create (not .Values.jwtKeys.existingSecret) -}} +{{- $component := "jwt-keys" -}} +{{- $name := include "impress.jwtKeys.serviceAccountName" . -}} +# The generating job is the only thing in this release allowed to touch the +# secret holding the keys, and all it is allowed to do is read whether it +# exists and create it — not read its content back, not replace it, not delete +# it. The services themselves get the keys through a volume, so they need no +# access to the kubernetes API at all. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace | quote }} + annotations: + # The same wave as the job that runs under it. An earlier wave looks safer + # and is not: argocd only moves to the next wave once the current one is + # healthy, and a ServiceAccount has no health of its own to report. Within + # a wave it applies by kind, and accounts, roles and bindings all come + # before jobs — which is the ordering actually needed here. + argocd.argoproj.io/sync-wave: "-2" + labels: + {{- include "impress.common.labels" (list . $component) | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace | quote }} + annotations: + argocd.argoproj.io/sync-wave: "-2" + labels: + {{- include "impress.common.labels" (list . $component) | nindent 4 }} +rules: + # creating cannot be restricted to a name, kubernetes has no such rule + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create"] + # reading can, and is restricted to the one secret — to its existence really, + # the job never looks at what it holds + - apiGroups: [""] + resources: ["secrets"] + resourceNames: + - {{ include "impress.jwtKeys.secretName" . | quote }} + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace | quote }} + annotations: + argocd.argoproj.io/sync-wave: "-2" + labels: + {{- include "impress.common.labels" (list . $component) | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $name }} +subjects: + - kind: ServiceAccount + name: {{ $name }} + namespace: {{ .Release.Namespace | quote }} +{{- end }} diff --git a/src/helm/impress/templates/yhub_deployment.yaml b/src/helm/impress/templates/yhub_deployment.yaml new file mode 100644 index 0000000000..8ca6772aac --- /dev/null +++ b/src/helm/impress/templates/yhub_deployment.yaml @@ -0,0 +1,178 @@ +{{- if .Values.yhub.enabled -}} +{{- $envVars := include "impress.common.env" (list . .Values.yhub) -}} +{{- $fullName := include "impress.yhub.fullname" . -}} +{{- $component := "yhub" -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $fullName }} + namespace: {{ .Release.Namespace | quote }} + annotations: + {{- with .Values.yhub.dpAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "impress.common.labels" (list . $component) | nindent 4 }} +spec: + replicas: {{ .Values.yhub.replicas }} + selector: + matchLabels: + {{- include "impress.common.selectorLabels" (list . $component) | nindent 6 }} + template: + metadata: + annotations: + {{- with .Values.yhub.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "impress.common.selectorLabels" (list . $component) | nindent 8 }} + spec: + {{- if $.Values.image.credentials }} + imagePullSecrets: + - name: {{ include "impress.secret.dockerconfigjson.name" (dict "fullname" (include "impress.fullname" .) "imageCredentials" $.Values.image.credentials) }} + {{- end}} + {{- if .Values.yhub.serviceAccountName }} + serviceAccountName: {{ .Values.yhub.serviceAccountName }} + {{- end }} + shareProcessNamespace: {{ .Values.yhub.shareProcessNamespace }} + # a websocket connection is dropped when the pod goes away, and the client + # reconnects to another one — but the updates it sent are only in redis + # until a worker persists them, so leave the embedded worker time to drain + terminationGracePeriodSeconds: {{ .Values.yhub.terminationGracePeriodSeconds }} + containers: + {{- with .Values.yhub.sidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} + - name: {{ .Chart.Name }} + image: "{{ (.Values.yhub.image | default dict).repository | default .Values.image.repository }}:{{ (.Values.yhub.image | default dict).tag | default .Values.image.tag }}" + imagePullPolicy: {{ (.Values.yhub.image | default dict).pullPolicy | default .Values.image.pullPolicy }} + {{- with .Values.yhub.command }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.yhub.args }} + args: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if or $envVars .Values.jwtKeys.enabled }} + env: + {{- $envVars | indent 12 }} + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.yhubEnv" . | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.yhub.envFrom }} + envFrom: + {{- toYaml .Values.yhub.envFrom | nindent 12 }} + {{- end }} + {{- with .Values.yhub.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: {{ .Values.yhub.service.targetPort }} + protocol: TCP + {{- if .Values.yhub.probes.liveness }} + livenessProbe: + {{- include "impress.probes.abstract" (merge .Values.yhub.probes.liveness (dict "targetPort" .Values.yhub.service.targetPort )) | nindent 12 }} + {{- end }} + {{- if .Values.yhub.probes.readiness }} + readinessProbe: + {{- include "impress.probes.abstract" (merge .Values.yhub.probes.readiness (dict "targetPort" .Values.yhub.service.targetPort )) | nindent 12 }} + {{- end }} + {{- if .Values.yhub.probes.startup }} + startupProbe: + {{- include "impress.probes.abstract" (merge .Values.yhub.probes.startup (dict "targetPort" .Values.yhub.service.targetPort )) | nindent 12 }} + {{- end }} + {{- with .Values.yhub.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.volumeMount" . | nindent 12 }} + {{- end }} + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + mountPath: {{ $value.path }} + subPath: content + {{- end }} + {{- range $name, $volume := .Values.yhub.persistence }} + - name: "{{ $name }}" + mountPath: "{{ $volume.mountPath }}" + {{- end }} + {{- range .Values.yhub.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- with .Values.yhub.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.yhub.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.yhub.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.volume" . | nindent 8 }} + {{- end }} + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + configMap: + name: "{{ include "impress.fullname" $ }}-files-{{ $index }}" + {{- end }} + {{- range $name, $volume := .Values.yhub.persistence }} + - name: "{{ $name }}" + {{- if eq $volume.type "emptyDir" }} + emptyDir: {} + {{- else }} + persistentVolumeClaim: + claimName: "{{ $fullName }}-{{ $name }}" + {{- end }} + {{- end }} + {{- range .Values.yhub.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .secret }} + secret: + {{ toYaml .secret | nindent 12 }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 12 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 12 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 12 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 12 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} +--- +{{ if .Values.yhub.pdb.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ $fullName }} + namespace: {{ .Release.Namespace | quote }} +spec: + maxUnavailable: 1 + selector: + matchLabels: + {{- include "impress.common.selectorLabels" (list . $component) | nindent 6 }} +{{ end }} +{{- end }} diff --git a/src/helm/impress/templates/yhub_job_init_db.yaml b/src/helm/impress/templates/yhub_job_init_db.yaml new file mode 100644 index 0000000000..aa3c63a130 --- /dev/null +++ b/src/helm/impress/templates/yhub_job_init_db.yaml @@ -0,0 +1,148 @@ +{{- if and .Values.yhub.enabled .Values.yhub.initDb.enabled -}} +{{- $envVars := include "impress.common.env" (list . .Values.yhub) -}} +{{- $fullName := include "impress.yhub.fullname" . -}} +{{- $component := "yhub" -}} +# yhub never runs DDL from the server or the worker: the schema is created by +# the script it ships, which creates the database when it is missing and every +# table the installed version needs. It is idempotent, and it has to run again +# on every yhub upgrade that adds a table — the counterpart of the backend +# migrate job, hence the same Replace=true so a re-sync re-runs it. +# +# No sync wave, again like the backend migrate job: it runs in the default one, +# alongside it, and waits for its database the way that one waits for Django's. +# An earlier wave only moved it ahead of the postgres it needs. +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $fullName }}-init-db + namespace: {{ .Release.Namespace | quote }} + annotations: + argocd.argoproj.io/sync-options: Replace=true,Force=true + {{- with .Values.yhub.initDbJobAnnotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "impress.common.labels" (list . $component) | nindent 4 }} +spec: + ttlSecondsAfterFinished: {{ .Values.yhub.jobs.ttlSecondsAfterFinished }} + backoffLimit: {{ .Values.yhub.jobs.backoffLimit }} + template: + metadata: + annotations: + {{- with .Values.yhub.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "impress.common.selectorLabels" (list . $component) | nindent 8 }} + spec: + {{- if $.Values.image.credentials }} + imagePullSecrets: + - name: {{ include "impress.secret.dockerconfigjson.name" (dict "fullname" (include "impress.fullname" .) "imageCredentials" $.Values.image.credentials) }} + {{- end}} + {{- if .Values.yhub.serviceAccountName }} + serviceAccountName: {{ .Values.yhub.serviceAccountName }} + {{- end }} + shareProcessNamespace: {{ .Values.yhub.shareProcessNamespace }} + containers: + - name: {{ .Chart.Name }} + image: "{{ (.Values.yhub.image | default dict).repository | default .Values.image.repository }}:{{ (.Values.yhub.image | default dict).tag | default .Values.image.tag }}" + imagePullPolicy: {{ (.Values.yhub.image | default dict).pullPolicy | default .Values.image.pullPolicy }} + {{- if .Values.yhub.initDb.command }} + command: + {{- toYaml .Values.yhub.initDb.command | nindent 12 }} + {{- else }} + # The script yhub ships, wrapped as `npm run init-db`, retried until + # the postgres server answers: nothing here creates it, and a chart + # sync does not wait for whatever does. Retrying the whole script + # rather than probing the port first — it is idempotent, so a run + # against a database that is up but incomplete is a no-op, and no + # postgres client has to be present in the image to ask. + command: + - /bin/sh + - -c + - | + set -u + + attempt=1 + until node node_modules/@y/hub/bin/init-db.js; do + if [ "$attempt" -ge {{ .Values.yhub.initDb.retries }} ]; then + echo "database still unreachable after $attempt attempts, giving up" + exit 1 + fi + echo "database not ready, retrying in {{ .Values.yhub.initDb.retryDelaySeconds }}s ($attempt/{{ .Values.yhub.initDb.retries }})" + attempt=$((attempt + 1)) + sleep {{ .Values.yhub.initDb.retryDelaySeconds }} + done + {{- end }} + {{- if $envVars}} + env: + {{- $envVars | indent 12 }} + {{- end }} + {{- if .Values.yhub.envFrom }} + envFrom: + {{- toYaml .Values.yhub.envFrom | nindent 12 }} + {{- end }} + {{- with .Values.yhub.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.yhub.initDb.resources | default .Values.yhub.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + mountPath: {{ $value.path }} + subPath: content + {{- end }} + {{- range .Values.yhub.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- with .Values.yhub.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.yhub.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.yhub.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + restartPolicy: {{ .Values.yhub.initDb.restartPolicy }} + volumes: + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + configMap: + name: "{{ include "impress.fullname" $ }}-files-{{ $index }}" + {{- end }} + {{- range .Values.yhub.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .secret }} + secret: + {{ toYaml .secret | nindent 12 }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 12 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 12 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 12 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 12 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} +{{- end }} diff --git a/src/helm/impress/templates/yhub_svc.yaml b/src/helm/impress/templates/yhub_svc.yaml new file mode 100644 index 0000000000..12f0577ecd --- /dev/null +++ b/src/helm/impress/templates/yhub_svc.yaml @@ -0,0 +1,22 @@ +{{- if .Values.yhub.enabled -}} +{{- $fullName := include "impress.yhub.fullname" . -}} +{{- $component := "yhub" -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ $fullName }} + namespace: {{ .Release.Namespace | quote }} + labels: + {{- include "impress.common.labels" (list . $component) | nindent 4 }} + annotations: + {{- toYaml $.Values.yhub.service.annotations | nindent 4 }} +spec: + type: {{ .Values.yhub.service.type }} + ports: + - port: {{ .Values.yhub.service.port }} + targetPort: {{ .Values.yhub.service.targetPort }} + protocol: TCP + name: http + selector: + {{- include "impress.common.selectorLabels" (list . $component) | nindent 4 }} +{{- end }} diff --git a/src/helm/impress/templates/yprovider_deployment.yaml b/src/helm/impress/templates/yprovider_deployment.yaml index 953c088db4..8baadf09a4 100644 --- a/src/helm/impress/templates/yprovider_deployment.yaml +++ b/src/helm/impress/templates/yprovider_deployment.yaml @@ -7,7 +7,7 @@ metadata: name: {{ $fullName }} namespace: {{ .Release.Namespace | quote }} annotations: - {{- with .Values.backend.dpAnnotations }} + {{- with .Values.yProvider.dpAnnotations }} {{- toYaml . | nindent 4 }} {{- end }} labels: diff --git a/src/helm/impress/templates/yprovider_deployment_converter.yaml b/src/helm/impress/templates/yprovider_deployment_converter.yaml deleted file mode 100644 index 0eb3dbe3ca..0000000000 --- a/src/helm/impress/templates/yprovider_deployment_converter.yaml +++ /dev/null @@ -1,189 +0,0 @@ -{{ if .Values.yProvider.converter.enabled -}} -{{- $yProvider := .Values.yProvider -}} -{{- $converter := .Values.yProvider.converter -}} -{{- $service := mergeOverwrite (dict) (default dict $yProvider.service) (default dict $converter.service) -}} -{{- $image := mergeOverwrite (dict) (default dict $yProvider.image) (default dict $converter.image) -}} -{{- $probes := mergeOverwrite (dict) (default dict $yProvider.probes) (default dict $converter.probes) -}} -{{- $pdb := mergeOverwrite (dict) (default dict $yProvider.pdb) (default dict $converter.pdb) -}} -{{- $dpAnnotations := mergeOverwrite (dict) (default dict $yProvider.dpAnnotations) (default dict $converter.dpAnnotations) -}} -{{- $podAnnotations := mergeOverwrite (dict) (default dict $yProvider.podAnnotations) (default dict $converter.podAnnotations) -}} -{{- $replicas := default $yProvider.replicas $converter.replicas -}} -{{- $serviceAccountName := default $yProvider.serviceAccountName $converter.serviceAccountName -}} -{{- $shareProcessNamespace := default $yProvider.shareProcessNamespace $converter.shareProcessNamespace -}} -{{- $sidecars := default $yProvider.sidecars $converter.sidecars -}} -{{- $command := default $yProvider.command $converter.command -}} -{{- $args := default $yProvider.args $converter.args -}} -{{- $envFrom := default $yProvider.envFrom $converter.envFrom -}} -{{- $securityContext := mergeOverwrite (dict) (default dict $yProvider.securityContext) (default dict $converter.securityContext) -}} -{{- $resources := mergeOverwrite (dict) (default dict $yProvider.resources) (default dict $converter.resources) -}} -{{- $nodeSelector := mergeOverwrite (dict) (default dict $yProvider.nodeSelector) (default dict $converter.nodeSelector) -}} -{{- $affinity := mergeOverwrite (dict) (default dict $yProvider.affinity) (default dict $converter.affinity) -}} -{{- $tolerations := default $yProvider.tolerations $converter.tolerations -}} -{{- $persistence := mergeOverwrite (dict) (default dict $yProvider.persistence) (default dict $converter.persistence) -}} -{{- $extraVolumeMounts := default $yProvider.extraVolumeMounts $converter.extraVolumeMounts -}} -{{- $extraVolumes := default $yProvider.extraVolumes $converter.extraVolumes -}} -{{- $envVarsScope := dict "envVars" (mergeOverwrite (dict) (default dict $yProvider.envVars) (default dict $converter.envVars)) -}} -{{- $envVars := include "impress.common.env" (list . $envVarsScope) -}} -{{- $fullName := include "impress.yProvider.converter.fullname" . -}} -{{- $component := "yProvider-converter" -}} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ $fullName }} - namespace: {{ .Release.Namespace | quote }} - annotations: - {{- with $dpAnnotations }} - {{- toYaml . | nindent 4 }} - {{- end }} - labels: - {{- include "impress.common.labels" (list . $component) | nindent 4 }} -spec: - replicas: {{ $replicas }} - selector: - matchLabels: - {{- include "impress.common.selectorLabels" (list . $component) | nindent 6 }} - template: - metadata: - annotations: - {{- with $podAnnotations }} - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "impress.common.selectorLabels" (list . $component) | nindent 8 }} - spec: - {{- if $.Values.image.credentials }} - imagePullSecrets: - - name: {{ include "impress.secret.dockerconfigjson.name" (dict "fullname" (include "impress.fullname" .) "imageCredentials" $.Values.image.credentials) }} - {{- end}} - {{- if $serviceAccountName }} - serviceAccountName: {{ $serviceAccountName }} - {{- end }} - shareProcessNamespace: {{ $shareProcessNamespace }} - containers: - {{- with $sidecars }} - {{- toYaml . | nindent 8 }} - {{- end }} - - name: {{ .Chart.Name }} - image: "{{ $image.repository | default $.Values.image.repository }}:{{ $image.tag | default $.Values.image.tag }}" - imagePullPolicy: {{ $image.pullPolicy | default $.Values.image.pullPolicy }} - {{- with $command }} - command: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- with $args }} - args: - {{- toYaml . | nindent 12 }} - {{- end }} - {{- if $envVars}} - env: - {{- $envVars | indent 12 }} - {{- end }} - {{- if $envFrom }} - envFrom: - {{- toYaml $envFrom | nindent 12 }} - {{- end }} - {{- with $securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - ports: - - name: http - containerPort: {{ $service.targetPort }} - protocol: TCP - {{- if $probes.liveness }} - livenessProbe: - {{- include "impress.probes.abstract" (merge $probes.liveness (dict "targetPort" $service.targetPort )) | nindent 12 }} - {{- end }} - {{- if $probes.readiness }} - readinessProbe: - {{- include "impress.probes.abstract" (merge $probes.readiness (dict "targetPort" $service.targetPort )) | nindent 12 }} - {{- end }} - {{- if $probes.startup }} - startupProbe: - {{- include "impress.probes.abstract" (merge $probes.startup (dict "targetPort" $service.targetPort )) | nindent 12 }} - {{- end }} - {{- with $resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} - volumeMounts: - {{- range $index, $value := .Values.mountFiles }} - - name: "files-{{ $index }}" - mountPath: {{ $value.path }} - subPath: content - {{- end }} - {{- range $name, $volume := $persistence }} - - name: "{{ $name }}" - mountPath: "{{ $volume.mountPath }}" - {{- end }} - {{- range $extraVolumeMounts }} - - name: {{ .name }} - mountPath: {{ .mountPath }} - subPath: {{ .subPath | default "" }} - readOnly: {{ .readOnly }} - {{- end }} - {{- with $nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with $tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - volumes: - {{- range $index, $value := .Values.mountFiles }} - - name: "files-{{ $index }}" - configMap: - name: "{{ include "impress.fullname" $ }}-files-{{ $index }}" - {{- end }} - {{- range $name, $volume := $persistence }} - - name: "{{ $name }}" - {{- if eq $volume.type "emptyDir" }} - emptyDir: {} - {{- else }} - persistentVolumeClaim: - claimName: "{{ $fullName }}-{{ $name }}" - {{- end }} - {{- end }} - {{- range $extraVolumes }} - - name: {{ .name }} - {{- if .existingClaim }} - persistentVolumeClaim: - claimName: {{ .existingClaim }} - {{- else if .secret }} - secret: - {{ toYaml .secret | nindent 12 }} - {{- else if .hostPath }} - hostPath: - {{ toYaml .hostPath | nindent 12 }} - {{- else if .csi }} - csi: - {{- toYaml .csi | nindent 12 }} - {{- else if .configMap }} - configMap: - {{- toYaml .configMap | nindent 12 }} - {{- else if .emptyDir }} - emptyDir: - {{- toYaml .emptyDir | nindent 12 }} - {{- else }} - emptyDir: {} - {{- end }} - {{- end }} ---- -{{ if $pdb.enabled }} -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: {{ $fullName }} - namespace: {{ .Release.Namespace | quote }} -spec: - maxUnavailable: 1 - selector: - matchLabels: - {{- include "impress.common.selectorLabels" (list . $component) | nindent 6 }} -{{ end }} -{{ end }} diff --git a/src/helm/impress/templates/yprovider_svc_converter.yaml b/src/helm/impress/templates/yprovider_svc_converter.yaml deleted file mode 100644 index 45cd1e7bf2..0000000000 --- a/src/helm/impress/templates/yprovider_svc_converter.yaml +++ /dev/null @@ -1,25 +0,0 @@ -{{ if .Values.yProvider.converter.enabled -}} -{{- $yProvider := .Values.yProvider -}} -{{- $converter := .Values.yProvider.converter -}} -{{- $service := mergeOverwrite (dict) (default dict $yProvider.service) (default dict $converter.service) -}} -{{- $fullName := include "impress.yProvider.converter.fullname" . -}} -{{- $component := "yProvider-converter" -}} -apiVersion: v1 -kind: Service -metadata: - name: {{ $fullName }} - namespace: {{ .Release.Namespace | quote }} - labels: - {{- include "impress.common.labels" (list . $component) | nindent 4 }} - annotations: - {{- toYaml $service.annotations | nindent 4 }} -spec: - type: {{ $service.type }} - ports: - - port: {{ $service.port }} - targetPort: {{ $service.targetPort }} - protocol: TCP - name: http - selector: - {{- include "impress.common.selectorLabels" (list . $component) | nindent 4 }} -{{ end -}} diff --git a/src/helm/impress/values.yaml b/src/helm/impress/values.yaml index 25ec087369..95671541b7 100644 --- a/src/helm/impress/values.yaml +++ b/src/helm/impress/values.yaml @@ -78,12 +78,14 @@ ingressCollaborationWS: ## @param ingressCollaborationWS.annotations.nginx.ingress.kubernetes.io/enable-websocket ## @param ingressCollaborationWS.annotations.nginx.ingress.kubernetes.io/proxy-read-timeout ## @param ingressCollaborationWS.annotations.nginx.ingress.kubernetes.io/proxy-send-timeout - ## @param ingressCollaborationWS.annotations.nginx.ingress.kubernetes.io/upstream-hash-by + ## + ## No upstream-hash-by: yhub passes updates between its replicas through + ## redis, so two clients editing the same document may land on different + ## pods — where the y-provider it replaces needed a room to stay on one. annotations: nginx.ingress.kubernetes.io/enable-websocket: "true" nginx.ingress.kubernetes.io/proxy-read-timeout: "86400" nginx.ingress.kubernetes.io/proxy-send-timeout: "86400" - nginx.ingress.kubernetes.io/upstream-hash-by: $arg_room ## @param ingressRedirects.enabled whether to enable the Ingress Redirects or not ## @param ingressRedirects.className IngressClass to use for the Ingress Redirects @@ -112,7 +114,22 @@ ingressCollaborationApi: enabled: false className: null host: impress.example.com + ## Only used when `paths` below is empty path: /collaboration/api/ + ## @param ingressCollaborationApi.paths Paths to route to the collaboration server, one rule each + ## + ## The routes yhub serves to browsers, guarded by the same document + ## authorization as the websocket. Everything it serves that is not listed + ## here stays in-cluster — `create-ydoc`, `reset-connections`, `migrate`, + ## `restore-ydoc` and `reset-ydoc` are called by the backend only, and + ## publishing them would put document deletion and the legacy migration one + ## request away from the internet. + ## + ## `jwks` is public on purpose: it carries the public halves of the keys + ## yhub signs with, and nothing else. + paths: + - /collaboration/ydoc/ + - /collaboration/jwks/ ## @param ingressCollaborationApi.hosts Additional host to configure for the Ingress hosts: [] # - chart-example.local @@ -129,9 +146,10 @@ ingressCollaborationApi: ## @param ingressCollaborationApi.customBackends Add custom backends to ingress customBackends: [] - ## @param ingressCollaborationApi.annotations.nginx.ingress.kubernetes.io/upstream-hash-by - annotations: - nginx.ingress.kubernetes.io/upstream-hash-by: $arg_room + ## @skip ingressCollaborationApi.annotations + ## Same as ingressCollaborationWS: no upstream-hash-by, any yhub replica + ## answers for any document. + annotations: {} ## @param ingressAdmin.enabled whether to enable the Ingress or not ## @param ingressAdmin.className IngressClass to use for the Ingress @@ -643,7 +661,11 @@ posthog: annotations: {} ## @section yProvider - +## +## The conversion service, and nothing else since the collaboration moved to +## yhub: this deployment *is* the converter the backend calls on +## `Y_PROVIDER_API_BASE_URL`, so there is no separate converter release to +## enable anymore. yProvider: ## @param yProvider.image.repository Repository to use to pull impress's yProvider container image ## @param yProvider.image.tag impress's yProvider container tag @@ -653,77 +675,6 @@ yProvider: pullPolicy: IfNotPresent tag: "latest" - converter: - ## @param yProvider.converter.enabled Enable the yProvider converter deployment and service - enabled: false - - ## @param yProvider.converter.replicas Amount of yProvider replicas - replicas: 3 - - ## @param yProvider.converter.resources Resource requirements for the yProvider container - resources: {} - - ## @param yProvider.converter.service.type yProvider converter Service type - ## @param yProvider.converter.service.port yProvider converter Service listening port - ## @param yProvider.converter.service.targetPort yProvider converter container listening port - ## @param yProvider.converter.service.annotations Annotations to add to the yProvider converter Service - service: {} - - ## @param yProvider.converter.command Override the yProvider converter container command - command: [] - - ## @param yProvider.converter.args Override the yProvider converter container args - args: [] - - ## @param yProvider.converter.shareProcessNamespace Enable share process namespace between containers - shareProcessNamespace: false - - ## @param yProvider.converter.sidecars Add sidecars containers to yProvider converter deployment - sidecars: [] - - ## @skip yProvider.converter.securityContext - securityContext: {} - - ## @skip yProvider.converter.envVars - envVars: {} - - ## @skip yProvider.converter.envFrom - envFrom: [] - - ## @param yProvider.converter.podAnnotations Annotations to add to the yProvider converter Pod - podAnnotations: {} - - ## @param yProvider.converter.dpAnnotations Annotations to add to the yProvider converter Deployment - dpAnnotations: {} - - ## @skip yProvider.converter.probes - probes: {} - - ## @param yProvider.converter.nodeSelector Node selector for the yProvider converter Pod - nodeSelector: {} - - ## @param yProvider.converter.tolerations Tolerations for the yProvider converter Pod - tolerations: [] - - ## @param yProvider.converter.affinity Affinity for the yProvider converter Pod - affinity: {} - - ## @param yProvider.converter.persistence Additional volumes to create and mount on the yProvider converter - persistence: {} - - ## @param yProvider.converter.extraVolumeMounts Additional volumes to mount on the yProvider converter - extraVolumeMounts: [] - - ## @param yProvider.converter.extraVolumes Additional volumes to mount on the yProvider converter - extraVolumes: [] - - ## @param yProvider.converter.pdb.enabled Enable pdb on yProvider converter - pdb: - enabled: true - - ## @param yProvider.converter.serviceAccountName Optional service account name to use for yProvider converter pods - serviceAccountName: null - ## @param yProvider.command Override the yProvider container command command: [] @@ -834,6 +785,310 @@ yProvider: ## @param yProvider.serviceAccountName Optional service account name to use for yProvider pods serviceAccountName: null +## @section JWT signing keys +## +## The services do not share a secret: each signs the calls it makes to the +## others with an RSA key of its own and publishes the public half on its JWKS +## endpoint, where the others read it. Enabling this generates those keys on +## the cluster — a job creates them once in a secret every service mounts +## read-only, and leaves them alone on the next run — and points the backend +## and the collaboration server at them. No key is ever templated into a +## manifest or written in a values file, and only that job may create the +## secret: nothing in the release can read it back through the api. +## +## Leave it disabled to keep providing the keys yourself, through +## `backend.envVars.JWT_PRIVATE_KEY_FILE` and +## `yhub.envVars.YHUB_JWT_PRIVATE_KEY_FILE` and volumes of your own — both are +## left untouched when they are set by hand, enabled or not. +jwtKeys: + ## @param jwtKeys.enabled Generate the JWT signing keys of the services on the cluster + enabled: false + + ## @param jwtKeys.existingSecret Secret already holding the keys, generated in a secret of the chart's own when empty + ## + ## It has to hold the two filenames below. Naming one skips the job and the + ## rights it needs, the services only mount what is there. + existingSecret: null + + ## @param jwtKeys.mountPath Path the keys are mounted at, in every service reading them + mountPath: /data/jwt + + ## @param jwtKeys.backendKeyFilename Name of the key signing the tokens the backend issues + backendKeyFilename: private.pem + + ## @param jwtKeys.yhubKeyFilename Name of the key signing the calls the collaboration server makes to the backend + yhubKeyFilename: yhub-private.pem + + ## @param jwtKeys.keySize Size, in bits, of the generated RSA keys + keySize: 2048 + + ## @param jwtKeys.rbac.create Create the service account and the role the job needs to create the secret + ## + ## Turning it off means providing `jwtKeys.job.serviceAccountName` with an + ## account allowed to `create` secrets and to `get` the one named above. + rbac: + create: true + + ## @param jwtKeys.image.repository Repository to use to pull the image generating the keys + ## @param jwtKeys.image.tag Tag of the image generating the keys + ## @param jwtKeys.image.pullPolicy Pull policy of the image generating the keys + ## + ## openssl and a shell, nothing else. Its entrypoint is openssl itself, which + ## the job replaces by the script generating both keys. + image: + repository: alpine/openssl + pullPolicy: IfNotPresent + tag: "3.5.7" + + ## @param jwtKeys.kubectlImage.repository Repository to use to pull the image handing the keys to the secret + ## @param jwtKeys.kubectlImage.tag Tag of the image handing the keys to the secret + ## @param jwtKeys.kubectlImage.pullPolicy Pull policy of the image handing the keys to the secret + ## + ## A second image because the openssl one carries no kubectl, and reaching + ## the api with what it does carry (busybox wget, which cannot be told about + ## the cluster ca) would mean sending the token over an unverified + ## connection. + kubectlImage: + repository: dtzar/helm-kubectl + pullPolicy: IfNotPresent + tag: "3.16.2" + + ## @param jwtKeys.job.podSecurityContext Pod security context of the generating job + ## @param jwtKeys.job.securityContext.allowPrivilegeEscalation Whether to allow privilege escalation for the job containers + ## @param jwtKeys.job.securityContext.capabilities.drop List of capabilities to drop for the job containers + ## @param jwtKeys.job.securityContext.runAsNonRoot Whether to run the job containers as a non-root user + ## @param jwtKeys.job.securityContext.runAsUser User the job containers run as, their images declaring none + ## @param jwtKeys.job.securityContext.runAsGroup Group the job containers run as + ## @param jwtKeys.job.securityContext.seccompProfile.type Seccomp profile type for the job containers + ## @param jwtKeys.job.restartPolicy Restart policy of the generating job + ## @param jwtKeys.job.backoffLimit Numbers of generating job retries + ## @param jwtKeys.job.ttlSecondsAfterFinished Period to wait before removing the generating job + ## @param jwtKeys.job.generateCommand Override the command generating the keys + ## @param jwtKeys.job.publishCommand Override the command creating the secret from the generated keys + ## @param jwtKeys.job.annotations Annotations to add to the generating job + ## @param jwtKeys.job.podAnnotations Annotations to add to the generating job Pod + ## @param jwtKeys.job.resources Resource requirements for the job containers + ## @param jwtKeys.job.nodeSelector Node selector for the generating job Pod + ## @param jwtKeys.job.tolerations Tolerations for the generating job Pod + ## @param jwtKeys.job.affinity Affinity for the generating job Pod + ## @param jwtKeys.job.serviceAccountName Service account of the generating job Pod, the one created above when empty + ## @skip jwtKeys.job.env + job: + podSecurityContext: {} + # neither image declares a user of its own, and kubernetes refuses to start + # a container asking for runAsNonRoot without knowing which user to run as + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault + restartPolicy: Never + backoffLimit: 2 + ttlSecondsAfterFinished: 30 + generateCommand: [] + publishCommand: [] + annotations: {} + podAnnotations: {} + resources: {} + nodeSelector: {} + tolerations: [] + affinity: {} + serviceAccountName: null + env: [] + +## @section yhub +## +## The collaboration server: it serves the whole /collaboration/ prefix, the +## websocket included, and replaces the y-provider on that role. It keeps the +## live state of a document in redis/valkey and persists it to its own +## PostgreSQL database, so it needs both — set `yhub.envVars.REDIS` and +## `yhub.envVars.POSTGRES`, there is nothing sensible to default them to. +## Disabling it sends the /collaboration/ ingresses back to the y-provider. +yhub: + ## @param yhub.enabled Enable the yhub collaboration server, its service and its init-db job + enabled: true + + ## @param yhub.image.repository Repository to use to pull the yhub container image + ## @param yhub.image.tag yhub container tag + ## @param yhub.image.pullPolicy yhub container image pull policy + image: + repository: lasuite/impress-yhub + pullPolicy: IfNotPresent + tag: "latest" + + ## @param yhub.command Override the yhub container command + command: [] + + ## @param yhub.args Override the yhub container args + args: [] + + ## @param yhub.replicas Amount of yhub replicas + ## Every replica also runs a worker (redis consumer groups hand each task to + ## one of them), and clients editing the same document need not land on the + ## same pod: updates travel through redis. + replicas: 3 + + ## @param yhub.shareProcessNamespace Enable share process namespace between containers + shareProcessNamespace: false + + ## @param yhub.sidecars Add sidecars containers to yhub deployment + sidecars: [] + + ## @param yhub.terminationGracePeriodSeconds Grace period given to a yhub pod to drain before it is killed + terminationGracePeriodSeconds: 60 + + ## @param yhub.securityContext.allowPrivilegeEscalation Whether to allow privilege escalation for the yhub container + ## @param yhub.securityContext.capabilities.drop List of capabilities to drop for the yhub container + ## @param yhub.securityContext.runAsNonRoot Whether to run the yhub container as a non-root user + ## @param yhub.securityContext.runAsUser User the yhub container runs as + ## @param yhub.securityContext.runAsGroup Group the yhub container runs as + ## @param yhub.securityContext.seccompProfile.type Seccomp profile type for the yhub container + ## + ## The user is named rather than left to the image: asking for runAsNonRoot + ## without it is refused outright by kubernetes ("container has runAsNonRoot + ## and image will run as root") on any image that declares none — which every + ## yhub image built before the un-privileged user was added to its Dockerfile + ## does. 1000 is the `node` user the base image already carries. + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + seccompProfile: + type: RuntimeDefault + + ## @param yhub.envVars Configure yhub container environment variables + ## @extra yhub.envVars.REDIS Required, redis/valkey url holding the live document state (e.g. redis://valkey:6379/0) + ## @extra yhub.envVars.POSTGRES Required, url of the yhub database, created by the init-db job (e.g. postgres://user:pass@postgres:5432/yhub) + ## @extra yhub.envVars.REDIS_PREFIX Namespace of the redis keys, when the instance is shared (default: yhub) + ## @extra yhub.envVars.COLLABORATION_BACKEND_BASE_URL Base url of the Docs backend, which yhub asks about users and document access rights + ## @extra yhub.envVars.COLLABORATION_SERVER_ORIGIN Comma separated list of the origins allowed to open a websocket + ## @extra yhub.envVars.YHUB_JWT_PRIVATE_KEY_FILE Path to the RSA private key (PEM) yhub signs its calls to the backend with, mounted from a secret + ## @extra yhub.envVars.SOFT_MIGRATION Set to "true" to seed rooms from the legacy Django/S3 document store on first access + ## @extra yhub.envVars.BY_VALUE Example environment variable by setting value directly + ## @extra yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name Name of a ConfigMap when configuring env vars from a ConfigMap + ## @extra yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.key Key within a ConfigMap when configuring env vars from a ConfigMap + ## @extra yhub.envVars.FROM_SECRET.secretKeyRef.name Name of a Secret when configuring env vars from a Secret + ## @extra yhub.envVars.FROM_SECRET.secretKeyRef.key Key within a Secret when configuring env vars from a Secret + ## @skip yhub.envVars + envVars: + <<: *commonEnvVars + + ## @skip yhub.envFrom List of environment variables taken from Secrets or configMaps + envFrom: [] + # envFrom: + # - secret: + # name: super-secret-user-credentials + # - configMapRef: + # name: my-environment-variables + + ## @param yhub.podAnnotations Annotations to add to the yhub Pod + podAnnotations: {} + + ## @param yhub.dpAnnotations Annotations to add to the yhub Deployment + dpAnnotations: {} + + ## @param yhub.initDbJobAnnotations Annotations for the yhub init-db job + initDbJobAnnotations: {} + + ## @param yhub.jobs.ttlSecondsAfterFinished Period to wait before removing the init-db job + ## @param yhub.jobs.backoffLimit Numbers of init-db job retries + jobs: + ttlSecondsAfterFinished: 30 + backoffLimit: 2 + + ## @param yhub.initDb.enabled Run the job creating and upgrading the yhub schema + ## @param yhub.initDb.command Override the command creating and upgrading the yhub schema + ## @param yhub.initDb.retries How many times the schema script is retried while the postgres server does not answer + ## @param yhub.initDb.retryDelaySeconds Seconds between two attempts + ## @param yhub.initDb.restartPolicy Restart policy of the init-db job + ## @skip yhub.initDb.resources Resource requirements for the init-db container, defaults to yhub.resources + ## + ## The job runs in the default sync wave, next to the backend migrate job, + ## and waits for its database the same way that one waits for Django's: + ## nothing in this chart creates the postgres server, so it has to be given + ## the time whatever does takes. The defaults below wait five minutes. + initDb: + enabled: true + command: [] + retries: 60 + retryDelaySeconds: 5 + restartPolicy: Never + resources: {} + + ## @param yhub.service.type yhub Service type + ## @param yhub.service.port yhub Service listening port + ## @param yhub.service.targetPort yhub container listening port + ## @param yhub.service.annotations Annotations to add to the yhub Service + service: + type: ClusterIP + port: 443 + targetPort: 3002 + annotations: {} + + ## @param yhub.probes.liveness.path Configure path for yhub HTTP liveness probe + ## @param yhub.probes.liveness.initialDelaySeconds Configure initial delay for yhub liveness probe + ## @param yhub.probes.readiness.path Configure path for yhub HTTP readiness probe + ## @param yhub.probes.readiness.initialDelaySeconds Configure initial delay for yhub readiness probe + ## @extra yhub.probes.liveness.targetPort Configure port for yhub HTTP liveness probe + ## @extra yhub.probes.liveness.timeoutSeconds Configure timeout for yhub liveness probe + ## @extra yhub.probes.readiness.targetPort Configure port for yhub HTTP readiness probe + ## @extra yhub.probes.readiness.timeoutSeconds Configure timeout for yhub readiness probe + ## @extra yhub.probes.startup.path Configure path for yhub HTTP startup probe + ## @extra yhub.probes.startup.targetPort Configure port for yhub HTTP startup probe + ## @extra yhub.probes.startup.initialDelaySeconds Configure initial delay for yhub startup probe + ## @extra yhub.probes.startup.timeoutSeconds Configure timeout for yhub startup probe + ## + ## The JWKS route is the only one yhub serves unauthenticated: it answers 200 + ## with the public keys it signs its calls to the backend with (an empty set + ## when no key is configured), and reads neither redis nor postgres. + probes: + liveness: + path: /collaboration/jwks/v1 + initialDelaySeconds: 10 + readiness: + path: /collaboration/jwks/v1 + initialDelaySeconds: 5 + + ## @param yhub.resources Resource requirements for the yhub container + resources: {} + + ## @param yhub.nodeSelector Node selector for the yhub Pod + nodeSelector: {} + + ## @param yhub.tolerations Tolerations for the yhub Pod + tolerations: [] + + ## @param yhub.affinity Affinity for the yhub Pod + affinity: {} + + ## @param yhub.persistence Additional volumes to create and mount on the yhub. Used for debugging purposes + ## @extra yhub.persistence.volume-name.size Size of the additional volume + ## @extra yhub.persistence.volume-name.type Type of the additional volume, persistentVolumeClaim or emptyDir + ## @extra yhub.persistence.volume-name.mountPath Path where the volume should be mounted to + persistence: {} + + ## @param yhub.extraVolumeMounts Additional volumes to mount on the yhub. Mounted on the init-db job too + extraVolumeMounts: [] + + ## @param yhub.extraVolumes Additional volumes to mount on the yhub. Mounted on the init-db job too + extraVolumes: [] + + ## @param yhub.pdb.enabled Enable pdb on yhub + pdb: + enabled: true + + ## @param yhub.serviceAccountName Optional service account name to use for yhub pods + serviceAccountName: null + ## @section docSpec docSpec: ## @param docSpec.enabled Enable docSpec deployment diff --git a/src/yhub-server/Dockerfile b/src/yhub-server/Dockerfile index 4d31804940..3cb8063b53 100644 --- a/src/yhub-server/Dockerfile +++ b/src/yhub-server/Dockerfile @@ -3,7 +3,9 @@ FROM node:22-trixie AS base WORKDIR /app -COPY package.json package-lock.json ./ +# built from the repository root, like every other image here — the entrypoint +# below lives outside this directory +COPY ./src/yhub-server/package.json ./src/yhub-server/package-lock.json ./ # ---- Development image ---- @@ -16,7 +18,7 @@ RUN npm ci # server.js, migration.js, env.js — glob so a new module cannot be forgotten. # compose bind-mounts the sources over /app on top of this copy, so an edit on # the host is seen immediately; the copy keeps the image usable on its own. -COPY *.js ./ +COPY ./src/yhub-server/*.js ./ EXPOSE 3002 @@ -32,8 +34,24 @@ FROM base AS yhub RUN npm ci --omit=dev -COPY *.js ./ +COPY ./src/yhub-server/*.js ./ EXPOSE 3002 +# Same entrypoint as the other services: it gives the container user an entry in +# /etc/passwd, which an arbitrary uid (kubernetes runAsUser) does not have. The +# group needs the same rights as the owner on /etc/passwd for it to write there. +COPY ./docker/files/usr/local/bin/entrypoint /usr/local/bin/entrypoint +RUN chmod g=u /etc/passwd + +# Un-privileged user running the application. The server writes nothing outside +# stdout, so it needs no home and no writable path. Defaulted, unlike the other +# images of this repository: the helm chart runs the pod with runAsNonRoot, and +# a build that forgot the argument would produce an image kubernetes refuses to +# start. +ARG DOCKER_USER=1000 +USER ${DOCKER_USER} + +ENTRYPOINT [ "/usr/local/bin/entrypoint" ] + CMD ["node", "server.js"] diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index 3b3e7d6075..849816738e 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -85,7 +85,15 @@ repository: `app crashed - waiting for file changes` and the next save starts the server again, - `yhub` — the production image: production dependencies only, `node - server.js`, sources baked in. + server.js`, sources baked in, and the un-privileged user and the entrypoint + the other services use (kubernetes runs the pod with `runAsNonRoot`). + +Both are built **from the repository root**, like every other image here — the +entrypoint they share lives outside this directory: + +``` +docker build -f src/yhub-server/Dockerfile --target yhub . +``` nodemon rather than node's own `--watch`: the latter watches inodes, so it stops seeing a file as soon as it is replaced by a rename — which is what `git From 04a0b8944ef3f6f15d9cfd5e2bebf7060a135675 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Thu, 13 Aug 2026 16:18:54 +0200 Subject: [PATCH 52/59] =?UTF-8?q?=E2=9C=A8(yhub)=20add=20custom=20probes?= =?UTF-8?q?=20more=20efficient?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The probes used before was using the only one public available endpoint. This endpoint is the jwks endpoint but it is not an appropriated one. For the readyness we check that it is possible to connect to postgres and to redis. for the liveness we made a ping pong just checking the http connection. --- CHANGELOG.md | 9 +++ src/helm/impress/README.md | 138 +++++++++++++++++------------------ src/helm/impress/values.yaml | 25 +++++-- src/yhub-server/README.md | 19 ++++- src/yhub-server/server.js | 97 +++++++++++++++++++++--- 5 files changed, 200 insertions(+), 88 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74a57ce555..1a51a46197 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -154,6 +154,15 @@ and this project adheres to and rolling the keys is deleting the secret and letting the next run create it again. `jwtKeys.existingSecret` points at keys of your own instead, and skips both the job and its rights +- ✨(collaboration) serve two probes on yhub: `GET /collaboration/ping/v1` + answers `pong` without touching a store — being answered is what a liveness + check should conclude, and restarting a server over a store it cannot reach + would drop the websockets it serves — and `GET /collaboration/ready/v1` asks + postgres and redis in parallel, answering `503` with the offending one marked + unreachable so the pod leaves the service endpoints while its siblings keep + serving. Both unauthenticated, like the JWKS; neither names the error in its + body, which a postgres client would gladly fill with its connection string. + The helm probes point at them - ✨(helm) deploy the collaboration server: the chart gains a `yhub` deployment, its service, and the job running the `init-db` script that creates and upgrades its schema — next to the backend migrate job, retrying while the diff --git a/src/helm/impress/README.md b/src/helm/impress/README.md index 568e971eab..1bf5fb7c95 100644 --- a/src/helm/impress/README.md +++ b/src/helm/impress/README.md @@ -339,75 +339,75 @@ ### yhub -| Name | Description | Value | -| -------------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | ------------------------ | -| `yhub.enabled` | Enable the yhub collaboration server, its service and its init-db job | `true` | -| `yhub.image.repository` | Repository to use to pull the yhub container image | `lasuite/impress-yhub` | -| `yhub.image.tag` | yhub container tag | `latest` | -| `yhub.image.pullPolicy` | yhub container image pull policy | `IfNotPresent` | -| `yhub.command` | Override the yhub container command | `[]` | -| `yhub.args` | Override the yhub container args | `[]` | -| `yhub.replicas` | Amount of yhub replicas | `3` | -| `yhub.shareProcessNamespace` | Enable share process namespace between containers | `false` | -| `yhub.sidecars` | Add sidecars containers to yhub deployment | `[]` | -| `yhub.terminationGracePeriodSeconds` | Grace period given to a yhub pod to drain before it is killed | `60` | -| `yhub.securityContext.allowPrivilegeEscalation` | Whether to allow privilege escalation for the yhub container | `false` | -| `yhub.securityContext.capabilities.drop` | List of capabilities to drop for the yhub container | `["ALL"]` | -| `yhub.securityContext.runAsNonRoot` | Whether to run the yhub container as a non-root user | `true` | -| `yhub.securityContext.runAsUser` | User the yhub container runs as | `1000` | -| `yhub.securityContext.runAsGroup` | Group the yhub container runs as | `1000` | -| `yhub.securityContext.seccompProfile.type` | Seccomp profile type for the yhub container | `RuntimeDefault` | -| `yhub.envVars` | Configure yhub container environment variables | `undefined` | -| `yhub.envVars.REDIS` | Required, redis/valkey url holding the live document state (e.g. redis://valkey:6379/0) | | -| `yhub.envVars.POSTGRES` | Required, url of the yhub database, created by the init-db job (e.g. postgres://user:pass@postgres:5432/yhub) | | -| `yhub.envVars.REDIS_PREFIX` | Namespace of the redis keys, when the instance is shared (default: yhub) | | -| `yhub.envVars.COLLABORATION_BACKEND_BASE_URL` | Base url of the Docs backend, which yhub asks about users and document access rights | | -| `yhub.envVars.COLLABORATION_SERVER_ORIGIN` | Comma separated list of the origins allowed to open a websocket | | -| `yhub.envVars.YHUB_JWT_PRIVATE_KEY_FILE` | Path to the RSA private key (PEM) yhub signs its calls to the backend with, mounted from a secret | | -| `yhub.envVars.SOFT_MIGRATION` | Set to "true" to seed rooms from the legacy Django/S3 document store on first access | | -| `yhub.envVars.BY_VALUE` | Example environment variable by setting value directly | | -| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name` | Name of a ConfigMap when configuring env vars from a ConfigMap | | -| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.key` | Key within a ConfigMap when configuring env vars from a ConfigMap | | -| `yhub.envVars.FROM_SECRET.secretKeyRef.name` | Name of a Secret when configuring env vars from a Secret | | -| `yhub.envVars.FROM_SECRET.secretKeyRef.key` | Key within a Secret when configuring env vars from a Secret | | -| `yhub.podAnnotations` | Annotations to add to the yhub Pod | `{}` | -| `yhub.dpAnnotations` | Annotations to add to the yhub Deployment | `{}` | -| `yhub.initDbJobAnnotations` | Annotations for the yhub init-db job | `{}` | -| `yhub.jobs.ttlSecondsAfterFinished` | Period to wait before removing the init-db job | `30` | -| `yhub.jobs.backoffLimit` | Numbers of init-db job retries | `2` | -| `yhub.initDb.enabled` | Run the job creating and upgrading the yhub schema | `true` | -| `yhub.initDb.command` | Override the command creating and upgrading the yhub schema | `[]` | -| `yhub.initDb.retries` | How many times the schema script is retried while the postgres server does not answer | `60` | -| `yhub.initDb.retryDelaySeconds` | Seconds between two attempts | `5` | -| `yhub.initDb.restartPolicy` | Restart policy of the init-db job | `Never` | -| `yhub.service.type` | yhub Service type | `ClusterIP` | -| `yhub.service.port` | yhub Service listening port | `443` | -| `yhub.service.targetPort` | yhub container listening port | `3002` | -| `yhub.service.annotations` | Annotations to add to the yhub Service | `{}` | -| `yhub.probes.liveness.path` | Configure path for yhub HTTP liveness probe | `/collaboration/jwks/v1` | -| `yhub.probes.liveness.initialDelaySeconds` | Configure initial delay for yhub liveness probe | `10` | -| `yhub.probes.readiness.path` | Configure path for yhub HTTP readiness probe | `/collaboration/jwks/v1` | -| `yhub.probes.readiness.initialDelaySeconds` | Configure initial delay for yhub readiness probe | `5` | -| `yhub.probes.liveness.targetPort` | Configure port for yhub HTTP liveness probe | | -| `yhub.probes.liveness.timeoutSeconds` | Configure timeout for yhub liveness probe | | -| `yhub.probes.readiness.targetPort` | Configure port for yhub HTTP readiness probe | | -| `yhub.probes.readiness.timeoutSeconds` | Configure timeout for yhub readiness probe | | -| `yhub.probes.startup.path` | Configure path for yhub HTTP startup probe | | -| `yhub.probes.startup.targetPort` | Configure port for yhub HTTP startup probe | | -| `yhub.probes.startup.initialDelaySeconds` | Configure initial delay for yhub startup probe | | -| `yhub.probes.startup.timeoutSeconds` | Configure timeout for yhub startup probe | | -| `yhub.resources` | Resource requirements for the yhub container | `{}` | -| `yhub.nodeSelector` | Node selector for the yhub Pod | `{}` | -| `yhub.tolerations` | Tolerations for the yhub Pod | `[]` | -| `yhub.affinity` | Affinity for the yhub Pod | `{}` | -| `yhub.persistence` | Additional volumes to create and mount on the yhub. Used for debugging purposes | `{}` | -| `yhub.persistence.volume-name.size` | Size of the additional volume | | -| `yhub.persistence.volume-name.type` | Type of the additional volume, persistentVolumeClaim or emptyDir | | -| `yhub.persistence.volume-name.mountPath` | Path where the volume should be mounted to | | -| `yhub.extraVolumeMounts` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` | -| `yhub.extraVolumes` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` | -| `yhub.pdb.enabled` | Enable pdb on yhub | `true` | -| `yhub.serviceAccountName` | Optional service account name to use for yhub pods | `nil` | +| Name | Description | Value | +| -------------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | ------------------------- | +| `yhub.enabled` | Enable the yhub collaboration server, its service and its init-db job | `true` | +| `yhub.image.repository` | Repository to use to pull the yhub container image | `lasuite/impress-yhub` | +| `yhub.image.tag` | yhub container tag | `latest` | +| `yhub.image.pullPolicy` | yhub container image pull policy | `IfNotPresent` | +| `yhub.command` | Override the yhub container command | `[]` | +| `yhub.args` | Override the yhub container args | `[]` | +| `yhub.replicas` | Amount of yhub replicas | `3` | +| `yhub.shareProcessNamespace` | Enable share process namespace between containers | `false` | +| `yhub.sidecars` | Add sidecars containers to yhub deployment | `[]` | +| `yhub.terminationGracePeriodSeconds` | Grace period given to a yhub pod to drain before it is killed | `60` | +| `yhub.securityContext.allowPrivilegeEscalation` | Whether to allow privilege escalation for the yhub container | `false` | +| `yhub.securityContext.capabilities.drop` | List of capabilities to drop for the yhub container | `["ALL"]` | +| `yhub.securityContext.runAsNonRoot` | Whether to run the yhub container as a non-root user | `true` | +| `yhub.securityContext.runAsUser` | User the yhub container runs as | `1000` | +| `yhub.securityContext.runAsGroup` | Group the yhub container runs as | `1000` | +| `yhub.securityContext.seccompProfile.type` | Seccomp profile type for the yhub container | `RuntimeDefault` | +| `yhub.envVars` | Configure yhub container environment variables | `undefined` | +| `yhub.envVars.REDIS` | Required, redis/valkey url holding the live document state (e.g. redis://valkey:6379/0) | | +| `yhub.envVars.POSTGRES` | Required, url of the yhub database, created by the init-db job (e.g. postgres://user:pass@postgres:5432/yhub) | | +| `yhub.envVars.REDIS_PREFIX` | Namespace of the redis keys, when the instance is shared (default: yhub) | | +| `yhub.envVars.COLLABORATION_BACKEND_BASE_URL` | Base url of the Docs backend, which yhub asks about users and document access rights | | +| `yhub.envVars.COLLABORATION_SERVER_ORIGIN` | Comma separated list of the origins allowed to open a websocket | | +| `yhub.envVars.YHUB_JWT_PRIVATE_KEY_FILE` | Path to the RSA private key (PEM) yhub signs its calls to the backend with, mounted from a secret | | +| `yhub.envVars.SOFT_MIGRATION` | Set to "true" to seed rooms from the legacy Django/S3 document store on first access | | +| `yhub.envVars.BY_VALUE` | Example environment variable by setting value directly | | +| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name` | Name of a ConfigMap when configuring env vars from a ConfigMap | | +| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.key` | Key within a ConfigMap when configuring env vars from a ConfigMap | | +| `yhub.envVars.FROM_SECRET.secretKeyRef.name` | Name of a Secret when configuring env vars from a Secret | | +| `yhub.envVars.FROM_SECRET.secretKeyRef.key` | Key within a Secret when configuring env vars from a Secret | | +| `yhub.podAnnotations` | Annotations to add to the yhub Pod | `{}` | +| `yhub.dpAnnotations` | Annotations to add to the yhub Deployment | `{}` | +| `yhub.initDbJobAnnotations` | Annotations for the yhub init-db job | `{}` | +| `yhub.jobs.ttlSecondsAfterFinished` | Period to wait before removing the init-db job | `30` | +| `yhub.jobs.backoffLimit` | Numbers of init-db job retries | `2` | +| `yhub.initDb.enabled` | Run the job creating and upgrading the yhub schema | `true` | +| `yhub.initDb.command` | Override the command creating and upgrading the yhub schema | `[]` | +| `yhub.initDb.retries` | How many times the schema script is retried while the postgres server does not answer | `60` | +| `yhub.initDb.retryDelaySeconds` | Seconds between two attempts | `5` | +| `yhub.initDb.restartPolicy` | Restart policy of the init-db job | `Never` | +| `yhub.service.type` | yhub Service type | `ClusterIP` | +| `yhub.service.port` | yhub Service listening port | `443` | +| `yhub.service.targetPort` | yhub container listening port | `3002` | +| `yhub.service.annotations` | Annotations to add to the yhub Service | `{}` | +| `yhub.probes.liveness.path` | Configure path for yhub HTTP liveness probe | `/collaboration/ping/v1` | +| `yhub.probes.liveness.initialDelaySeconds` | Configure initial delay for yhub liveness probe | `10` | +| `yhub.probes.liveness.timeoutSeconds` | Configure timeout for yhub liveness probe | `2` | +| `yhub.probes.readiness.path` | Configure path for yhub HTTP readiness probe | `/collaboration/ready/v1` | +| `yhub.probes.readiness.initialDelaySeconds` | Configure initial delay for yhub readiness probe | `5` | +| `yhub.probes.readiness.timeoutSeconds` | Configure timeout for yhub readiness probe | `3` | +| `yhub.probes.liveness.targetPort` | Configure port for yhub HTTP liveness probe | | +| `yhub.probes.readiness.targetPort` | Configure port for yhub HTTP readiness probe | | +| `yhub.probes.startup.path` | Configure path for yhub HTTP startup probe | | +| `yhub.probes.startup.targetPort` | Configure port for yhub HTTP startup probe | | +| `yhub.probes.startup.initialDelaySeconds` | Configure initial delay for yhub startup probe | | +| `yhub.probes.startup.timeoutSeconds` | Configure timeout for yhub startup probe | | +| `yhub.resources` | Resource requirements for the yhub container | `{}` | +| `yhub.nodeSelector` | Node selector for the yhub Pod | `{}` | +| `yhub.tolerations` | Tolerations for the yhub Pod | `[]` | +| `yhub.affinity` | Affinity for the yhub Pod | `{}` | +| `yhub.persistence` | Additional volumes to create and mount on the yhub. Used for debugging purposes | `{}` | +| `yhub.persistence.volume-name.size` | Size of the additional volume | | +| `yhub.persistence.volume-name.type` | Type of the additional volume, persistentVolumeClaim or emptyDir | | +| `yhub.persistence.volume-name.mountPath` | Path where the volume should be mounted to | | +| `yhub.extraVolumeMounts` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` | +| `yhub.extraVolumes` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` | +| `yhub.pdb.enabled` | Enable pdb on yhub | `true` | +| `yhub.serviceAccountName` | Optional service account name to use for yhub pods | `nil` | ### docSpec diff --git a/src/helm/impress/values.yaml b/src/helm/impress/values.yaml index 95671541b7..35b33708a2 100644 --- a/src/helm/impress/values.yaml +++ b/src/helm/impress/values.yaml @@ -1036,27 +1036,38 @@ yhub: ## @param yhub.probes.liveness.path Configure path for yhub HTTP liveness probe ## @param yhub.probes.liveness.initialDelaySeconds Configure initial delay for yhub liveness probe + ## @param yhub.probes.liveness.timeoutSeconds Configure timeout for yhub liveness probe ## @param yhub.probes.readiness.path Configure path for yhub HTTP readiness probe ## @param yhub.probes.readiness.initialDelaySeconds Configure initial delay for yhub readiness probe + ## @param yhub.probes.readiness.timeoutSeconds Configure timeout for yhub readiness probe ## @extra yhub.probes.liveness.targetPort Configure port for yhub HTTP liveness probe - ## @extra yhub.probes.liveness.timeoutSeconds Configure timeout for yhub liveness probe ## @extra yhub.probes.readiness.targetPort Configure port for yhub HTTP readiness probe - ## @extra yhub.probes.readiness.timeoutSeconds Configure timeout for yhub readiness probe ## @extra yhub.probes.startup.path Configure path for yhub HTTP startup probe ## @extra yhub.probes.startup.targetPort Configure port for yhub HTTP startup probe ## @extra yhub.probes.startup.initialDelaySeconds Configure initial delay for yhub startup probe ## @extra yhub.probes.startup.timeoutSeconds Configure timeout for yhub startup probe ## - ## The JWKS route is the only one yhub serves unauthenticated: it answers 200 - ## with the public keys it signs its calls to the backend with (an empty set - ## when no key is configured), and reads neither redis nor postgres. + ## Two routes yhub serves unauthenticated, and they answer different + ## questions on purpose: + ## + ## - `ping` returns 200 without touching anything. Being answered at all is + ## the proof the http channel and the event loop are alive, which is as far + ## as a liveness probe should ever go: restarting a server over a store it + ## does not reach would drop the websockets it is happily serving. + ## - `ready` asks postgres and redis whether they answer, and returns 503 + ## when either does not. That takes the pod out of the service endpoints + ## and leaves its siblings serving, which is what readiness is for. Its + ## timeout is above the two seconds the server itself gives each store, so + ## an unreachable one is reported rather than cut off. probes: liveness: - path: /collaboration/jwks/v1 + path: /collaboration/ping/v1 initialDelaySeconds: 10 + timeoutSeconds: 2 readiness: - path: /collaboration/jwks/v1 + path: /collaboration/ready/v1 initialDelaySeconds: 5 + timeoutSeconds: 3 ## @param yhub.resources Resource requirements for the yhub container resources: {} diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index 849816738e..324ff0a2b1 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -60,6 +60,20 @@ It is not a fork of yhub — it is a thin wrapper: its own tokens: neither side is configured with a copy of the key of the other, so either can roll its key on its own. Served unauthenticated, as any JWKS is, +- answers two probes, unauthenticated like the JWKS and deliberately asking + different questions: + - `GET /collaboration/ping/v1` → `200 {"status":"pong"}` without touching + anything. Being answered at all proves the http channel and the event loop + are alive, which is as far as a **liveness** check should go: restarting a + server over a store it cannot reach would drop the websockets it is + serving perfectly well, + - `GET /collaboration/ready/v1` → `200 {"status":"ready","checks":{…}}`, or + `503` with the offending store marked `unreachable`, after asking postgres + (`SELECT 1`) and redis (`PING`) in parallel, each with a two second + budget. A **readiness** failure takes the pod out of the service endpoints + and leaves its siblings serving. The body names the store but never the + error: the route is public, and a postgres client will happily put its + connection string in the message it raises — that goes to the log instead, - mirrors the environment conventions used elsewhere in this repository (`*_FILE` secret indirection, `COLLABORATION_SERVER_ORIGIN` allowlist, …). @@ -70,7 +84,10 @@ authorization and are meant to be reachable by browsers, as is `/collaboration/jwks/v1`, which carries public keys and nothing else. The one exception is `/collaboration/reset-connections/`, `/collaboration/migrate/`, `/collaboration/restore-ydoc/` and `/collaboration/reset-ydoc/`, which are -backend-internal and should not be routed through the public ingress. +backend-internal and should not be routed through the public ingress. The two +probes are not worth publishing either — kubelet calls them from inside — and +the helm chart's ingress lists what it routes rather than what it hides, so +they stay in-cluster on their own. ## Container image diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index f2cfc9a391..d63a4e1f91 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -43,9 +43,20 @@ const ORG = process.env.YHUB_ORG || 'docs'; // URL scheme Docs already routes to the collaboration server. Hardcoded like // the audiences: the backend builds its urls with the same prefix. const API_PREFIX = 'collaboration'; -// Path the JWKS endpoint declared in `api` is mounted at. readAuthInfo reads -// the raw request, without any route context, hence the duplication. -const JWKS_PATH = `/${API_PREFIX}/jwks/v1`; +// Paths of the routes declared in `api` that are served to anyone: the JWKS, +// which carries public keys and which the backend must read before it can +// authenticate anything we send it, and the two probes, which kubernetes calls +// with no cookie and no token. readAuthInfo reads the raw request, without any +// route context, hence the duplication of the paths here. +const PUBLIC_PATHS = new Set([ + `/${API_PREFIX}/jwks/v1`, + `/${API_PREFIX}/ping/v1`, + `/${API_PREFIX}/ready/v1`, +]); +// What the readiness check gives a store before reporting it unreachable. Short +// on purpose: the point of the probe is to answer, and answering "not ready" +// early is more useful than holding the connection until kubelet times out. +const READINESS_TIMEOUT_MS = 2000; // Requiring this audience stops a valid admin JWT that Django issued for // another service (today: the y-converter token in converter_services.py, // which is handed to the converter process) from being replayed against yhub. @@ -207,12 +218,10 @@ const auth = createAuthPlugin({ const cookie = req.getHeader('cookie'); const origin = req.getHeader('origin'); const gcOff = req.getQuery('gc') === 'false'; - // The JWKS holds public keys and nothing else, and the backend must be - // able to fetch it before it can authenticate anything we send it — so it - // is served to anyone, as the backend serves its own. This identity is - // granted the 'jwks' purpose and nothing else (getGlobalAccessType), and - // the check is on the exact path of that one route. - if (url === JWKS_PATH) { + // The JWKS and the probes are served to anyone (see PUBLIC_PATHS). This + // identity is granted their purposes and nothing else + // (getGlobalAccessType), and the check is on their exact paths. + if (PUBLIC_PATHS.has(url)) { return { userid: 'anonymous' }; } if (authorization !== '') { @@ -284,9 +293,12 @@ const auth = createAuthPlugin({ return { userid: `anon:${anon}`, cookie, origin }; } }, - // Authorizes the global-scoped endpoints, of which the JWKS is the only one. + // Authorizes the global-scoped endpoints: the JWKS and the two probes, all + // of them read-only and public. Anything else is refused here. async getGlobalAccessType(authInfo, purpose) { - return purpose === 'jwks' ? 'r' : null; + return purpose === 'jwks' || purpose === 'ping' || purpose === 'ready' + ? 'r' + : null; }, async getAccessType(authInfo, { org, docid, branch }, purpose) { if (authInfo.admin === true) { @@ -388,7 +400,70 @@ const eraseContent = async (yhub, room, by) => { await yhub.persistence.deleteTombstone(room); }; +const readyLog = logger.child({ module: 'readiness' }); + +// One readiness check: is that store answering? The error never leaves the +// server — the route is unauthenticated, and a postgres client is happy to put +// its connection string, password included, in the message it raises. +const checkStore = async (name, probe) => { + let timer; + try { + await Promise.race([ + probe(), + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`no answer in ${READINESS_TIMEOUT_MS}ms`)), + READINESS_TIMEOUT_MS, + ); + }), + ]); + return [name, 'ok']; + } catch (err) { + readyLog.warn({ store: name, err: err?.message }, 'store is unreachable'); + return [name, 'unreachable']; + } finally { + clearTimeout(timer); + } +}; + const api = [ + // GET /collaboration/ping/v1 — liveness. It answers, therefore the http + // channel and the event loop are alive, which is all a liveness probe should + // ever conclude: touching redis or postgres here would restart a server that + // holds perfectly good websocket connections every time a store blinks. + createApiEndpoint('ping', { + scope: 'global', + accessPurpose: 'ping', + get: { + handler: () => jsonResponse(200, { status: 'pong' }), + }, + }), + // GET /collaboration/ready/v1 — readiness. The two stores this server cannot + // serve a single document without: the postgres holding the persisted state + // and the redis carrying the updates between replicas. Answering 503 takes + // this pod out of the service endpoints and leaves the others serving, which + // is the whole difference with the liveness probe above. + createApiEndpoint('ready', { + scope: 'global', + accessPurpose: 'ready', + get: { + handler: async (req) => { + // both at once: a probe is not the place to add the latency of one + // store to the latency of the other + const checks = Object.fromEntries( + await Promise.all([ + checkStore('postgres', () => req.yhub.persistence.sql`SELECT 1`), + checkStore('redis', () => req.yhub.stream.redis.ping()), + ]), + ); + const ready = Object.values(checks).every((state) => state === 'ok'); + return jsonResponse(ready ? 200 : 503, { + status: ready ? 'ready' : 'unready', + checks, + }); + }, + }, + }), // GET /collaboration/jwks/v1 — the public keys verifying the tokens we sign // to call the backend, in the JSON Web Key Set format (RFC 7517). Global // scope: it is about this server, not about a document, so the route carries From f59588c6f87fb024b2862763f77f24aeb5ab9498 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Thu, 13 Aug 2026 17:50:41 +0200 Subject: [PATCH 53/59] =?UTF-8?q?=E2=9C=A8(yhub)=20allow=20to=20configure?= =?UTF-8?q?=20yhub=20worker=20and=20server?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We want to be able to configure both server and worker, the idea is to be able to deploy separately the server and the worker and to scale them. --- CHANGELOG.md | 10 ++ src/helm/env.d/dev/values.impress.yaml.gotmpl | 6 +- .../env.d/feature/values.impress.yaml.gotmpl | 6 +- src/helm/impress/README.md | 10 ++ src/helm/impress/templates/_helpers.tpl | 36 ++++ .../impress/templates/yhub_deployment.yaml | 4 +- .../templates/yhub_worker_deployment.yaml | 160 ++++++++++++++++++ src/helm/impress/values.yaml | 42 ++++- src/yhub-server/README.md | 30 ++++ src/yhub-server/server.js | 33 +++- 10 files changed, 329 insertions(+), 8 deletions(-) create mode 100644 src/helm/impress/templates/yhub_worker_deployment.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a51a46197..328e47c523 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -154,6 +154,16 @@ and this project adheres to and rolling the keys is deleting the secret and letting the next run create it again. `jwtKeys.existingSecret` points at keys of your own instead, and skips both the job and its rights +- ✨(collaboration) split the yhub server and worker with `YHUB_ROLE`: unset (or + `all`) runs both halves in one process as before, `server` holds the + websockets and the routes without claiming a task, `worker` drains the redis + stream into postgres without binding a port. They share the two stores and + nothing else, so each scales on what drives it — connected editors on one + side, write throughput on the other. Any other value is refused at startup. + In the helm chart, `yhub.worker.enabled` turns the single deployment into + two, sets the variable on each, and gives the worker no service and no probes + since it binds nothing; everything not named under `yhub.worker` is the + server's - ✨(collaboration) serve two probes on yhub: `GET /collaboration/ping/v1` answers `pong` without touching a store — being answered is what a liveness check should conclude, and restarting a server over a store it cannot reach diff --git a/src/helm/env.d/dev/values.impress.yaml.gotmpl b/src/helm/env.d/dev/values.impress.yaml.gotmpl index df87cc03a7..c4d1cc72d5 100644 --- a/src/helm/env.d/dev/values.impress.yaml.gotmpl +++ b/src/helm/env.d/dev/values.impress.yaml.gotmpl @@ -197,7 +197,11 @@ jwtKeys: enabled: true yhub: - replicas: 1 + replicas: 3 + + worker: + enabled: true + replicas: 2 image: repository: localhost:5001/impress-yhub diff --git a/src/helm/env.d/feature/values.impress.yaml.gotmpl b/src/helm/env.d/feature/values.impress.yaml.gotmpl index e2ff44aca6..dd4a5748f8 100644 --- a/src/helm/env.d/feature/values.impress.yaml.gotmpl +++ b/src/helm/env.d/feature/values.impress.yaml.gotmpl @@ -160,7 +160,11 @@ jwtKeys: enabled: true yhub: - replicas: 1 + replicas: 3 + + worker: + enabled: true + replicas: 2 image: repository: lasuite/impress-yhub diff --git a/src/helm/impress/README.md b/src/helm/impress/README.md index 1bf5fb7c95..545aaf0e35 100644 --- a/src/helm/impress/README.md +++ b/src/helm/impress/README.md @@ -348,6 +348,16 @@ | `yhub.command` | Override the yhub container command | `[]` | | `yhub.args` | Override the yhub container args | `[]` | | `yhub.replicas` | Amount of yhub replicas | `3` | +| `yhub.worker.enabled` | Deploy the worker apart from the server, each scaling on its own | `false` | +| `yhub.worker.replicas` | Amount of yhub worker replicas | `1` | +| `yhub.worker.resources` | Resource requirements for the yhub worker container, the server ones when empty | `{}` | +| `yhub.worker.podAnnotations` | Annotations to add to the yhub worker Pod, the server ones when empty | `{}` | +| `yhub.worker.dpAnnotations` | Annotations to add to the yhub worker Deployment, the server ones when empty | `{}` | +| `yhub.worker.nodeSelector` | Node selector for the yhub worker Pod, the server one when empty | `{}` | +| `yhub.worker.tolerations` | Tolerations for the yhub worker Pod, the server ones when empty | `[]` | +| `yhub.worker.affinity` | Affinity for the yhub worker Pod, the server one when empty | `{}` | +| `yhub.worker.terminationGracePeriodSeconds` | Grace period given to a worker pod to finish its task, the server one when empty | `nil` | +| `yhub.worker.pdb.enabled` | Enable pdb on the yhub worker | `true` | | `yhub.shareProcessNamespace` | Enable share process namespace between containers | `false` | | `yhub.sidecars` | Add sidecars containers to yhub deployment | `[]` | | `yhub.terminationGracePeriodSeconds` | Grace period given to a yhub pod to drain before it is killed | `60` | diff --git a/src/helm/impress/templates/_helpers.tpl b/src/helm/impress/templates/_helpers.tpl index 8f84245712..1b5d4a78f9 100644 --- a/src/helm/impress/templates/_helpers.tpl +++ b/src/helm/impress/templates/_helpers.tpl @@ -204,6 +204,42 @@ Requires top level scope {{ include "impress.fullname" . }}-yhub {{- end }} +{{/* +Full name for the yhub worker, when it is deployed apart from the server + +Requires top level scope +*/}} +{{- define "impress.yhub.worker.fullname" -}} +{{ include "impress.yhub.fullname" . }}-worker +{{- end }} + +{{/* +yhub worker env vars - combines common yhub.envVars with yhub.worker.envVars +*/}} +{{- define "impress.yhub.worker.env" -}} +{{- $topLevelScope := index . 0 -}} +{{- $workerScope := index . 1 -}} +{{- include "impress.env.transformDict" $workerScope.envVars -}} +{{- include "impress.env.transformDict" (($workerScope.worker | default dict).envVars | default dict) -}} +{{- end }} + +{{/* +The role a yhub pod runs, as an environment variable. Only when the worker is +deployed apart: a single deployment runs both halves, which is what yhub does +when the variable is absent. Skipped when the deployment names the role itself, +in either env map — an explicit value wins, as everywhere else here. + +Usage: {{ include "impress.yhub.roleEnv" (dict "root" $ "role" "server") }} +*/}} +{{- define "impress.yhub.roleEnv" -}} +{{- $root := .root -}} +{{- $named := merge (dict) (($root.Values.yhub.worker | default dict).envVars | default dict) ($root.Values.yhub.envVars | default dict) -}} +{{- if and $root.Values.yhub.worker.enabled (not (hasKey $named "YHUB_ROLE")) }} +- name: "YHUB_ROLE" + value: {{ .role | quote }} +{{- end }} +{{- end }} + {{/* JWT signing keys — the RSA keys the services sign the calls they make to each other with. The jwt-keys job generates them once into a secret every service diff --git a/src/helm/impress/templates/yhub_deployment.yaml b/src/helm/impress/templates/yhub_deployment.yaml index 8ca6772aac..cdaf3da092 100644 --- a/src/helm/impress/templates/yhub_deployment.yaml +++ b/src/helm/impress/templates/yhub_deployment.yaml @@ -54,12 +54,14 @@ spec: args: {{- toYaml . | nindent 12 }} {{- end }} - {{- if or $envVars .Values.jwtKeys.enabled }} + {{- $roleEnv := include "impress.yhub.roleEnv" (dict "root" . "role" "server") }} + {{- if or $envVars .Values.jwtKeys.enabled $roleEnv }} env: {{- $envVars | indent 12 }} {{- if .Values.jwtKeys.enabled }} {{- include "impress.jwtKeys.yhubEnv" . | nindent 12 }} {{- end }} + {{- $roleEnv | indent 12 }} {{- end }} {{- if .Values.yhub.envFrom }} envFrom: diff --git a/src/helm/impress/templates/yhub_worker_deployment.yaml b/src/helm/impress/templates/yhub_worker_deployment.yaml new file mode 100644 index 0000000000..0da9a6e7d5 --- /dev/null +++ b/src/helm/impress/templates/yhub_worker_deployment.yaml @@ -0,0 +1,160 @@ +{{- if and .Values.yhub.enabled .Values.yhub.worker.enabled -}} +{{- $envVars := include "impress.yhub.worker.env" (list . .Values.yhub) -}} +{{- $fullName := include "impress.yhub.worker.fullname" . -}} +{{- $component := "yhub-worker" -}} +{{- $worker := .Values.yhub.worker -}} +# The half of yhub that drains the redis stream into postgres, deployed apart +# from the one serving the websockets (YHUB_ROLE). It scales on the write +# throughput rather than on the connected editors, and redis consumer groups +# hand each task to exactly one of these pods. +# +# No service, no ports and no probes: a worker binds nothing, it claims tasks. +# Its health is its process being alive — the task loop logs and backs off on +# error rather than dying, so a pod that stopped working is one that exited, +# and kubelet restarts it. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $fullName }} + namespace: {{ .Release.Namespace | quote }} + annotations: + {{- with ($worker.dpAnnotations | default .Values.yhub.dpAnnotations) }} + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + {{- include "impress.common.labels" (list . $component) | nindent 4 }} +spec: + replicas: {{ $worker.replicas }} + selector: + matchLabels: + {{- include "impress.common.selectorLabels" (list . $component) | nindent 6 }} + template: + metadata: + annotations: + {{- with ($worker.podAnnotations | default .Values.yhub.podAnnotations) }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "impress.common.selectorLabels" (list . $component) | nindent 8 }} + spec: + {{- if $.Values.image.credentials }} + imagePullSecrets: + - name: {{ include "impress.secret.dockerconfigjson.name" (dict "fullname" (include "impress.fullname" .) "imageCredentials" $.Values.image.credentials) }} + {{- end}} + {{- if .Values.yhub.serviceAccountName }} + serviceAccountName: {{ .Values.yhub.serviceAccountName }} + {{- end }} + shareProcessNamespace: {{ .Values.yhub.shareProcessNamespace }} + # a task claimed by a pod that goes away is redelivered to another one, + # but letting the current one finish saves that round trip + terminationGracePeriodSeconds: {{ $worker.terminationGracePeriodSeconds | default .Values.yhub.terminationGracePeriodSeconds }} + containers: + {{- with .Values.yhub.sidecars }} + {{- toYaml . | nindent 8 }} + {{- end }} + - name: {{ .Chart.Name }} + image: "{{ (.Values.yhub.image | default dict).repository | default .Values.image.repository }}:{{ (.Values.yhub.image | default dict).tag | default .Values.image.tag }}" + imagePullPolicy: {{ (.Values.yhub.image | default dict).pullPolicy | default .Values.image.pullPolicy }} + {{- with .Values.yhub.command }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.yhub.args }} + args: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- $roleEnv := include "impress.yhub.roleEnv" (dict "root" . "role" "worker") }} + {{- if or $envVars .Values.jwtKeys.enabled $roleEnv }} + env: + {{- $envVars | indent 12 }} + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.yhubEnv" . | nindent 12 }} + {{- end }} + {{- $roleEnv | indent 12 }} + {{- end }} + {{- if .Values.yhub.envFrom }} + envFrom: + {{- toYaml .Values.yhub.envFrom | nindent 12 }} + {{- end }} + {{- with .Values.yhub.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with ($worker.resources | default .Values.yhub.resources) }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.volumeMount" . | nindent 12 }} + {{- end }} + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + mountPath: {{ $value.path }} + subPath: content + {{- end }} + {{- range .Values.yhub.extraVolumeMounts }} + - name: {{ .name }} + mountPath: {{ .mountPath }} + subPath: {{ .subPath | default "" }} + readOnly: {{ .readOnly }} + {{- end }} + {{- with ($worker.nodeSelector | default .Values.yhub.nodeSelector) }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with ($worker.affinity | default .Values.yhub.affinity) }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with ($worker.tolerations | default .Values.yhub.tolerations) }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + {{- if .Values.jwtKeys.enabled }} + {{- include "impress.jwtKeys.volume" . | nindent 8 }} + {{- end }} + {{- range $index, $value := .Values.mountFiles }} + - name: "files-{{ $index }}" + configMap: + name: "{{ include "impress.fullname" $ }}-files-{{ $index }}" + {{- end }} + {{- range .Values.yhub.extraVolumes }} + - name: {{ .name }} + {{- if .existingClaim }} + persistentVolumeClaim: + claimName: {{ .existingClaim }} + {{- else if .secret }} + secret: + {{ toYaml .secret | nindent 12 }} + {{- else if .hostPath }} + hostPath: + {{ toYaml .hostPath | nindent 12 }} + {{- else if .csi }} + csi: + {{- toYaml .csi | nindent 12 }} + {{- else if .configMap }} + configMap: + {{- toYaml .configMap | nindent 12 }} + {{- else if .emptyDir }} + emptyDir: + {{- toYaml .emptyDir | nindent 12 }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} +--- +{{ if $worker.pdb.enabled }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ $fullName }} + namespace: {{ .Release.Namespace | quote }} +spec: + maxUnavailable: 1 + selector: + matchLabels: + {{- include "impress.common.selectorLabels" (list . $component) | nindent 6 }} +{{ end }} +{{- end }} diff --git a/src/helm/impress/values.yaml b/src/helm/impress/values.yaml index 35b33708a2..35064b7efd 100644 --- a/src/helm/impress/values.yaml +++ b/src/helm/impress/values.yaml @@ -928,11 +928,47 @@ yhub: args: [] ## @param yhub.replicas Amount of yhub replicas - ## Every replica also runs a worker (redis consumer groups hand each task to - ## one of them), and clients editing the same document need not land on the - ## same pod: updates travel through redis. + ## Clients editing the same document need not land on the same pod: updates + ## travel through redis. Each replica also runs a worker unless the worker is + ## deployed apart, see below. replicas: 3 + ## @param yhub.worker.enabled Deploy the worker apart from the server, each scaling on its own + ## @param yhub.worker.replicas Amount of yhub worker replicas + ## @param yhub.worker.resources Resource requirements for the yhub worker container, the server ones when empty + ## @param yhub.worker.podAnnotations Annotations to add to the yhub worker Pod, the server ones when empty + ## @param yhub.worker.dpAnnotations Annotations to add to the yhub worker Deployment, the server ones when empty + ## @param yhub.worker.nodeSelector Node selector for the yhub worker Pod, the server one when empty + ## @param yhub.worker.tolerations Tolerations for the yhub worker Pod, the server ones when empty + ## @param yhub.worker.affinity Affinity for the yhub worker Pod, the server one when empty + ## @param yhub.worker.terminationGracePeriodSeconds Grace period given to a worker pod to finish its task, the server one when empty + ## @param yhub.worker.pdb.enabled Enable pdb on the yhub worker + ## @skip yhub.worker.envVars Environment variables of the worker only, on top of yhub.envVars + ## + ## yhub is two halves sharing nothing but redis and postgres: the server + ## holds the websockets and serves the routes, the worker drains the stream + ## into postgres. One process runs both by default. Enabling this splits them + ## into two deployments — `YHUB_ROLE=server` and `YHUB_ROLE=worker`, the only + ## difference between them — so the server scales with the connected editors + ## and the worker with the write throughput. + ## + ## The worker binds nothing: no service, no ingress, and no probes to give it + ## (its liveness is its process). Everything not named here is the server's: + ## same image, same envVars, same secrets, same volumes. + worker: + enabled: false + replicas: 1 + envVars: {} + resources: {} + podAnnotations: {} + dpAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + terminationGracePeriodSeconds: null + pdb: + enabled: true + ## @param yhub.shareProcessNamespace Enable share process namespace between containers shareProcessNamespace: false diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index 324ff0a2b1..815b072c30 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -89,6 +89,36 @@ probes are not worth publishing either — kubelet calls them from inside — an the helm chart's ingress lists what it routes rather than what it hides, so they stay in-cluster on their own. +## Roles (`YHUB_ROLE`) + +yhub is two halves that share the two stores and nothing else — no in-process +state, no ordering between them: + +- the **server** accepts the websocket connections, serves the routes above, + and writes every update to the redis stream, +- the **worker** claims tasks from that stream, merges the updates and stores + the result in postgres, then trims what it persisted. + +One process runs both, which is the default and what `YHUB_ROLE` unset means. +Setting it splits them, so each can be scaled on its own — the server with the +connected editors, the worker with the write throughput: + +| `YHUB_ROLE` | websocket + routes | drains the stream | +| ----------- | ------------------ | ----------------- | +| unset, `all` | yes | yes | +| `server` | yes | no | +| `worker` | no | yes | + +A `worker` process binds no port: no probes to give it and no service to put in +front of it. A `server` process claims no task, so a deployment of servers +alone accepts edits and never persists them — the two halves are split +together or not at all. Any other value is refused at startup rather than +guessed. + +Redis consumer groups hand each task to exactly one worker, so the number of +workers is a throughput knob and nothing else: no leader, no partitioning, no +coordination between them. + ## Container image The `Dockerfile` has two final stages, like the other services of this diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index d63a4e1f91..2ac2a04111 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -39,6 +39,24 @@ const allowedOrigins = ( ).split(','); const Y_PROVIDER_API_KEY = secret('Y_PROVIDER_API_KEY', 'yprovider-api-key'); const ORG = process.env.YHUB_ORG || 'docs'; +// Which halves of yhub this process runs. The server accepts the websocket +// connections and serves the REST routes; the worker drains the redis stream +// into postgres. They share the two stores and nothing else — no in-process +// state, no ordering between them — so one process can run both (the default) +// or a deployment can split them and scale each on its own: the server with +// the connected editors, the worker with the write throughput. +// +// A stream is only drained by the workers that are running: a deployment of +// `server` alone keeps accepting edits and never persists them, so the two +// halves are split together or not at all. +const ROLE = process.env.YHUB_ROLE || 'all'; +if (!['all', 'server', 'worker'].includes(ROLE)) { + throw new Error( + `YHUB_ROLE must be one of "all", "server" or "worker" (got "${ROLE}")`, + ); +} +const RUNS_SERVER = ROLE !== 'worker'; +const RUNS_WORKER = ROLE !== 'server'; // Segment every route is mounted under (`server.apiPrefix` below), matching the // URL scheme Docs already routes to the collaboration server. Hardcoded like // the audiences: the backend builds its urls with the same prefix. @@ -818,8 +836,19 @@ const yhub = await createYHub({ }, postgres: POSTGRES, persistence: [], // blobs live in yhub's postgres + // Both halves are declared, and YHUB_ROLE decides which are built: a null + // server binds no port at all (a `worker` pod has no http surface, hence no + // probes and no service in front of it), a null worker claims no task. + // // apiPrefix mounts every route — built-ins, our custom endpoints, and the // websocket (/collaboration/ws/v1/{org}/{docid}) — under /collaboration/. - server: { port: PORT, auth, api, apiPrefix: API_PREFIX }, - worker: { taskConcurrency: 5, events: workerEvents }, + server: RUNS_SERVER + ? { port: PORT, auth, api, apiPrefix: API_PREFIX } + : null, + worker: RUNS_WORKER ? { taskConcurrency: 5, events: workerEvents } : null, }); + +logger.info( + { role: ROLE, server: RUNS_SERVER, worker: RUNS_WORKER }, + 'yhub role', +); From a423f05e5bcbf06f0dd9998536c0b18622148493 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Fri, 14 Aug 2026 09:38:56 +0200 Subject: [PATCH 54/59] =?UTF-8?q?=F0=9F=94=A7(thub)=20allow=20to=20configu?= =?UTF-8?q?re=20task=20concurrency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We want to configure the number of concurrency tasks a work run. For this a new environment variable YHUB_TASK_CONCURRENCY is added --- CHANGELOG.md | 4 +++- src/helm/env.d/dev/values.impress.yaml.gotmpl | 1 + .../env.d/feature/values.impress.yaml.gotmpl | 2 ++ src/helm/impress/README.md | 1 + src/helm/impress/templates/_helpers.tpl | 10 ++++++-- src/helm/impress/values.yaml | 6 +++++ src/yhub-server/README.md | 8 +++++++ src/yhub-server/server.js | 23 +++++++++++++++++-- 8 files changed, 50 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 328e47c523..a1a56bb9a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -163,7 +163,9 @@ and this project adheres to In the helm chart, `yhub.worker.enabled` turns the single deployment into two, sets the variable on each, and gives the worker no service and no probes since it binds nothing; everything not named under `yhub.worker` is the - server's + server's. `YHUB_TASK_CONCURRENCY` (default 5, unchanged) sets how many tasks + one worker process claims at once — the other half of the throughput knob the + replica count is, since redis hands each task to a single worker - ✨(collaboration) serve two probes on yhub: `GET /collaboration/ping/v1` answers `pong` without touching a store — being answered is what a liveness check should conclude, and restarting a server over a store it cannot reach diff --git a/src/helm/env.d/dev/values.impress.yaml.gotmpl b/src/helm/env.d/dev/values.impress.yaml.gotmpl index c4d1cc72d5..ff1930c25e 100644 --- a/src/helm/env.d/dev/values.impress.yaml.gotmpl +++ b/src/helm/env.d/dev/values.impress.yaml.gotmpl @@ -219,6 +219,7 @@ yhub: COLLABORATION_SERVER_ORIGIN: https://docs.127.0.0.1.nip.io NODE_EXTRA_CA_CERTS: /cert/cacert.pem # YHUB_JWT_PRIVATE_KEY_FILE comes from the jwtKeys job below + LOG_LEVEL: debug # Extra volume mounts to manage our local custom CA and avoid to set ssl_verify: false extraVolumeMounts: diff --git a/src/helm/env.d/feature/values.impress.yaml.gotmpl b/src/helm/env.d/feature/values.impress.yaml.gotmpl index dd4a5748f8..9fc55e9e3e 100644 --- a/src/helm/env.d/feature/values.impress.yaml.gotmpl +++ b/src/helm/env.d/feature/values.impress.yaml.gotmpl @@ -180,6 +180,8 @@ yhub: COLLABORATION_BACKEND_BASE_URL: https://{{ .Values.feature }}-docs.{{ .Values.domain }} COLLABORATION_SERVER_ORIGIN: https://{{ .Values.feature }}-docs.{{ .Values.domain }} NODE_OPTIONS: "--max-old-space-size=1024" + UWS_HTTP_MAX_HEADERS_SIZE: 32768 + LOG_LEVEL: debug docSpec: enabled: true diff --git a/src/helm/impress/README.md b/src/helm/impress/README.md index 545aaf0e35..ef359fe4dd 100644 --- a/src/helm/impress/README.md +++ b/src/helm/impress/README.md @@ -374,6 +374,7 @@ | `yhub.envVars.COLLABORATION_BACKEND_BASE_URL` | Base url of the Docs backend, which yhub asks about users and document access rights | | | `yhub.envVars.COLLABORATION_SERVER_ORIGIN` | Comma separated list of the origins allowed to open a websocket | | | `yhub.envVars.YHUB_JWT_PRIVATE_KEY_FILE` | Path to the RSA private key (PEM) yhub signs its calls to the backend with, mounted from a secret | | +| `yhub.envVars.YHUB_TASK_CONCURRENCY` | Tasks one worker process claims at once, times the replicas running a worker (default: 5) | | | `yhub.envVars.SOFT_MIGRATION` | Set to "true" to seed rooms from the legacy Django/S3 document store on first access | | | `yhub.envVars.BY_VALUE` | Example environment variable by setting value directly | | | `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name` | Name of a ConfigMap when configuring env vars from a ConfigMap | | diff --git a/src/helm/impress/templates/_helpers.tpl b/src/helm/impress/templates/_helpers.tpl index 1b5d4a78f9..b3e39c2674 100644 --- a/src/helm/impress/templates/_helpers.tpl +++ b/src/helm/impress/templates/_helpers.tpl @@ -215,12 +215,18 @@ Requires top level scope {{/* yhub worker env vars - combines common yhub.envVars with yhub.worker.envVars + +Merged rather than appended: a variable the worker sets differently from the +server (YHUB_TASK_CONCURRENCY, typically) is meant to replace it, and emitting +both would leave the value to kubernetes' last-one-wins rule and show the +variable twice in the pod. deepCopy because merge writes into its first +argument, which is a live values map. */}} {{- define "impress.yhub.worker.env" -}} {{- $topLevelScope := index . 0 -}} {{- $workerScope := index . 1 -}} -{{- include "impress.env.transformDict" $workerScope.envVars -}} -{{- include "impress.env.transformDict" (($workerScope.worker | default dict).envVars | default dict) -}} +{{- $workerEnvVars := ($workerScope.worker | default dict).envVars | default dict -}} +{{- include "impress.env.transformDict" (merge (deepCopy $workerEnvVars) $workerScope.envVars) -}} {{- end }} {{/* diff --git a/src/helm/impress/values.yaml b/src/helm/impress/values.yaml index 35064b7efd..85e90de279 100644 --- a/src/helm/impress/values.yaml +++ b/src/helm/impress/values.yaml @@ -955,6 +955,11 @@ yhub: ## The worker binds nothing: no service, no ingress, and no probes to give it ## (its liveness is its process). Everything not named here is the server's: ## same image, same envVars, same secrets, same volumes. + ## + ## How much each pod chews through is `worker.envVars.YHUB_TASK_CONCURRENCY` + ## (default 5, and `yhub.envVars` when the two halves share a process), the + ## other half of the throughput knob `worker.replicas` is: redis hands each + ## task to a single worker, so the two multiply. worker: enabled: false replicas: 1 @@ -1008,6 +1013,7 @@ yhub: ## @extra yhub.envVars.COLLABORATION_BACKEND_BASE_URL Base url of the Docs backend, which yhub asks about users and document access rights ## @extra yhub.envVars.COLLABORATION_SERVER_ORIGIN Comma separated list of the origins allowed to open a websocket ## @extra yhub.envVars.YHUB_JWT_PRIVATE_KEY_FILE Path to the RSA private key (PEM) yhub signs its calls to the backend with, mounted from a secret + ## @extra yhub.envVars.YHUB_TASK_CONCURRENCY Tasks one worker process claims at once, times the replicas running a worker (default: 5) ## @extra yhub.envVars.SOFT_MIGRATION Set to "true" to seed rooms from the legacy Django/S3 document store on first access ## @extra yhub.envVars.BY_VALUE Example environment variable by setting value directly ## @extra yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name Name of a ConfigMap when configuring env vars from a ConfigMap diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index 815b072c30..a4c282cb6f 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -119,6 +119,14 @@ Redis consumer groups hand each task to exactly one worker, so the number of workers is a throughput knob and nothing else: no leader, no partitioning, no coordination between them. +`YHUB_TASK_CONCURRENCY` (default `5`) is the other half of that knob: how many +tasks one process claims at once. What actually runs in parallel is that number +times the processes running a worker, so the two are interchangeable up to the +point where a pod runs out of memory — each task holds the document it merges. +A value that is not a positive integer is refused at startup, like an unknown +role: `Number()` would otherwise read a typo as `NaN` and leave the worker +claiming nothing. + ## Container image The `Dockerfile` has two final stages, like the other services of this diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index 2ac2a04111..273f9c36aa 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -57,6 +57,18 @@ if (!['all', 'server', 'worker'].includes(ROLE)) { } const RUNS_SERVER = ROLE !== 'worker'; const RUNS_WORKER = ROLE !== 'server'; +// How many tasks one worker process claims at once. Redis hands each task to a +// single worker, so what a deployment actually runs in parallel is this times +// the number of worker processes — the two knobs are interchangeable up to the +// point where a pod runs out of memory, each task holding the document it +// merges. Refused rather than guessed when it is not a positive integer: +// `Number()` would otherwise turn a typo into NaN and yhub into an idle worker. +const TASK_CONCURRENCY = Number(process.env.YHUB_TASK_CONCURRENCY || 5); +if (!Number.isInteger(TASK_CONCURRENCY) || TASK_CONCURRENCY < 1) { + throw new Error( + `YHUB_TASK_CONCURRENCY must be a positive integer (got "${process.env.YHUB_TASK_CONCURRENCY}")`, + ); +} // Segment every route is mounted under (`server.apiPrefix` below), matching the // URL scheme Docs already routes to the collaboration server. Hardcoded like // the audiences: the backend builds its urls with the same prefix. @@ -845,10 +857,17 @@ const yhub = await createYHub({ server: RUNS_SERVER ? { port: PORT, auth, api, apiPrefix: API_PREFIX } : null, - worker: RUNS_WORKER ? { taskConcurrency: 5, events: workerEvents } : null, + worker: RUNS_WORKER + ? { taskConcurrency: TASK_CONCURRENCY, events: workerEvents } + : null, }); logger.info( - { role: ROLE, server: RUNS_SERVER, worker: RUNS_WORKER }, + { + role: ROLE, + server: RUNS_SERVER, + worker: RUNS_WORKER, + taskConcurrency: RUNS_WORKER ? TASK_CONCURRENCY : null, + }, 'yhub role', ); From 55c9b22c2861486f0300eec6202c3257b728fea2 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Fri, 14 Aug 2026 10:04:37 +0200 Subject: [PATCH 55/59] =?UTF-8?q?=F0=9F=94=A7(yhub)=20allow=20to=20configu?= =?UTF-8?q?re=20every=20createYHub=20parameters?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the redis section there were still hard coded values, we want to allow the configurations of this settings. The last part will be the persistence plugin. --- CHANGELOG.md | 11 ++++++++ src/helm/impress/README.md | 2 ++ src/helm/impress/values.yaml | 2 ++ src/yhub-server/README.md | 48 ++++++++++++++++++++++++++++----- src/yhub-server/server.js | 51 ++++++++++++++++++++++++++++-------- 5 files changed, 96 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1a56bb9a8..806970d6d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -166,6 +166,17 @@ and this project adheres to server's. `YHUB_TASK_CONCURRENCY` (default 5, unchanged) sets how many tasks one worker process claims at once — the other half of the throughput knob the replica count is, since redis hands each task to a single worker +- ✨(collaboration) configure the two stream timings that were compiled into + the yhub wrapper: `YHUB_TASK_DEBOUNCE_MS` (default 10000, unchanged), how + long an update waits on the redis stream before a worker persists it — the + delay between an edit and its row in postgres, and the window over which the + edits of a busy document are merged into one task — and + `YHUB_MIN_MESSAGE_LIFETIME_MS` (default 60000, unchanged), how long persisted + updates stay replayable from redis rather than being read back out of + postgres. Neither is a durability setting: the trim never goes past what + postgres holds. Like the concurrency, they are refused at startup when they + are not whole numbers in range, and all of them are logged with the role on + the first line a pod writes - ✨(collaboration) serve two probes on yhub: `GET /collaboration/ping/v1` answers `pong` without touching a store — being answered is what a liveness check should conclude, and restarting a server over a store it cannot reach diff --git a/src/helm/impress/README.md b/src/helm/impress/README.md index ef359fe4dd..d59691da79 100644 --- a/src/helm/impress/README.md +++ b/src/helm/impress/README.md @@ -375,6 +375,8 @@ | `yhub.envVars.COLLABORATION_SERVER_ORIGIN` | Comma separated list of the origins allowed to open a websocket | | | `yhub.envVars.YHUB_JWT_PRIVATE_KEY_FILE` | Path to the RSA private key (PEM) yhub signs its calls to the backend with, mounted from a secret | | | `yhub.envVars.YHUB_TASK_CONCURRENCY` | Tasks one worker process claims at once, times the replicas running a worker (default: 5) | | +| `yhub.envVars.YHUB_TASK_DEBOUNCE_MS` | How long an update waits on the redis stream before a worker persists it, in ms (default: 10000) | | +| `yhub.envVars.YHUB_MIN_MESSAGE_LIFETIME_MS` | How long persisted updates stay replayable from redis, in ms (default: 60000) | | | `yhub.envVars.SOFT_MIGRATION` | Set to "true" to seed rooms from the legacy Django/S3 document store on first access | | | `yhub.envVars.BY_VALUE` | Example environment variable by setting value directly | | | `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name` | Name of a ConfigMap when configuring env vars from a ConfigMap | | diff --git a/src/helm/impress/values.yaml b/src/helm/impress/values.yaml index 85e90de279..88d646ca52 100644 --- a/src/helm/impress/values.yaml +++ b/src/helm/impress/values.yaml @@ -1014,6 +1014,8 @@ yhub: ## @extra yhub.envVars.COLLABORATION_SERVER_ORIGIN Comma separated list of the origins allowed to open a websocket ## @extra yhub.envVars.YHUB_JWT_PRIVATE_KEY_FILE Path to the RSA private key (PEM) yhub signs its calls to the backend with, mounted from a secret ## @extra yhub.envVars.YHUB_TASK_CONCURRENCY Tasks one worker process claims at once, times the replicas running a worker (default: 5) + ## @extra yhub.envVars.YHUB_TASK_DEBOUNCE_MS How long an update waits on the redis stream before a worker persists it, in ms (default: 10000) + ## @extra yhub.envVars.YHUB_MIN_MESSAGE_LIFETIME_MS How long persisted updates stay replayable from redis, in ms (default: 60000) ## @extra yhub.envVars.SOFT_MIGRATION Set to "true" to seed rooms from the legacy Django/S3 document store on first access ## @extra yhub.envVars.BY_VALUE Example environment variable by setting value directly ## @extra yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name Name of a ConfigMap when configuring env vars from a ConfigMap diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index a4c282cb6f..0a7d97e3fb 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -119,13 +119,47 @@ Redis consumer groups hand each task to exactly one worker, so the number of workers is a throughput knob and nothing else: no leader, no partitioning, no coordination between them. -`YHUB_TASK_CONCURRENCY` (default `5`) is the other half of that knob: how many -tasks one process claims at once. What actually runs in parallel is that number -times the processes running a worker, so the two are interchangeable up to the -point where a pod runs out of memory — each task holds the document it merges. -A value that is not a positive integer is refused at startup, like an unknown -role: `Number()` would otherwise read a typo as `NaN` and leave the worker -claiming nothing. +`YHUB_TASK_CONCURRENCY` is the other half of that knob — see below. + +## Tuning + +Three numbers this wrapper passes to yhub, all of them environment variables +whose defaults are what Docs ran with before they were configurable: + +| Variable | Default | What it changes | +| -------- | ------- | --------------- | +| `YHUB_TASK_CONCURRENCY` | `5` | Tasks one worker process claims at once | +| `YHUB_TASK_DEBOUNCE_MS` | `10000` | How long an update waits on the stream before a worker persists it | +| `YHUB_MIN_MESSAGE_LIFETIME_MS` | `60000` | How long persisted updates stay replayable from redis | + +**Concurrency** multiplies with the number of processes running a worker, since +redis hands each task to exactly one of them: the two are interchangeable up to +the point where a pod runs out of memory, each task holding the document it +merges. + +**The debounce** is the delay between an edit and its row in postgres, and the +window over which the edits of a busy document are merged into a single task. +Lowering it persists sooner and compacts more often; raising it does the +reverse. yhub's own default is 120s, which is a long time to lose when a pod is +killed, hence the 10s here. + +**The message lifetime** is not a durability setting: the trim stops at the +older of that age and the point postgres already holds, so nothing unpersisted +is ever dropped. It buys how much recent history a server can replay from redis +instead of reading the document back out of postgres, and it is paid for in +redis memory. + +All three are refused at startup, like an unknown role, when they are not whole +numbers in range (`YHUB_TASK_CONCURRENCY must be an integer >= 1 (got "abc")`): +`Number()` would otherwise read a typo as `NaN` and hand it to yhub, which +takes it — a worker that claims nothing, or a stream that is never trimmed, +with nothing in the logs to say so. Unset and empty both mean the default, so a +kubernetes variable left blank behaves as if it were absent. The effective +values are logged at startup, next to the role: + +```json +{"role":"all","server":true,"worker":true,"taskConcurrency":5,"taskDebounceMs":10000,"minMessageLifetimeMs":60000,"msg":"yhub configuration"} +``` ## Container image diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index 273f9c36aa..16697c14db 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -28,10 +28,38 @@ import { migrationLog, } from './migration.js'; +// A numeric setting, read from the environment and refused rather than guessed +// when it is not a whole number at or above `min`: `Number()` reads a typo as +// NaN, which yhub takes as-is and turns into a worker that claims nothing or a +// stream that is never trimmed — a deployment that looks healthy and is not. +// An unset or empty variable is the default, so a kubernetes env var left blank +// behaves as if it had not been set at all. +const intEnv = (name, dflt, min = 1) => { + const raw = process.env[name]; + const value = raw == null || raw === '' ? dflt : Number(raw); + if (!Number.isInteger(value) || value < min) { + throw new Error(`${name} must be an integer >= ${min} (got "${raw}")`); + } + return value; +}; + const PORT = Number(process.env.PORT || 3002); const REDIS = process.env.REDIS; const POSTGRES = process.env.POSTGRES; const REDIS_PREFIX = process.env.REDIS_PREFIX || 'yhub'; +// How long an update waits on the stream before a worker claims the compaction +// task it belongs to. It is the delay between an edit and its row in postgres, +// and the window over which the edits of a busy document are merged into one +// task: lowering it persists sooner and compacts more often, raising it does +// the reverse. yhub defaults to 120s, which is a long time to lose when a pod +// is killed — Docs asks for 10s. +const TASK_DEBOUNCE_MS = intEnv('YHUB_TASK_DEBOUNCE_MS', 10000, 0); +// How long messages a worker has already persisted are kept on the stream. The +// trim stops at the older of that age and the point postgres holds, so this is +// not a durability setting — nothing unpersisted is ever trimmed. It is how +// much recent history stays replayable from redis instead of being read back +// out of postgres, paid for in memory on the redis side. +const MIN_MESSAGE_LIFETIME_MS = intEnv('YHUB_MIN_MESSAGE_LIFETIME_MS', 60000, 0); const COLLABORATION_BACKEND_BASE_URL = process.env.COLLABORATION_BACKEND_BASE_URL || 'http://app-dev:8000'; const allowedOrigins = ( @@ -61,14 +89,8 @@ const RUNS_WORKER = ROLE !== 'server'; // single worker, so what a deployment actually runs in parallel is this times // the number of worker processes — the two knobs are interchangeable up to the // point where a pod runs out of memory, each task holding the document it -// merges. Refused rather than guessed when it is not a positive integer: -// `Number()` would otherwise turn a typo into NaN and yhub into an idle worker. -const TASK_CONCURRENCY = Number(process.env.YHUB_TASK_CONCURRENCY || 5); -if (!Number.isInteger(TASK_CONCURRENCY) || TASK_CONCURRENCY < 1) { - throw new Error( - `YHUB_TASK_CONCURRENCY must be a positive integer (got "${process.env.YHUB_TASK_CONCURRENCY}")`, - ); -} +// merges. +const TASK_CONCURRENCY = intEnv('YHUB_TASK_CONCURRENCY', 5, 1); // Segment every route is mounted under (`server.apiPrefix` below), matching the // URL scheme Docs already routes to the collaboration server. Hardcoded like // the audiences: the backend builds its urls with the same prefix. @@ -843,8 +865,8 @@ const yhub = await createYHub({ redis: { url: REDIS, prefix: REDIS_PREFIX, - taskDebounce: 10000, - minMessageLifetime: 60000, + taskDebounce: TASK_DEBOUNCE_MS, + minMessageLifetime: MIN_MESSAGE_LIFETIME_MS, }, postgres: POSTGRES, persistence: [], // blobs live in yhub's postgres @@ -862,12 +884,19 @@ const yhub = await createYHub({ : null, }); +// What this process was configured to be, in one line: yhub's own startup log +// reports neither the role nor the stream settings, and every one of them is an +// environment variable a deployment can get wrong. The two timings are read +// back off the instance rather than from the constants above, so the line says +// what yhub is using and not merely what it was asked for. logger.info( { role: ROLE, server: RUNS_SERVER, worker: RUNS_WORKER, taskConcurrency: RUNS_WORKER ? TASK_CONCURRENCY : null, + taskDebounceMs: yhub.stream.taskDebounce, + minMessageLifetimeMs: yhub.stream.minMessageLifetime, }, - 'yhub role', + 'yhub configuration', ); From a50374fdf8413937cfa396746eaa8e6f8ca783a0 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Fri, 14 Aug 2026 10:34:50 +0200 Subject: [PATCH 56/59] =?UTF-8?q?=E2=99=BB=EF=B8=8F(yhub)=20prefix=20S3=20?= =?UTF-8?q?envirionment=20variables=20used=20by=20the=20migration?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We want to allow the usage of different buckets when migrating legascy documents. --- CHANGELOG.md | 7 +- env.d/development/common | 9 ++ src/helm/impress/README.md | 169 ++++++++++++++++++----------------- src/helm/impress/values.yaml | 12 +++ src/yhub-server/README.md | 16 +++- src/yhub-server/migration.js | 40 +++++---- 6 files changed, 150 insertions(+), 103 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 806970d6d4..2c20868210 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,7 +85,12 @@ and this project adheres to top of content that exists. Backend reads carrying the admin JWT are seeded too, so a server-side read of an unmigrated document never answers with an empty one. Enabled in the dev - stack via compose.yml + stack via compose.yml. The bucket it reads is configured under `LEGACY_S3_*` + (`_ENDPOINT_URL`, `_ACCESS_KEY_ID`, `_SECRET_ACCESS_KEY`, `_REGION_NAME`, + `_BUCKET_NAME`), a set of its own and not the backend's `AWS_S3_*`: this is + the bucket the collaboration server migrates *out of*, while the one it will + persist *into* when the yhub S3 persistence plugin is enabled is a separate + bucket that may well sit on another provider with credentials of its own - ✨(collaboration) add a migrate endpoint on yhub: `POST /collaboration/migrate/v1/docs/{id}` replays a document's **full** legacy version history from the versioned S3 media bucket into a diff --git a/env.d/development/common b/env.d/development/common index 4b09afab7b..954128b041 100644 --- a/env.d/development/common +++ b/env.d/development/common @@ -37,6 +37,15 @@ AWS_S3_ACCESS_KEY_ID=impress AWS_S3_SECRET_ACCESS_KEY=password MEDIA_BASE_URL=http://localhost:8083 +# The same bucket, read by yhub's soft migration under names of its own: the +# bucket the collaboration server migrates *out of* is not the one it will +# persist *into* once the S3 persistence plugin is turned on, so it does not +# read the backend's AWS_S3_* settings. Locally they hold the same minio, and +# an override of the three above wants the same override here. +LEGACY_S3_ENDPOINT_URL=http://minio:9000 +LEGACY_S3_ACCESS_KEY_ID=impress +LEGACY_S3_SECRET_ACCESS_KEY=password + # OIDC OIDC_OP_JWKS_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/certs OIDC_OP_AUTHORIZATION_ENDPOINT=http://localhost:8083/realms/impress/protocol/openid-connect/auth diff --git a/src/helm/impress/README.md b/src/helm/impress/README.md index d59691da79..d20b054790 100644 --- a/src/helm/impress/README.md +++ b/src/helm/impress/README.md @@ -339,88 +339,93 @@ ### yhub -| Name | Description | Value | -| -------------------------------------------------- | ------------------------------------------------------------------------------------------------------------- | ------------------------- | -| `yhub.enabled` | Enable the yhub collaboration server, its service and its init-db job | `true` | -| `yhub.image.repository` | Repository to use to pull the yhub container image | `lasuite/impress-yhub` | -| `yhub.image.tag` | yhub container tag | `latest` | -| `yhub.image.pullPolicy` | yhub container image pull policy | `IfNotPresent` | -| `yhub.command` | Override the yhub container command | `[]` | -| `yhub.args` | Override the yhub container args | `[]` | -| `yhub.replicas` | Amount of yhub replicas | `3` | -| `yhub.worker.enabled` | Deploy the worker apart from the server, each scaling on its own | `false` | -| `yhub.worker.replicas` | Amount of yhub worker replicas | `1` | -| `yhub.worker.resources` | Resource requirements for the yhub worker container, the server ones when empty | `{}` | -| `yhub.worker.podAnnotations` | Annotations to add to the yhub worker Pod, the server ones when empty | `{}` | -| `yhub.worker.dpAnnotations` | Annotations to add to the yhub worker Deployment, the server ones when empty | `{}` | -| `yhub.worker.nodeSelector` | Node selector for the yhub worker Pod, the server one when empty | `{}` | -| `yhub.worker.tolerations` | Tolerations for the yhub worker Pod, the server ones when empty | `[]` | -| `yhub.worker.affinity` | Affinity for the yhub worker Pod, the server one when empty | `{}` | -| `yhub.worker.terminationGracePeriodSeconds` | Grace period given to a worker pod to finish its task, the server one when empty | `nil` | -| `yhub.worker.pdb.enabled` | Enable pdb on the yhub worker | `true` | -| `yhub.shareProcessNamespace` | Enable share process namespace between containers | `false` | -| `yhub.sidecars` | Add sidecars containers to yhub deployment | `[]` | -| `yhub.terminationGracePeriodSeconds` | Grace period given to a yhub pod to drain before it is killed | `60` | -| `yhub.securityContext.allowPrivilegeEscalation` | Whether to allow privilege escalation for the yhub container | `false` | -| `yhub.securityContext.capabilities.drop` | List of capabilities to drop for the yhub container | `["ALL"]` | -| `yhub.securityContext.runAsNonRoot` | Whether to run the yhub container as a non-root user | `true` | -| `yhub.securityContext.runAsUser` | User the yhub container runs as | `1000` | -| `yhub.securityContext.runAsGroup` | Group the yhub container runs as | `1000` | -| `yhub.securityContext.seccompProfile.type` | Seccomp profile type for the yhub container | `RuntimeDefault` | -| `yhub.envVars` | Configure yhub container environment variables | `undefined` | -| `yhub.envVars.REDIS` | Required, redis/valkey url holding the live document state (e.g. redis://valkey:6379/0) | | -| `yhub.envVars.POSTGRES` | Required, url of the yhub database, created by the init-db job (e.g. postgres://user:pass@postgres:5432/yhub) | | -| `yhub.envVars.REDIS_PREFIX` | Namespace of the redis keys, when the instance is shared (default: yhub) | | -| `yhub.envVars.COLLABORATION_BACKEND_BASE_URL` | Base url of the Docs backend, which yhub asks about users and document access rights | | -| `yhub.envVars.COLLABORATION_SERVER_ORIGIN` | Comma separated list of the origins allowed to open a websocket | | -| `yhub.envVars.YHUB_JWT_PRIVATE_KEY_FILE` | Path to the RSA private key (PEM) yhub signs its calls to the backend with, mounted from a secret | | -| `yhub.envVars.YHUB_TASK_CONCURRENCY` | Tasks one worker process claims at once, times the replicas running a worker (default: 5) | | -| `yhub.envVars.YHUB_TASK_DEBOUNCE_MS` | How long an update waits on the redis stream before a worker persists it, in ms (default: 10000) | | -| `yhub.envVars.YHUB_MIN_MESSAGE_LIFETIME_MS` | How long persisted updates stay replayable from redis, in ms (default: 60000) | | -| `yhub.envVars.SOFT_MIGRATION` | Set to "true" to seed rooms from the legacy Django/S3 document store on first access | | -| `yhub.envVars.BY_VALUE` | Example environment variable by setting value directly | | -| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name` | Name of a ConfigMap when configuring env vars from a ConfigMap | | -| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.key` | Key within a ConfigMap when configuring env vars from a ConfigMap | | -| `yhub.envVars.FROM_SECRET.secretKeyRef.name` | Name of a Secret when configuring env vars from a Secret | | -| `yhub.envVars.FROM_SECRET.secretKeyRef.key` | Key within a Secret when configuring env vars from a Secret | | -| `yhub.podAnnotations` | Annotations to add to the yhub Pod | `{}` | -| `yhub.dpAnnotations` | Annotations to add to the yhub Deployment | `{}` | -| `yhub.initDbJobAnnotations` | Annotations for the yhub init-db job | `{}` | -| `yhub.jobs.ttlSecondsAfterFinished` | Period to wait before removing the init-db job | `30` | -| `yhub.jobs.backoffLimit` | Numbers of init-db job retries | `2` | -| `yhub.initDb.enabled` | Run the job creating and upgrading the yhub schema | `true` | -| `yhub.initDb.command` | Override the command creating and upgrading the yhub schema | `[]` | -| `yhub.initDb.retries` | How many times the schema script is retried while the postgres server does not answer | `60` | -| `yhub.initDb.retryDelaySeconds` | Seconds between two attempts | `5` | -| `yhub.initDb.restartPolicy` | Restart policy of the init-db job | `Never` | -| `yhub.service.type` | yhub Service type | `ClusterIP` | -| `yhub.service.port` | yhub Service listening port | `443` | -| `yhub.service.targetPort` | yhub container listening port | `3002` | -| `yhub.service.annotations` | Annotations to add to the yhub Service | `{}` | -| `yhub.probes.liveness.path` | Configure path for yhub HTTP liveness probe | `/collaboration/ping/v1` | -| `yhub.probes.liveness.initialDelaySeconds` | Configure initial delay for yhub liveness probe | `10` | -| `yhub.probes.liveness.timeoutSeconds` | Configure timeout for yhub liveness probe | `2` | -| `yhub.probes.readiness.path` | Configure path for yhub HTTP readiness probe | `/collaboration/ready/v1` | -| `yhub.probes.readiness.initialDelaySeconds` | Configure initial delay for yhub readiness probe | `5` | -| `yhub.probes.readiness.timeoutSeconds` | Configure timeout for yhub readiness probe | `3` | -| `yhub.probes.liveness.targetPort` | Configure port for yhub HTTP liveness probe | | -| `yhub.probes.readiness.targetPort` | Configure port for yhub HTTP readiness probe | | -| `yhub.probes.startup.path` | Configure path for yhub HTTP startup probe | | -| `yhub.probes.startup.targetPort` | Configure port for yhub HTTP startup probe | | -| `yhub.probes.startup.initialDelaySeconds` | Configure initial delay for yhub startup probe | | -| `yhub.probes.startup.timeoutSeconds` | Configure timeout for yhub startup probe | | -| `yhub.resources` | Resource requirements for the yhub container | `{}` | -| `yhub.nodeSelector` | Node selector for the yhub Pod | `{}` | -| `yhub.tolerations` | Tolerations for the yhub Pod | `[]` | -| `yhub.affinity` | Affinity for the yhub Pod | `{}` | -| `yhub.persistence` | Additional volumes to create and mount on the yhub. Used for debugging purposes | `{}` | -| `yhub.persistence.volume-name.size` | Size of the additional volume | | -| `yhub.persistence.volume-name.type` | Type of the additional volume, persistentVolumeClaim or emptyDir | | -| `yhub.persistence.volume-name.mountPath` | Path where the volume should be mounted to | | -| `yhub.extraVolumeMounts` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` | -| `yhub.extraVolumes` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` | -| `yhub.pdb.enabled` | Enable pdb on yhub | `true` | -| `yhub.serviceAccountName` | Optional service account name to use for yhub pods | `nil` | +| Name | Description | Value | +| -------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | ------------------------- | +| `yhub.enabled` | Enable the yhub collaboration server, its service and its init-db job | `true` | +| `yhub.image.repository` | Repository to use to pull the yhub container image | `lasuite/impress-yhub` | +| `yhub.image.tag` | yhub container tag | `latest` | +| `yhub.image.pullPolicy` | yhub container image pull policy | `IfNotPresent` | +| `yhub.command` | Override the yhub container command | `[]` | +| `yhub.args` | Override the yhub container args | `[]` | +| `yhub.replicas` | Amount of yhub replicas | `3` | +| `yhub.worker.enabled` | Deploy the worker apart from the server, each scaling on its own | `false` | +| `yhub.worker.replicas` | Amount of yhub worker replicas | `1` | +| `yhub.worker.resources` | Resource requirements for the yhub worker container, the server ones when empty | `{}` | +| `yhub.worker.podAnnotations` | Annotations to add to the yhub worker Pod, the server ones when empty | `{}` | +| `yhub.worker.dpAnnotations` | Annotations to add to the yhub worker Deployment, the server ones when empty | `{}` | +| `yhub.worker.nodeSelector` | Node selector for the yhub worker Pod, the server one when empty | `{}` | +| `yhub.worker.tolerations` | Tolerations for the yhub worker Pod, the server ones when empty | `[]` | +| `yhub.worker.affinity` | Affinity for the yhub worker Pod, the server one when empty | `{}` | +| `yhub.worker.terminationGracePeriodSeconds` | Grace period given to a worker pod to finish its task, the server one when empty | `nil` | +| `yhub.worker.pdb.enabled` | Enable pdb on the yhub worker | `true` | +| `yhub.shareProcessNamespace` | Enable share process namespace between containers | `false` | +| `yhub.sidecars` | Add sidecars containers to yhub deployment | `[]` | +| `yhub.terminationGracePeriodSeconds` | Grace period given to a yhub pod to drain before it is killed | `60` | +| `yhub.securityContext.allowPrivilegeEscalation` | Whether to allow privilege escalation for the yhub container | `false` | +| `yhub.securityContext.capabilities.drop` | List of capabilities to drop for the yhub container | `["ALL"]` | +| `yhub.securityContext.runAsNonRoot` | Whether to run the yhub container as a non-root user | `true` | +| `yhub.securityContext.runAsUser` | User the yhub container runs as | `1000` | +| `yhub.securityContext.runAsGroup` | Group the yhub container runs as | `1000` | +| `yhub.securityContext.seccompProfile.type` | Seccomp profile type for the yhub container | `RuntimeDefault` | +| `yhub.envVars` | Configure yhub container environment variables | `undefined` | +| `yhub.envVars.REDIS` | Required, redis/valkey url holding the live document state (e.g. redis://valkey:6379/0) | | +| `yhub.envVars.POSTGRES` | Required, url of the yhub database, created by the init-db job (e.g. postgres://user:pass@postgres:5432/yhub) | | +| `yhub.envVars.REDIS_PREFIX` | Namespace of the redis keys, when the instance is shared (default: yhub) | | +| `yhub.envVars.COLLABORATION_BACKEND_BASE_URL` | Base url of the Docs backend, which yhub asks about users and document access rights | | +| `yhub.envVars.COLLABORATION_SERVER_ORIGIN` | Comma separated list of the origins allowed to open a websocket | | +| `yhub.envVars.YHUB_JWT_PRIVATE_KEY_FILE` | Path to the RSA private key (PEM) yhub signs its calls to the backend with, mounted from a secret | | +| `yhub.envVars.YHUB_TASK_CONCURRENCY` | Tasks one worker process claims at once, times the replicas running a worker (default: 5) | | +| `yhub.envVars.YHUB_TASK_DEBOUNCE_MS` | How long an update waits on the redis stream before a worker persists it, in ms (default: 10000) | | +| `yhub.envVars.YHUB_MIN_MESSAGE_LIFETIME_MS` | How long persisted updates stay replayable from redis, in ms (default: 60000) | | +| `yhub.envVars.SOFT_MIGRATION` | Set to "true" to seed rooms from the legacy Django/S3 document store on first access | | +| `yhub.envVars.LEGACY_S3_ENDPOINT_URL` | Required by SOFT_MIGRATION, endpoint of the legacy Django media bucket, without a path (e.g. https://s3.example.com) | | +| `yhub.envVars.LEGACY_S3_ACCESS_KEY_ID` | Required by SOFT_MIGRATION, read access to the legacy bucket (or LEGACY_S3_ACCESS_KEY_ID_FILE) | | +| `yhub.envVars.LEGACY_S3_SECRET_ACCESS_KEY` | Required by SOFT_MIGRATION, secret of the key above (or LEGACY_S3_SECRET_ACCESS_KEY_FILE) | | +| `yhub.envVars.LEGACY_S3_REGION_NAME` | Region of the legacy bucket, when its provider needs one | | +| `yhub.envVars.LEGACY_S3_BUCKET_NAME` | Name of the legacy Django media bucket (default: impress-media-storage) | | +| `yhub.envVars.BY_VALUE` | Example environment variable by setting value directly | | +| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name` | Name of a ConfigMap when configuring env vars from a ConfigMap | | +| `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.key` | Key within a ConfigMap when configuring env vars from a ConfigMap | | +| `yhub.envVars.FROM_SECRET.secretKeyRef.name` | Name of a Secret when configuring env vars from a Secret | | +| `yhub.envVars.FROM_SECRET.secretKeyRef.key` | Key within a Secret when configuring env vars from a Secret | | +| `yhub.podAnnotations` | Annotations to add to the yhub Pod | `{}` | +| `yhub.dpAnnotations` | Annotations to add to the yhub Deployment | `{}` | +| `yhub.initDbJobAnnotations` | Annotations for the yhub init-db job | `{}` | +| `yhub.jobs.ttlSecondsAfterFinished` | Period to wait before removing the init-db job | `30` | +| `yhub.jobs.backoffLimit` | Numbers of init-db job retries | `2` | +| `yhub.initDb.enabled` | Run the job creating and upgrading the yhub schema | `true` | +| `yhub.initDb.command` | Override the command creating and upgrading the yhub schema | `[]` | +| `yhub.initDb.retries` | How many times the schema script is retried while the postgres server does not answer | `60` | +| `yhub.initDb.retryDelaySeconds` | Seconds between two attempts | `5` | +| `yhub.initDb.restartPolicy` | Restart policy of the init-db job | `Never` | +| `yhub.service.type` | yhub Service type | `ClusterIP` | +| `yhub.service.port` | yhub Service listening port | `443` | +| `yhub.service.targetPort` | yhub container listening port | `3002` | +| `yhub.service.annotations` | Annotations to add to the yhub Service | `{}` | +| `yhub.probes.liveness.path` | Configure path for yhub HTTP liveness probe | `/collaboration/ping/v1` | +| `yhub.probes.liveness.initialDelaySeconds` | Configure initial delay for yhub liveness probe | `10` | +| `yhub.probes.liveness.timeoutSeconds` | Configure timeout for yhub liveness probe | `2` | +| `yhub.probes.readiness.path` | Configure path for yhub HTTP readiness probe | `/collaboration/ready/v1` | +| `yhub.probes.readiness.initialDelaySeconds` | Configure initial delay for yhub readiness probe | `5` | +| `yhub.probes.readiness.timeoutSeconds` | Configure timeout for yhub readiness probe | `3` | +| `yhub.probes.liveness.targetPort` | Configure port for yhub HTTP liveness probe | | +| `yhub.probes.readiness.targetPort` | Configure port for yhub HTTP readiness probe | | +| `yhub.probes.startup.path` | Configure path for yhub HTTP startup probe | | +| `yhub.probes.startup.targetPort` | Configure port for yhub HTTP startup probe | | +| `yhub.probes.startup.initialDelaySeconds` | Configure initial delay for yhub startup probe | | +| `yhub.probes.startup.timeoutSeconds` | Configure timeout for yhub startup probe | | +| `yhub.resources` | Resource requirements for the yhub container | `{}` | +| `yhub.nodeSelector` | Node selector for the yhub Pod | `{}` | +| `yhub.tolerations` | Tolerations for the yhub Pod | `[]` | +| `yhub.affinity` | Affinity for the yhub Pod | `{}` | +| `yhub.persistence` | Additional volumes to create and mount on the yhub. Used for debugging purposes | `{}` | +| `yhub.persistence.volume-name.size` | Size of the additional volume | | +| `yhub.persistence.volume-name.type` | Type of the additional volume, persistentVolumeClaim or emptyDir | | +| `yhub.persistence.volume-name.mountPath` | Path where the volume should be mounted to | | +| `yhub.extraVolumeMounts` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` | +| `yhub.extraVolumes` | Additional volumes to mount on the yhub. Mounted on the init-db job too | `[]` | +| `yhub.pdb.enabled` | Enable pdb on yhub | `true` | +| `yhub.serviceAccountName` | Optional service account name to use for yhub pods | `nil` | ### docSpec diff --git a/src/helm/impress/values.yaml b/src/helm/impress/values.yaml index 88d646ca52..906f849c04 100644 --- a/src/helm/impress/values.yaml +++ b/src/helm/impress/values.yaml @@ -909,6 +909,13 @@ jwtKeys: ## PostgreSQL database, so it needs both — set `yhub.envVars.REDIS` and ## `yhub.envVars.POSTGRES`, there is nothing sensible to default them to. ## Disabling it sends the /collaboration/ ingresses back to the y-provider. +## +## Turning on `SOFT_MIGRATION` adds a bucket to that list, the legacy Django +## media one it reads old documents out of. It is configured under +## `LEGACY_S3_*` rather than the backend's `AWS_S3_*`: the two are read by +## different processes, may be different buckets on different providers, and +## the collaboration server is meant to gain a bucket of its own — the S3 +## persistence plugin, once it is enabled — without either being ambiguous. yhub: ## @param yhub.enabled Enable the yhub collaboration server, its service and its init-db job enabled: true @@ -1017,6 +1024,11 @@ yhub: ## @extra yhub.envVars.YHUB_TASK_DEBOUNCE_MS How long an update waits on the redis stream before a worker persists it, in ms (default: 10000) ## @extra yhub.envVars.YHUB_MIN_MESSAGE_LIFETIME_MS How long persisted updates stay replayable from redis, in ms (default: 60000) ## @extra yhub.envVars.SOFT_MIGRATION Set to "true" to seed rooms from the legacy Django/S3 document store on first access + ## @extra yhub.envVars.LEGACY_S3_ENDPOINT_URL Required by SOFT_MIGRATION, endpoint of the legacy Django media bucket, without a path (e.g. https://s3.example.com) + ## @extra yhub.envVars.LEGACY_S3_ACCESS_KEY_ID Required by SOFT_MIGRATION, read access to the legacy bucket (or LEGACY_S3_ACCESS_KEY_ID_FILE) + ## @extra yhub.envVars.LEGACY_S3_SECRET_ACCESS_KEY Required by SOFT_MIGRATION, secret of the key above (or LEGACY_S3_SECRET_ACCESS_KEY_FILE) + ## @extra yhub.envVars.LEGACY_S3_REGION_NAME Region of the legacy bucket, when its provider needs one + ## @extra yhub.envVars.LEGACY_S3_BUCKET_NAME Name of the legacy Django media bucket (default: impress-media-storage) ## @extra yhub.envVars.BY_VALUE Example environment variable by setting value directly ## @extra yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name Name of a ConfigMap when configuring env vars from a ConfigMap ## @extra yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.key Key within a ConfigMap when configuring env vars from a ConfigMap diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index 0a7d97e3fb..e427cf9181 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -367,14 +367,22 @@ Guarantees and failure behavior: document then opens as an *empty* room. Keep the flag on until a backfill has migrated the full corpus. -Configuration: `AWS_S3_ENDPOINT_URL`, `AWS_S3_ACCESS_KEY_ID`, -`AWS_S3_SECRET_ACCESS_KEY` (both with `*_FILE` indirection), optional -`AWS_S3_REGION_NAME`, and `AWS_STORAGE_BUCKET_NAME` (defaults to Django's dev +Configuration: `LEGACY_S3_ENDPOINT_URL`, `LEGACY_S3_ACCESS_KEY_ID`, +`LEGACY_S3_SECRET_ACCESS_KEY` (both with `*_FILE` indirection), optional +`LEGACY_S3_REGION_NAME`, and `LEGACY_S3_BUCKET_NAME` (defaults to Django's dev default `impress-media-storage`; production uses a different bucket name and must set it explicitly). The server refuses to boot when the flag is set without endpoint and credentials. In development the values arrive via `env.d/development/common`. +The prefix is deliberate: these name **the bucket this server migrates out +of**, which is the backend's media bucket and not the one yhub will persist +into once the S3 persistence plugin is enabled. That one gets a set of its own, +and the two are free to be different buckets, on different providers, with +different credentials. Nothing here reads the backend's `AWS_S3_*` settings — +a pod that carries them, for the backend's own reasons, must not quietly +migrate documents out of whatever they point at. + Operational notes: - Use **read-only, bucket-scoped S3 credentials** in production — never the @@ -383,7 +391,7 @@ Operational notes: to `s3:GetObject`: without it, S3 reports a missing object as `403 AccessDenied` instead of `404 NoSuchKey`, and every brand-new document would fail closed instead of starting empty. -- `AWS_S3_ENDPOINT_URL` must not contain a path (the minio client cannot +- `LEGACY_S3_ENDPOINT_URL` must not contain a path (the minio client cannot address a base path); the server refuses to boot otherwise. - After manually wiping a room's yhub state (postgres row + stream key), **restart yhub** so the in-process verdict cache cannot serve a stale diff --git a/src/yhub-server/migration.js b/src/yhub-server/migration.js index 70401a5c0b..0235b9f757 100644 --- a/src/yhub-server/migration.js +++ b/src/yhub-server/migration.js @@ -29,13 +29,19 @@ import { Client as S3Client } from 'minio'; import { secret } from './env.js'; export const SOFT_MIGRATION = process.env.SOFT_MIGRATION === 'true'; -const AWS_S3_ENDPOINT_URL = process.env.AWS_S3_ENDPOINT_URL; -const AWS_S3_ACCESS_KEY_ID = secret('AWS_S3_ACCESS_KEY_ID'); -const AWS_S3_SECRET_ACCESS_KEY = secret('AWS_S3_SECRET_ACCESS_KEY'); -const AWS_S3_REGION_NAME = process.env.AWS_S3_REGION_NAME; +// The legacy Django media bucket, the one documents are migrated *out of*. It +// carries a prefix of its own because it is not the only bucket in play: the +// S3 persistence plugin, once it is enabled, persists *into* a bucket that may +// sit on another provider with credentials of its own, and the backend's +// `AWS_S3_*` settings — which a pod may perfectly well carry — name a third. +// Each set is read by exactly the process it belongs to. +const LEGACY_S3_ENDPOINT_URL = process.env.LEGACY_S3_ENDPOINT_URL; +const LEGACY_S3_ACCESS_KEY_ID = secret('LEGACY_S3_ACCESS_KEY_ID'); +const LEGACY_S3_SECRET_ACCESS_KEY = secret('LEGACY_S3_SECRET_ACCESS_KEY'); +const LEGACY_S3_REGION_NAME = process.env.LEGACY_S3_REGION_NAME; // Django's default bucket name (impress settings.py) — prod overrides it -const AWS_STORAGE_BUCKET_NAME = - process.env.AWS_STORAGE_BUCKET_NAME || 'impress-media-storage'; +const LEGACY_S3_BUCKET_NAME = + process.env.LEGACY_S3_BUCKET_NAME || 'impress-media-storage'; // the same limit create-ydoc applies to a posted update in server.js: one // legacy snapshot handed to a compute worker, or written to the stream as a // single message @@ -60,21 +66,23 @@ const EMPTY_YDOC = Y.encodeStateAsUpdate(new Y.Doc()); if ( SOFT_MIGRATION && - (!AWS_S3_ENDPOINT_URL || !AWS_S3_ACCESS_KEY_ID || !AWS_S3_SECRET_ACCESS_KEY) + (!LEGACY_S3_ENDPOINT_URL || + !LEGACY_S3_ACCESS_KEY_ID || + !LEGACY_S3_SECRET_ACCESS_KEY) ) { // fail at boot instead of as an opaque 401 storm on first connect throw new Error( - 'SOFT_MIGRATION=true requires AWS_S3_ENDPOINT_URL, AWS_S3_ACCESS_KEY_ID and AWS_S3_SECRET_ACCESS_KEY', + 'SOFT_MIGRATION=true requires LEGACY_S3_ENDPOINT_URL, LEGACY_S3_ACCESS_KEY_ID and LEGACY_S3_SECRET_ACCESS_KEY', ); } const s3 = SOFT_MIGRATION ? (() => { - const url = new URL(AWS_S3_ENDPOINT_URL); + const url = new URL(LEGACY_S3_ENDPOINT_URL); if (url.pathname !== '/' && url.pathname !== '') { // boto3 accepts path-prefixed endpoints but the minio client cannot // address a base path — dropping it silently would probe the wrong // keys and "migrate" every doc as empty - throw new Error('AWS_S3_ENDPOINT_URL must not contain a path'); + throw new Error('LEGACY_S3_ENDPOINT_URL must not contain a path'); } return new S3Client({ endPoint: url.hostname, @@ -85,9 +93,9 @@ const s3 = SOFT_MIGRATION ? 443 : 80, useSSL: url.protocol === 'https:', - accessKey: AWS_S3_ACCESS_KEY_ID, - secretKey: AWS_S3_SECRET_ACCESS_KEY, - ...(AWS_S3_REGION_NAME ? { region: AWS_S3_REGION_NAME } : {}), + accessKey: LEGACY_S3_ACCESS_KEY_ID, + secretKey: LEGACY_S3_SECRET_ACCESS_KEY, + ...(LEGACY_S3_REGION_NAME ? { region: LEGACY_S3_REGION_NAME } : {}), }); })() : null; @@ -133,7 +141,7 @@ const fetchLegacyDoc = async (docid, versionId = null) => { let objPromise; try { objPromise = s3.getObject( - AWS_STORAGE_BUCKET_NAME, + LEGACY_S3_BUCKET_NAME, `${docid}/file`, // minio stringifies the whole opts object into the query — pass // undefined, not {}, so the unversioned read stays byte-identical @@ -202,7 +210,7 @@ const listLegacyVersions = async (docid) => { const key = `${docid}/file`; const found = await new Promise((resolve, reject) => { const versions = []; - const stream = s3.listObjects(AWS_STORAGE_BUCKET_NAME, key, true, { + const stream = s3.listObjects(LEGACY_S3_BUCKET_NAME, key, true, { IncludeVersion: true, }); const timer = setTimeout(() => { @@ -439,7 +447,7 @@ export const maybeMigrate = async (yhub, room) => { err, docid: room.docid, permanent, - bucket: AWS_STORAGE_BUCKET_NAME, + bucket: LEGACY_S3_BUCKET_NAME, key: `${room.docid}/file`, }, permanent From 3d27b84b0ececa011eec441e43399b1489eac345 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Fri, 14 Aug 2026 10:57:04 +0200 Subject: [PATCH 57/59] =?UTF-8?q?=E2=99=BB=EF=B8=8F(yhub)=20create=20dedic?= =?UTF-8?q?ated=20file=20for=20environment=20variables=20in=20development?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For now the environment variable for yhub were added to the common file. This number of environment is growing and is specific to yhub so we decided to create a dedicated file for yhub --- .github/workflows/impress.yml | 7 ++++++- CHANGELOG.md | 3 ++- Makefile | 1 + compose.yml | 15 ++++----------- env.d/development/common | 9 --------- env.d/development/yhub | 34 ++++++++++++++++++++++++++++++++++ src/yhub-server/README.md | 5 +++-- 7 files changed, 50 insertions(+), 24 deletions(-) create mode 100644 env.d/development/yhub diff --git a/.github/workflows/impress.yml b/.github/workflows/impress.yml index f9c13f2a08..593c30c8b7 100644 --- a/.github/workflows/impress.yml +++ b/.github/workflows/impress.yml @@ -263,8 +263,13 @@ jobs: REDIS_PREFIX: yhub COLLABORATION_BACKEND_BASE_URL: http://localhost:8000 COLLABORATION_SERVER_ORIGIN: http://localhost:3000 - AWS_STORAGE_BUCKET_NAME: impress-media-storage + # the legacy Django bucket it migrates documents out of, named apart + # from the AWS_S3_* the job sets for django itself SOFT_MIGRATION: "true" + LEGACY_S3_ENDPOINT_URL: http://localhost:9000 + LEGACY_S3_ACCESS_KEY_ID: impress + LEGACY_S3_SECRET_ACCESS_KEY: password + LEGACY_S3_BUCKET_NAME: impress-media-storage run: | nohup node server.js > /tmp/yhub.log 2>&1 & dockerize -wait tcp://localhost:3002 -timeout 30s diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c20868210..3757e6f96a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,7 +85,8 @@ and this project adheres to top of content that exists. Backend reads carrying the admin JWT are seeded too, so a server-side read of an unmigrated document never answers with an empty one. Enabled in the dev - stack via compose.yml. The bucket it reads is configured under `LEGACY_S3_*` + stack through `env.d/development/yhub`, the collaboration server's own + environment file. The bucket it reads is configured under `LEGACY_S3_*` (`_ENDPOINT_URL`, `_ACCESS_KEY_ID`, `_SECRET_ACCESS_KEY`, `_REGION_NAME`, `_BUCKET_NAME`), a set of its own and not the backend's `AWS_S3_*`: this is the bucket the collaboration server migrates *out of*, while the one it will diff --git a/Makefile b/Makefile index 5753f44753..603fe4d912 100644 --- a/Makefile +++ b/Makefile @@ -88,6 +88,7 @@ create-env-local-files: @touch env.d/development/postgresql.local @touch env.d/development/kc_auth.local @touch env.d/development/kc_postgresql.local + @touch env.d/development/yhub.local .PHONY: create-env-local-files generate-secret-keys: diff --git a/compose.yml b/compose.yml index a53036eb70..509587c080 100644 --- a/compose.yml +++ b/compose.yml @@ -241,18 +241,11 @@ services: image: impress:yhub-development environment: HOME: /tmp # same reason as node-based services above (unmapped uid) - PORT: 3002 - REDIS: redis://yhub-valkey:6379 - POSTGRES: postgres://yhub:yhub@yhub-postgres:5432/yhub - REDIS_PREFIX: yhub - # seed rooms from the legacy Django/S3 document store on first access — - # S3 endpoint/credentials come from env.d/development/common - SOFT_MIGRATION: "true" - # signs the calls made to the backend, which holds the public half - YHUB_JWT_PRIVATE_KEY_FILE: /data/jwt/yhub-private.pem + # its own file rather than the backend's: this server reads none of the + # django settings `common` carries, and everything it does read is in there env_file: - - env.d/development/common - - env.d/development/common.local + - env.d/development/yhub + - env.d/development/yhub.local volumes: - ./data/jwt:/data/jwt:ro # editing a source file restarts the server (nodemon), no rebuild diff --git a/env.d/development/common b/env.d/development/common index 954128b041..4b09afab7b 100644 --- a/env.d/development/common +++ b/env.d/development/common @@ -37,15 +37,6 @@ AWS_S3_ACCESS_KEY_ID=impress AWS_S3_SECRET_ACCESS_KEY=password MEDIA_BASE_URL=http://localhost:8083 -# The same bucket, read by yhub's soft migration under names of its own: the -# bucket the collaboration server migrates *out of* is not the one it will -# persist *into* once the S3 persistence plugin is turned on, so it does not -# read the backend's AWS_S3_* settings. Locally they hold the same minio, and -# an override of the three above wants the same override here. -LEGACY_S3_ENDPOINT_URL=http://minio:9000 -LEGACY_S3_ACCESS_KEY_ID=impress -LEGACY_S3_SECRET_ACCESS_KEY=password - # OIDC OIDC_OP_JWKS_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/certs OIDC_OP_AUTHORIZATION_ENDPOINT=http://localhost:8083/realms/impress/protocol/openid-connect/auth diff --git a/env.d/development/yhub b/env.d/development/yhub new file mode 100644 index 0000000000..8cf36c467c --- /dev/null +++ b/env.d/development/yhub @@ -0,0 +1,34 @@ +# Collaboration server (yhub) +# +# Everything the collaboration server reads, and nothing else: it shares the +# backend's stores and origins by value, not by loading the backend's own +# environment. Override any of it in yhub.local, which is not committed. + +# Stores. Its own valkey and its own postgres database — the backend's live +# next to them and are never touched from here. +PORT=3002 +REDIS=redis://yhub-valkey:6379 +POSTGRES=postgres://yhub:yhub@yhub-postgres:5432/yhub +REDIS_PREFIX=yhub + +# Backend. It answers who a user is and what they may do with a document, and +# publishes the JWKS the admin tokens it signs are verified against. The origin +# list is what a browser may open a websocket from — the frontend dev server. +COLLABORATION_BACKEND_BASE_URL=http://app-dev:8000 +COLLABORATION_SERVER_ORIGIN=http://localhost:3000 +# sent as X-Y-Provider-Key on the calls made to the backend; the same value as +# in `common`, which is where the backend reads the one it compares it to +Y_PROVIDER_API_KEY=yprovider-api-key + +# Signs the calls made to the backend, which holds the public half. Generated +# by `make generate-secret-keys`, never committed. +YHUB_JWT_PRIVATE_KEY_FILE=/data/jwt/yhub-private.pem + +# Soft migration: seed a room from the legacy Django/S3 document store the +# first time it is opened. The bucket read here is the backend's media one — +# in this stack the same minio, under the credentials of this server rather +# than the backend's own AWS_S3_* settings. +SOFT_MIGRATION=true +LEGACY_S3_ENDPOINT_URL=http://minio:9000 +LEGACY_S3_ACCESS_KEY_ID=impress +LEGACY_S3_SECRET_ACCESS_KEY=password diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index e427cf9181..cad57347df 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -372,8 +372,9 @@ Configuration: `LEGACY_S3_ENDPOINT_URL`, `LEGACY_S3_ACCESS_KEY_ID`, `LEGACY_S3_REGION_NAME`, and `LEGACY_S3_BUCKET_NAME` (defaults to Django's dev default `impress-media-storage`; production uses a different bucket name and must set it explicitly). The server refuses to boot when the flag is set -without endpoint and credentials. In development the values arrive via -`env.d/development/common`. +without endpoint and credentials. In development they come, like everything +else this server reads, from `env.d/development/yhub` (and `yhub.local`, which +is not committed — `make create-env-local-files` creates it). The prefix is deliberate: these name **the bucket this server migrates out of**, which is the backend's media bucket and not the one yhub will persist From b949f7a7ecd616c73bd3a0c8b0c8257c660009d2 Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Fri, 14 Aug 2026 17:04:42 +0200 Subject: [PATCH 58/59] =?UTF-8?q?=E2=99=BB=EF=B8=8F(yhub)=20replace=20mini?= =?UTF-8?q?o=20client=20by=20S3=20sdk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We have some signature errors when using the minio client to list all the versions of an existing document. To avoid this error we have decided to use the S3 sdk and allow to configure the signature versino the user wants. Also, the check on the document size has been removed, there is no limitation on the document size. --- CHANGELOG.md | 14 +- src/helm/impress/README.md | 1 + src/helm/impress/values.yaml | 1 + src/yhub-server/README.md | 35 ++- src/yhub-server/migration.js | 267 ++++++++++---------- src/yhub-server/package-lock.json | 405 +++++++++++++++++++++++++++++- src/yhub-server/package.json | 4 +- src/yhub-server/server.js | 11 +- 8 files changed, 586 insertions(+), 152 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3757e6f96a..3a3cb9ca12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,7 +77,7 @@ and this project adheres to collide with the real per-version times the migrate endpoint writes) before the connection is admitted. A missing S3 object means a brand-new document and yields an empty room. Seeding never decides access: a legacy object that - cannot be migrated (undecodable or oversized) opens as a new document, logged + cannot be migrated (it does not decode) opens as a new document, logged per access, since no retry could fix it and refusing would make the document permanently unopenable. Every other failure — an unreachable store, but also any refusal from S3 such as `AccessDenied` on a rotated key or a wrong bucket @@ -88,10 +88,14 @@ and this project adheres to stack through `env.d/development/yhub`, the collaboration server's own environment file. The bucket it reads is configured under `LEGACY_S3_*` (`_ENDPOINT_URL`, `_ACCESS_KEY_ID`, `_SECRET_ACCESS_KEY`, `_REGION_NAME`, - `_BUCKET_NAME`), a set of its own and not the backend's `AWS_S3_*`: this is - the bucket the collaboration server migrates *out of*, while the one it will - persist *into* when the yhub S3 persistence plugin is enabled is a separate - bucket that may well sit on another provider with credentials of its own + `_BUCKET_NAME`, `_SIGNATURE_VERSION`), a set of its own and not the backend's + `AWS_S3_*`: this is the bucket the collaboration server migrates *out of*, + while the one it will persist *into* when the yhub S3 persistence plugin is + enabled is a separate bucket that may well sit on another provider with + credentials of its own. It is read with the AWS SDK for JavaScript v3, whose + signature version is configurable (`s3v4` by default, as in Django) because a + provider expecting another one answers 403, which reads exactly like wrong + credentials - ✨(collaboration) add a migrate endpoint on yhub: `POST /collaboration/migrate/v1/docs/{id}` replays a document's **full** legacy version history from the versioned S3 media bucket into a diff --git a/src/helm/impress/README.md b/src/helm/impress/README.md index d20b054790..219eadd90e 100644 --- a/src/helm/impress/README.md +++ b/src/helm/impress/README.md @@ -383,6 +383,7 @@ | `yhub.envVars.LEGACY_S3_SECRET_ACCESS_KEY` | Required by SOFT_MIGRATION, secret of the key above (or LEGACY_S3_SECRET_ACCESS_KEY_FILE) | | | `yhub.envVars.LEGACY_S3_REGION_NAME` | Region of the legacy bucket, when its provider needs one | | | `yhub.envVars.LEGACY_S3_BUCKET_NAME` | Name of the legacy Django media bucket (default: impress-media-storage) | | +| `yhub.envVars.LEGACY_S3_SIGNATURE_VERSION` | How the calls to the legacy bucket are signed, s3v4 or v4 (default: s3v4) | | | `yhub.envVars.BY_VALUE` | Example environment variable by setting value directly | | | `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name` | Name of a ConfigMap when configuring env vars from a ConfigMap | | | `yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.key` | Key within a ConfigMap when configuring env vars from a ConfigMap | | diff --git a/src/helm/impress/values.yaml b/src/helm/impress/values.yaml index 906f849c04..a4a8b08299 100644 --- a/src/helm/impress/values.yaml +++ b/src/helm/impress/values.yaml @@ -1029,6 +1029,7 @@ yhub: ## @extra yhub.envVars.LEGACY_S3_SECRET_ACCESS_KEY Required by SOFT_MIGRATION, secret of the key above (or LEGACY_S3_SECRET_ACCESS_KEY_FILE) ## @extra yhub.envVars.LEGACY_S3_REGION_NAME Region of the legacy bucket, when its provider needs one ## @extra yhub.envVars.LEGACY_S3_BUCKET_NAME Name of the legacy Django media bucket (default: impress-media-storage) + ## @extra yhub.envVars.LEGACY_S3_SIGNATURE_VERSION How the calls to the legacy bucket are signed, s3v4 or v4 (default: s3v4) ## @extra yhub.envVars.BY_VALUE Example environment variable by setting value directly ## @extra yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.name Name of a ConfigMap when configuring env vars from a ConfigMap ## @extra yhub.envVars.FROM_CONFIGMAP.configMapKeyRef.key Key within a ConfigMap when configuring env vars from a ConfigMap diff --git a/src/yhub-server/README.md b/src/yhub-server/README.md index cad57347df..d3500a79c0 100644 --- a/src/yhub-server/README.md +++ b/src/yhub-server/README.md @@ -296,8 +296,8 @@ documents into yhub lazily, on first access: then the valkey stream (uncompacted `ydoc:update:v1` messages), then the `SELECT` again to close the compaction race. Verdicts are cached in-process (existing docs 10 min, empty docs 60 s, failures 5 min). -2. If the room is unknown, the legacy object is fetched from S3 (10 s - timeout, 10 MiB decoded cap — the same limit as `create-ydoc`), decoded, +2. If the room is unknown, the legacy object is fetched from S3 whole, + whatever its size (10 s timeout for the request and its body), decoded, diffed through yhub's compute pool and appended to the room's stream — attributed to the `system` identity with a `migration=s3` custom attribution. This completes before the websocket upgrade resolves, so the @@ -350,7 +350,7 @@ Guarantees and failure behavior: guessing wrong the other way costs the document. A cached failure verdict prevents retry storms from hammering S3 — permanent - failures (corrupt/oversized objects) for 5 minutes, transient ones (network + failures (objects that do not decode) for 5 minutes, transient ones (network errors, timeouts) for 15 seconds, and per-replica seed backpressure (more than 20 concurrent seeds) is not cached at all, so the client's next retry goes through. @@ -369,12 +369,29 @@ Guarantees and failure behavior: Configuration: `LEGACY_S3_ENDPOINT_URL`, `LEGACY_S3_ACCESS_KEY_ID`, `LEGACY_S3_SECRET_ACCESS_KEY` (both with `*_FILE` indirection), optional -`LEGACY_S3_REGION_NAME`, and `LEGACY_S3_BUCKET_NAME` (defaults to Django's dev -default `impress-media-storage`; production uses a different bucket name and -must set it explicitly). The server refuses to boot when the flag is set -without endpoint and credentials. In development they come, like everything -else this server reads, from `env.d/development/yhub` (and `yhub.local`, which -is not committed — `make create-env-local-files` creates it). +`LEGACY_S3_REGION_NAME` (`us-east-1` when unset, which every S3-compatible +provider answers to), `LEGACY_S3_SIGNATURE_VERSION` (see below), and +`LEGACY_S3_BUCKET_NAME` (defaults to Django's dev default +`impress-media-storage`; production uses a different bucket name and must set +it explicitly). The server refuses to boot when the flag is set without +endpoint and credentials. In development they come, like everything else this +server reads, from `env.d/development/yhub` (and `yhub.local`, which is not +committed — `make create-env-local-files` creates it). + +The bucket is read with the **AWS SDK for JavaScript v3** +(`@aws-sdk/client-s3`), the same library family boto3 is to Django, so the +provider quirks the backend already deals with apply here too. Two settings +follow from that: + +- `LEGACY_S3_SIGNATURE_VERSION` — the counterpart of Django's + `AWS_S3_SIGNATURE_VERSION`, since a provider expecting the other signature + answers `403`, which reads exactly like wrong credentials. It defaults to + `s3v4` and accepts `s3v4` or `v4`. **SigV2 (boto3's `s3`) is not available**: + the AWS SDK v3 dropped it, so asking for it fails at boot instead of signing + the other way and being bounced, +- addressing style is chosen from the endpoint: path style (`{host}/{bucket}`) + everywhere but `amazonaws.com`, which prefers virtual-host style. Self-hosted + providers have no per-bucket DNS record, so path style is what they need. The prefix is deliberate: these name **the bucket this server migrates out of**, which is the backend's media bucket and not the one yhub will persist diff --git a/src/yhub-server/migration.js b/src/yhub-server/migration.js index 0235b9f757..ea248e6381 100644 --- a/src/yhub-server/migration.js +++ b/src/yhub-server/migration.js @@ -22,9 +22,13 @@ import { randomUUID } from 'node:crypto'; +import { + GetObjectCommand, + ListObjectVersionsCommand, + S3Client, +} from '@aws-sdk/client-s3'; import { logger } from '@y/hub'; import * as Y from '@y/y'; -import { Client as S3Client } from 'minio'; import { secret } from './env.js'; @@ -42,13 +46,18 @@ const LEGACY_S3_REGION_NAME = process.env.LEGACY_S3_REGION_NAME; // Django's default bucket name (impress settings.py) — prod overrides it const LEGACY_S3_BUCKET_NAME = process.env.LEGACY_S3_BUCKET_NAME || 'impress-media-storage'; -// the same limit create-ydoc applies to a posted update in server.js: one -// legacy snapshot handed to a compute worker, or written to the stream as a -// single message -const MAX_LEGACY_BYTES = 10 * 1024 * 1024; -// base64 inflates 3 bytes to 4 — cap the streamed read at the encoded size of -// MAX_LEGACY_BYTES plus padding slack -const MAX_LEGACY_B64_BYTES = Math.ceil(MAX_LEGACY_BYTES / 3) * 4 + 1024; +// How the requests are signed, the counterpart of Django's +// AWS_S3_SIGNATURE_VERSION: a provider that expects the other one answers 403, +// which reads exactly like wrong credentials, so it is worth being explicit +// about. Only SigV4 is offered — see SIGNATURE_VERSIONS below. +const LEGACY_S3_SIGNATURE_VERSION = + process.env.LEGACY_S3_SIGNATURE_VERSION || 's3v4'; +// What that variable accepts, mapped to what it means for the client. The AWS +// SDK v3 signs with SigV4 and dropped SigV2 altogether, so the spellings of +// SigV4 are the whole set: a value asking for SigV2 (`s3`, boto3's other +// choice) is refused at boot rather than silently signed the other way and +// bounced by the provider as a credentials error. +const SIGNATURE_VERSIONS = { s3v4: 'sigv4', v4: 'sigv4' }; const S3_FETCH_TIMEOUT_MS = 10000; const MIGRATE_LOCK_TTL_MS = 30000; const MAX_CONCURRENT_SEEDS = 20; @@ -79,23 +88,36 @@ const s3 = SOFT_MIGRATION ? (() => { const url = new URL(LEGACY_S3_ENDPOINT_URL); if (url.pathname !== '/' && url.pathname !== '') { - // boto3 accepts path-prefixed endpoints but the minio client cannot - // address a base path — dropping it silently would probe the wrong - // keys and "migrate" every doc as empty + // boto3 accepts path-prefixed endpoints but an S3 endpoint cannot + // carry a base path — dropping it silently would probe the wrong keys + // and "migrate" every doc as empty throw new Error('LEGACY_S3_ENDPOINT_URL must not contain a path'); } + const signature = + SIGNATURE_VERSIONS[LEGACY_S3_SIGNATURE_VERSION.toLowerCase()]; + if (signature == null) { + throw new Error( + `LEGACY_S3_SIGNATURE_VERSION must be one of ${Object.keys( + SIGNATURE_VERSIONS, + ).join(', ')} (got "${LEGACY_S3_SIGNATURE_VERSION}")`, + ); + } return new S3Client({ - endPoint: url.hostname, - port: - url.port !== '' - ? Number(url.port) - : url.protocol === 'https:' - ? 443 - : 80, - useSSL: url.protocol === 'https:', - accessKey: LEGACY_S3_ACCESS_KEY_ID, - secretKey: LEGACY_S3_SECRET_ACCESS_KEY, - ...(LEGACY_S3_REGION_NAME ? { region: LEGACY_S3_REGION_NAME } : {}), + endpoint: url.origin, + // required by the sdk even where the provider ignores it; us-east-1 is + // what every S3-compatible implementation answers to by default + region: LEGACY_S3_REGION_NAME || 'us-east-1', + credentials: { + accessKeyId: LEGACY_S3_ACCESS_KEY_ID, + secretAccessKey: LEGACY_S3_SECRET_ACCESS_KEY, + }, + // `sigv4` today, and the client is built from the setting rather than + // from the default so that the value is what decides + authSchemePreference: [`aws.auth#${signature}`], + // Virtual-host style addresses a bucket as `{bucket}.{host}`, which + // needs a DNS record self-hosted providers do not have. AWS is the one + // endpoint that prefers it — and the one deprecating path style. + forcePathStyle: !/(^|\.)amazonaws\.com$/i.test(url.hostname), }); })() : null; @@ -114,93 +136,60 @@ const migrateLockKey = (yhub, room) => // content twice (see fullMigrate). const migratedSetKey = (yhub) => `${yhub.stream.prefix}:migrated:v1`; +// An aborted request surfaces as whatever the sdk or the body stream raises +// when the socket goes away ("aborted", TimeoutError, …). Say what actually +// happened instead, and leave it unmarked so it stays retryable — a slow S3 +// may well recover. +const asTimeout = (err, signal, what, ms) => + signal.aborted ? new Error(`${what} timed out after ${ms}ms`) : err; + // Legacy Django document store: object `{docid}/file`, body = UTF-8 text that // is the base64 encoding of a raw Yjs update. With `versionId`, reads that // specific object version instead of the current one. Returns null when the // object (or version) does not exist — a document that never had content -// saved, e.g. brand new. Throws on any other failure (network, auth, timeout, -// oversize); corrupt base64 decodes leniently to garbage that the callers -// reject. +// saved, e.g. brand new. Throws on any other failure (network, auth, timeout); +// corrupt base64 decodes leniently to garbage that the callers reject. const fetchLegacyDoc = async (docid, versionId = null) => { - let stream = null; - let cancelTimeout = () => {}; - // minio 8 takes no AbortSignal — race a timer that also destroys the body - // stream once reading, so a stalled transfer cannot hold the ws upgrade - const timeout = new Promise((_, reject) => { - const timer = setTimeout(() => { - // unmarked, so it counts as retryable: a slow S3 may recover - const err = new Error( - `s3 fetch timed out after ${S3_FETCH_TIMEOUT_MS}ms`, - ); - stream?.destroy(err); - reject(err); - }, S3_FETCH_TIMEOUT_MS); - cancelTimeout = () => clearTimeout(timer); - }); + // One budget for the whole read, headers and body alike: the sdk aborts the + // request when it fires and the body stream dies with it, so a stalled + // transfer cannot hold the ws upgrade open. + const abortSignal = AbortSignal.timeout(S3_FETCH_TIMEOUT_MS); + let body; try { - let objPromise; - try { - objPromise = s3.getObject( - LEGACY_S3_BUCKET_NAME, - `${docid}/file`, - // minio stringifies the whole opts object into the query — pass - // undefined, not {}, so the unversioned read stays byte-identical - versionId != null ? { versionId } : undefined, - ); - stream = await Promise.race([objPromise, timeout]); - } catch (err) { - // NoSuchVersion: the version vanished between listing and reading - if (err?.code === 'NoSuchKey' || err?.code === 'NoSuchVersion') { - return null; - } - // if the timeout won the race, getObject may still resolve later — - // destroy the late-arriving response stream, otherwise its never-read - // socket leaks (minio 8 sets no request timeout and cannot abort) - objPromise?.then( - (s) => s.destroy(err), - () => {}, - ); - throw err; - } - const body = await Promise.race([ - new Promise((resolve, reject) => { - const chunks = []; - let received = 0; - stream.on('data', (chunk) => { - received += chunk.byteLength; - if (received > MAX_LEGACY_B64_BYTES) { - const err = new Error( - `legacy object exceeds the ${MAX_LEGACY_B64_BYTES}B cap`, - ); - err.permanent = true; // the object will be this big next time too - stream.destroy(err); - return; - } - chunks.push(chunk); - }); - stream.on('error', reject); - stream.on('end', () => resolve(Buffer.concat(chunks))); + ({ Body: body } = await s3.send( + new GetObjectCommand({ + Bucket: LEGACY_S3_BUCKET_NAME, + Key: `${docid}/file`, + ...(versionId != null ? { VersionId: versionId } : {}), }), - timeout, - ]); - const decoded = Buffer.from(body.toString('utf8'), 'base64'); - if (decoded.byteLength > MAX_LEGACY_BYTES) { - const err = new Error( - `decoded legacy update (${decoded.byteLength}B) exceeds the ${MAX_LEGACY_BYTES}B cap`, - ); - err.permanent = true; // the object will be this big next time too - throw err; + { abortSignal }, + )); + } catch (err) { + // NoSuchVersion: the version vanished between listing and reading. + // NotFound is the bare 404 some S3-compatible providers answer with + // instead; a missing *bucket* has a name of its own and is not caught + // here — that one is a misconfiguration, not an absent document. + if ( + err?.name === 'NoSuchKey' || + err?.name === 'NoSuchVersion' || + err?.name === 'NotFound' + ) { + return null; } - // compute-task schema requires an exact Uint8Array (lib0 compares the - // constructor) — re-view the Buffer without copying - return new Uint8Array( - decoded.buffer, - decoded.byteOffset, - decoded.byteLength, - ); - } finally { - cancelTimeout(); + throw asTimeout(err, abortSignal, 's3 fetch', S3_FETCH_TIMEOUT_MS); + } + let encoded; + try { + // the object whole, whatever its size: it is one document's content, and + // refusing to read it is refusing to migrate that document at all + encoded = await body.transformToString('utf8'); + } catch (err) { + throw asTimeout(err, abortSignal, 's3 fetch', S3_FETCH_TIMEOUT_MS); } + const decoded = Buffer.from(encoded, 'base64'); + // compute-task schema requires an exact Uint8Array (lib0 compares the + // constructor) — re-view the Buffer without copying + return new Uint8Array(decoded.buffer, decoded.byteOffset, decoded.byteLength); }; // Every version of the legacy object, oldest first. Delete markers are skipped @@ -208,36 +197,58 @@ const fetchLegacyDoc = async (docid, versionId = null) => { // the prefix — S3 has no exact-key version listing. const listLegacyVersions = async (docid) => { const key = `${docid}/file`; - const found = await new Promise((resolve, reject) => { - const versions = []; - const stream = s3.listObjects(LEGACY_S3_BUCKET_NAME, key, true, { - IncludeVersion: true, - }); - const timer = setTimeout(() => { - const err = new Error( - `s3 version listing timed out after ${S3_LIST_TIMEOUT_MS}ms`, + // one budget for the whole listing, however many pages it takes + const abortSignal = AbortSignal.timeout(S3_LIST_TIMEOUT_MS); + const found = []; + try { + let keyMarker; + let versionIdMarker; + let truncated = true; + while (truncated) { + const page = await s3.send( + new ListObjectVersionsCommand({ + Bucket: LEGACY_S3_BUCKET_NAME, + Prefix: key, + KeyMarker: keyMarker, + VersionIdMarker: versionIdMarker, + }), + { abortSignal }, ); - stream.destroy(err); - }, S3_LIST_TIMEOUT_MS); - stream.on('data', (obj) => { - if (obj.name === key && obj.isDeleteMarker !== true && obj.versionId) { - versions.push({ - versionId: String(obj.versionId), - // the moment S3 accepted the write: what the backend's version - // listing reports as `last_modified`, and what we attribute to - timestamp: obj.lastModified?.getTime() ?? 0, - }); + // delete markers record a deletion and carry no body; they come in a + // list of their own here, so reading `Versions` skips them by itself + for (const version of page.Versions ?? []) { + if (version.Key === key && version.VersionId) { + found.push({ + versionId: String(version.VersionId), + // the moment S3 accepted the write: what the backend's version + // listing reports as `last_modified`, and what we attribute to + timestamp: version.LastModified?.getTime() ?? 0, + }); + } } - }); - stream.on('error', (err) => { - clearTimeout(timer); - reject(err); - }); - stream.on('end', () => { - clearTimeout(timer); - resolve(versions); - }); - }); + truncated = page.IsTruncated === true; + keyMarker = page.NextKeyMarker; + versionIdMarker = page.NextVersionIdMarker; + } + } catch (err) { + const failure = asTimeout( + err, + abortSignal, + 's3 version listing', + S3_LIST_TIMEOUT_MS, + ); + migrationLog.error( + { + event: 'list_version.failed', + err: failure, + docid, + bucket: LEGACY_S3_BUCKET_NAME, + key, + }, + 'impossible to list object version', + ); + throw failure; + } // S3 lists a key's versions newest first; reverse to replay them in write // order. The sort is a stable safeguard across paginated listings — equal // timestamps keep S3's own ordering. diff --git a/src/yhub-server/package-lock.json b/src/yhub-server/package-lock.json index d12a212b59..267f09f211 100644 --- a/src/yhub-server/package-lock.json +++ b/src/yhub-server/package-lock.json @@ -6,10 +6,10 @@ "": { "name": "yhub-server", "dependencies": { + "@aws-sdk/client-s3": "3.1110.0", "@y/hub": "0.6.0", "@y/y": "14.0.0-rc.24", - "jose": "6.2.8", - "minio": "8.0.7" + "jose": "6.2.8" }, "devDependencies": { "nodemon": "3.1.14" @@ -18,6 +18,314 @@ "node": ">=22" } }, + "node_modules/@aws-sdk/checksums": { + "version": "3.1000.27", + "resolved": "https://registry.npmjs.org/@aws-sdk/checksums/-/checksums-3.1000.27.tgz", + "integrity": "sha512-insWOqKKNUrbN/dohEG7BJ0U5GkyqhjbMb/NHNaLUtq+7my2M8C4EnZZZoxMmXRqCC+P9dEr+KyJA2JGGzoKLg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.7", + "@aws-sdk/types": "^3.974.3", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-s3": { + "version": "3.1110.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.1110.0.tgz", + "integrity": "sha512-40xbEcWjdaYKlZ4/NvndIJ3LotAEQAvHVQ7Z4NVy4Z4xGRN7xXJlHI9bMh/4aMJQ++6h5W5sv+wqjfk0rEKOBg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/checksums": "^3.1000.27", + "@aws-sdk/core": "^3.977.7", + "@aws-sdk/credential-provider-node": "^3.972.79", + "@aws-sdk/middleware-sdk-s3": "^3.972.73", + "@aws-sdk/signature-v4-multi-region": "^3.996.44", + "@aws-sdk/types": "^3.974.3", + "@smithy/core": "^3.31.1", + "@smithy/fetch-http-handler": "^5.6.13", + "@smithy/node-http-handler": "^4.9.13", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.977.7", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.977.7.tgz", + "integrity": "sha512-I88Iov89NVmjSmJLKSv7Cn9M2J+a2942OkA8nZCbz+sl4ZeY4zEOcoLOrbt1GRfQ8zEQKnjAJdXixA3J/p1fDQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.3", + "@aws-sdk/xml-builder": "^3.972.38", + "@aws/lambda-invoke-store": "^0.3.0", + "@smithy/core": "^3.31.1", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.16.1", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.68", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.68.tgz", + "integrity": "sha512-2a20A/IdNOwUvaDq91iqqS7BA0XlNMfW3iLGZGZLJv0EbUqhSxB0PIx4rQQqssvWj1uXImb3/UCCdHz/+1dOiA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.7", + "@aws-sdk/types": "^3.974.3", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.70", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.70.tgz", + "integrity": "sha512-0yRem2Fs52r/Nn6UAqIlpjexfaYj8ziEozOe9tamtAVT/5bzFLKx8O2r7MaRqgS3hGKHIa1Jij9nKHSsNnb04A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.7", + "@aws-sdk/types": "^3.974.3", + "@smithy/core": "^3.31.1", + "@smithy/fetch-http-handler": "^5.6.13", + "@smithy/node-http-handler": "^4.9.13", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.973.13", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.13.tgz", + "integrity": "sha512-2M39DE02XpYYaSWYk/4AsImXYUU/1L2xmTMLUpMMWq7DfLv191/vCRy3baKtdr45AkJQyVgSjmuVOLm15SwrRQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.7", + "@aws-sdk/credential-provider-env": "^3.972.68", + "@aws-sdk/credential-provider-http": "^3.972.70", + "@aws-sdk/credential-provider-login": "^3.972.75", + "@aws-sdk/credential-provider-process": "^3.972.68", + "@aws-sdk/credential-provider-sso": "^3.973.12", + "@aws-sdk/credential-provider-web-identity": "^3.972.74", + "@aws-sdk/nested-clients": "^3.997.42", + "@aws-sdk/types": "^3.974.3", + "@smithy/core": "^3.31.1", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.75", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.75.tgz", + "integrity": "sha512-jaTESuJlQsoUZ44f/i2puyPt8VlF/dMMJ9HM3cStYtk7eKX4N9UWi83OLixUkoOJH3BwWlPLCq9YIK9nfWhVBg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.7", + "@aws-sdk/nested-clients": "^3.997.42", + "@aws-sdk/types": "^3.974.3", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.79", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.79.tgz", + "integrity": "sha512-RIw5dof1EHkWubrZzPC941CDtnFG1iAXsxbFgLkhdYZXHc4icU13c/uxSMI0J5eUx9bxa7LjfpdjfClBB1QsDA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.68", + "@aws-sdk/credential-provider-http": "^3.972.70", + "@aws-sdk/credential-provider-ini": "^3.973.13", + "@aws-sdk/credential-provider-process": "^3.972.68", + "@aws-sdk/credential-provider-sso": "^3.973.12", + "@aws-sdk/credential-provider-web-identity": "^3.972.74", + "@aws-sdk/types": "^3.974.3", + "@smithy/core": "^3.31.1", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.68", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.68.tgz", + "integrity": "sha512-nLP3Pda2MQTFJ25hKBMmUuB9Uv+bTZQNlufbeCwklP549Vwnkd8bRLJoCKp5k6xjmdyptrPrOfGOhN0mKuca8A==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.7", + "@aws-sdk/types": "^3.974.3", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.973.12", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.12.tgz", + "integrity": "sha512-EmgyyHn+f9WCcelp3L/vci+LGbX8GigWaVphRArjVo5Pktkr9YnLy/mQ6VDkDyBD72dtfRNTgHmD2ts4rTDXKQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.7", + "@aws-sdk/nested-clients": "^3.997.42", + "@aws-sdk/token-providers": "3.1108.0", + "@aws-sdk/types": "^3.974.3", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.74", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.74.tgz", + "integrity": "sha512-0YfczxGXF3RjGj8z7QG/Ho2HnLGKDHfPSHiTs47UU1U/+mmwISDN+rvGKt2zh+3FX8NdT4xd95LGBGyhQw2dgQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.7", + "@aws-sdk/nested-clients": "^3.997.42", + "@aws-sdk/types": "^3.974.3", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3": { + "version": "3.972.73", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.972.73.tgz", + "integrity": "sha512-oy7sRA5HvHcAvkcKX6F8RI240jcOf3c8y/Gqjs9qemIibdKQqGBIi0uwa+47ZRYqGLpdEO28TQU4G73yUzo06Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.7", + "@aws-sdk/signature-v4-multi-region": "^3.996.44", + "@aws-sdk/types": "^3.974.3", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.997.42", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.42.tgz", + "integrity": "sha512-XWRyon2MTHXD/zMoo0Mbge6Vwf+iE0qQaM/RyGO6NfZ9WukCFiQL27nQVZjYy2JwSIg+iXZxKOX95OBXqlSM4w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.7", + "@aws-sdk/signature-v4-multi-region": "^3.996.44", + "@aws-sdk/types": "^3.974.3", + "@smithy/core": "^3.31.1", + "@smithy/fetch-http-handler": "^5.6.13", + "@smithy/node-http-handler": "^4.9.13", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.44", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.44.tgz", + "integrity": "sha512-ZSfQ35Qn4MhSY+A0Whyr+KBx+wJKZUyBsOrjB2pSHOafRzbFe47T8XcXM8hZqUAC69qnqIy0C9ArxTuud0CC2w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.3", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.1108.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1108.0.tgz", + "integrity": "sha512-rI80zxDxGJ6904eC/YbjkdjY6JdaZvQ01kOmrMvw7cFQGIHo27fhnIVbMSVDS4T6foQImjxYSRoOu/uSJscXDw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.7", + "@aws-sdk/nested-clients": "^3.997.42", + "@aws-sdk/types": "^3.974.3", + "@smithy/core": "^3.31.1", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.974.3", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.3.tgz", + "integrity": "sha512-ECAqfpNsef+7MO8qtR0h9KcFIBAygaE7Cm6UOiQl+ft+uVap+1G7bNEjs4mdJE2OnA4m6k7i8peH8uGIAsOMGw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.972.38", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.38.tgz", + "integrity": "sha512-grf7mzfVxBS5AlsuTvBN7uDpzqohFww9fRPCO+EBSUdvtsYMcPSKdz54h/7XiscqNcUM1Ae1MF7JLHmiYYuzbQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", + "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@nodable/entities": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", @@ -108,6 +416,87 @@ "@redis/client": "^5.12.1" } }, + "node_modules/@smithy/core": { + "version": "3.33.0", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.33.0.tgz", + "integrity": "sha512-uKbkxgqLyepQDZoq8aRSdUqD1ID//rOqG96ixBhp++O7vBtmwYM6fwldGhr9HJP0iYrdc7GP/AlgzPWEZIrNRg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.17.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.5.0.tgz", + "integrity": "sha512-2jsPi+7Zv2hSzD9IXR9D7DTqSn7mv4XalzRm+bESh53jiaUS3NKEUbpQFTJP0HhQy9qzZvluxQ3yS24zdRrqsA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.32.0", + "@smithy/types": "^4.17.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.7.0.tgz", + "integrity": "sha512-W/exA8T0LEzCQtJ02w4IzaEQPIspgarqZprb7W8FwnYiDowgCrjl2fTQ6FvuSSUnJORuepBF81abmBJwqh+0XQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.32.0", + "@smithy/types": "^4.17.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.11.0", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.11.0.tgz", + "integrity": "sha512-ssHIZsadPUA3lGdnoByxfnjtb9xPYQLvdfJRLKIwxOoa6tO1suG4sLFSsgd7D/CsvYd8QbBIuKTImuJha5l6aQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.0", + "@smithy/types": "^4.17.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.7.0.tgz", + "integrity": "sha512-hCynhm22wMJ8wTF9crcwu8mxggtUrSLLJgDcGUvYFBqpofxycYJCGKOMYg4xtPPFtgNiDJSYmhsWLTrcU/g59Q==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.32.0", + "@smithy/types": "^4.17.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.17.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.17.0.tgz", + "integrity": "sha512-Aw4joiM0ZdErpo39lCj8phT2lxoiKZV+KZzBxnnQhWVtU2Is/WffQSL04uUWRcXUse9Ln8vXZK6V/FwqRVnQpg==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@y-crdt/yn": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/@y-crdt/yn/-/yn-0.1.4.tgz", @@ -252,6 +641,12 @@ "readable-stream": "^3.4.0" } }, + "node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "5.0.9", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.9.tgz", @@ -1067,6 +1462,12 @@ "nodetouch": "bin/nodetouch.js" } }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, "node_modules/undefsafe": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz", diff --git a/src/yhub-server/package.json b/src/yhub-server/package.json index a4d33c6815..3684696974 100644 --- a/src/yhub-server/package.json +++ b/src/yhub-server/package.json @@ -8,10 +8,10 @@ "init-db": "node node_modules/@y/hub/bin/init-db.js" }, "dependencies": { + "@aws-sdk/client-s3": "3.1110.0", "@y/hub": "0.6.0", "@y/y": "14.0.0-rc.24", - "jose": "6.2.8", - "minio": "8.0.7" + "jose": "6.2.8" }, "devDependencies": { "nodemon": "3.1.14" diff --git a/src/yhub-server/server.js b/src/yhub-server/server.js index 16697c14db..2d4c10e184 100644 --- a/src/yhub-server/server.js +++ b/src/yhub-server/server.js @@ -16,7 +16,6 @@ import { jwtVerify, SignJWT, } from 'jose'; -import { Client as S3Client } from 'minio'; import { secret } from './env.js'; // legacy Django/S3 document store — see migration.js and README.md @@ -237,11 +236,11 @@ const backendFetch = async (path, { cookie, origin }) => { // Seeding never decides whether the caller may read the document — that is the // backend's answer alone. There are two ways this ends other than a seed: // -// the legacy object cannot be migrated (it does not decode, or it is bigger -// than we will load) — retrying will not change that, so the room opens as -// a new document. Refusing instead would lock a document nobody can repair -// from the outside. Logged per access, because the caller is now editing -// alongside legacy content that stayed behind in S3. +// the legacy object cannot be migrated (it does not decode) — retrying will +// not change that, so the room opens as a new document. Refusing instead +// would lock a document nobody can repair from the outside. Logged per +// access, because the caller is now editing alongside legacy content that +// stayed behind in S3. // the legacy store could not be reached (timeout, network, backpressure) — // the same request later may well succeed, so it answers 503 rather than // silently starting an empty document on top of content that exists. From 1d306e6bab6c48616575db7e1f128bc0aa5b1a0c Mon Sep 17 00:00:00 2001 From: Manuel Raynaud Date: Fri, 14 Aug 2026 17:06:34 +0200 Subject: [PATCH 59/59] =?UTF-8?q?=E2=9C=A8(backend)=20allow=20too=20migrat?= =?UTF-8?q?e=20a=20specific=20document?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The management command migrating document to yhub didn't allow to target a specific document. This can be usefull for debugging purpose but also to replay the migration of a specific document. --- .../management/commands/migrate_documents.py | 29 +++++++++++- .../tests/commands/test_migrate_documents.py | 46 +++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/src/backend/core/management/commands/migrate_documents.py b/src/backend/core/management/commands/migrate_documents.py index 0e7d965bf7..078776d215 100644 --- a/src/backend/core/management/commands/migrate_documents.py +++ b/src/backend/core/management/commands/migrate_documents.py @@ -10,13 +10,17 @@ collaboration server answers "already" for anything it has already migrated, so a run that is interrupted, rate limited or killed simply picks up where it stopped. Nothing is destroyed, on either side. + +One document can also be handed over on its own with `--document-id`, which is +how a document a run left behind is dealt with once its cause is understood. """ import logging import time +import uuid from concurrent.futures import ThreadPoolExecutor -from django.core.management.base import BaseCommand +from django.core.management.base import BaseCommand, CommandError from django.utils import timezone from core import models @@ -46,6 +50,16 @@ class Command(BaseCommand): def add_arguments(self, parser): """Define the arguments of the command.""" + parser.add_argument( + "--document-id", + type=uuid.UUID, + default=None, + help=( + "Migrate this document alone, whatever a previous run recorded " + "for it. The filters selecting a corpus (--created-before, " + "--limit, --retry-failed) do not apply to it." + ), + ) parser.add_argument( "--concurrency", type=int, @@ -136,7 +150,20 @@ def get_queryset(self, options): A document is opened before it is missed: the ones edited recently are the ones users are about to read, and until a document is migrated the collaboration server only seeds its latest state, without its history. + + Naming one document is an instruction rather than a filter: it is handed + over even when a previous run recorded it as done, which costs a call + the collaboration server answers with "already". Nothing else would be + useful — a command asked for one document and reporting that it had + nothing to do says neither what happened nor why. """ + if options["document_id"]: + queryset = models.Document.objects.filter(pk=options["document_id"]) + if not queryset.exists(): + raise CommandError(f"No document with id {options['document_id']}") + + return queryset + queryset = models.Document.objects.all() if options["created_before"]: diff --git a/src/backend/core/tests/commands/test_migrate_documents.py b/src/backend/core/tests/commands/test_migrate_documents.py index de580689e3..e33e8b4363 100644 --- a/src/backend/core/tests/commands/test_migrate_documents.py +++ b/src/backend/core/tests/commands/test_migrate_documents.py @@ -1,9 +1,11 @@ """Unit tests for the `migrate_documents` command.""" +import uuid from io import StringIO from unittest import mock from django.core.management import call_command +from django.core.management.base import CommandError import pytest @@ -167,3 +169,47 @@ def test_commands_migrate_documents_dry_run(collaboration_server): assert "2 documents to migrate" in output collaboration_server.assert_not_called() assert not models.DocumentMigration.objects.exists() + + +def test_commands_migrate_documents_document_id(collaboration_server): + """Naming a document should hand over that one and leave the corpus alone.""" + factories.DocumentFactory.create_batch(3) + document = factories.DocumentFactory() + + output = run_command(document_id=document.pk) + + assert collaboration_server.call_count == 1 + assert collaboration_server.call_args[0][0].pk == document.pk + assert models.DocumentMigration.objects.get().document_id == document.pk + assert "ok=1" in output + + +def test_commands_migrate_documents_document_id_already_migrated( + collaboration_server, +): + """ + A document already recorded as done should be handed over again when named. + + Asking for a document by its id is an instruction, not a filter over what is + left to do: the collaboration server answers "already" when it has nothing + to replay, which is the answer the run records. + """ + document = factories.DocumentFactory() + run_command() + collaboration_server.return_value = migrated(status="already") + + run_command(document_id=document.pk) + + assert collaboration_server.call_count == 2 + assert ( + models.DocumentMigration.objects.get(document=document).status + == models.DocumentMigrationStatus.ALREADY + ) + + +def test_commands_migrate_documents_document_id_unknown(collaboration_server): + """An id that is no document should stop the command, not migrate nothing.""" + with pytest.raises(CommandError, match="No document with id"): + run_command(document_id=uuid.uuid4()) + + collaboration_server.assert_not_called()