Upstream main

.github/workflows/integration-test.yml

@@ -192,6 +192,7 @@ jobs:
         run: |
           ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080
           ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish
+          ls -la context_chat_backend/persistent_storage/*
 
       - name: Scan files, baseline
         run: |
@@ -216,8 +217,8 @@ jobs:
 
       - name: Run the prompts
         run: |
-          ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
-          ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
+          ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' > worker1_logs 2>&1 &
+          ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' > worker2_logs 2>&1 &
 
           OUT1=$(./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?")
           echo "$OUT1"
@@ -259,15 +260,40 @@ jobs:
             echo "Memory usage during prompt is stable. No memory leak detected."
           fi
 
-      - name: Show logs
+      - name: Show server logs
         if: always()
         run: |
           cat data/nextcloud.log
-          echo '--------------------------------------------------'
+
+      - name: Show context_chat specific logs
+        if: always()
+        run: |
+          cat data/context_chat.log
+
+      - name: Show task processing worker logs
+        if: always()
+        run: |
+          tail -v -n +1 worker?_logs || echo "No worker logs"
+
+      - name: Show main app logs
+        if: always()
+        run: |
           cat context_chat_backend/backend_logs || echo "No main backend logs"
+
+      - name: Show main app JSON logs
+        if: always()
+        run: |
+          tail -v -n +1 context_chat_backend/persistent_storage/logs/ccb.log* || echo "No logs in logs directory"
+
+      - name: Show embedding server logs
+        if: always()
+        run: |
           cat context_chat_backend/em_backend_logs || echo "No main backend logs"
-          echo '--------------------------------------------------'
-          tail -v -n +1 context_chat_backend/persistent_storage/logs/* || echo "No logs in logs directory"
+
+      - name: Show embedding server JSON logs
+        if: always()
+        run: |
+          tail -v -n +1 context_chat_backend/persistent_storage/logs/em_server.log* || echo "No logs in logs directory"
 
   summary:
     permissions:

Dockerfile

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
 # SPDX-License-Identifier: AGPL-3.0-or-later
 #
-FROM nvidia/cuda:12.2.2-runtime-ubuntu22.04
+FROM docker.io/nvidia/cuda:12.2.2-runtime-ubuntu22.04
 
 ARG CCB_DB_NAME=ccb
 ARG CCB_DB_USER=ccbuser

Makefile

@@ -21,7 +21,7 @@ help:
 #.PHONY: build-push
 #build-push:
 #	docker login ghcr.io
-#	docker buildx build --push --platform linux/arm64/v8,linux/amd64 --tag ghcr.io/nextcloud/context_chat_backend:4.4.1 --tag ghcr.io/nextcloud/context_chat_backend:latest .
+#	docker buildx build --push --platform linux/arm64/v8,linux/amd64 --tag ghcr.io/nextcloud/context_chat_backend:4.5.0 --tag ghcr.io/nextcloud/context_chat_backend:latest .
 
 .PHONY: run
 run:
@@ -34,5 +34,5 @@ run:
 register:
 	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister context_chat_backend --silent || true
 	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register context_chat_backend manual_install --json-info \
-  "{\"id\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"4.4.1\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system\":0}" \
+  "{\"id\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"4.5.0\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system\":0}" \
   --force-scopes --wait-finish

README.md

@@ -11,11 +11,11 @@
 >
 > The HTTP request timeout is 50 minutes for all requests and can be changed with the `request_timeout` app config for the php app `context_chat` using the occ command (`occ config:app:set context_chat request_timeout --value=3000`, value is in seconds). The same also needs to be done for docker socket proxy. See [Slow responding ExApps](https://github.com/cloud-py-api/docker-socket-proxy?tab=readme-ov-file#slow-responding-exapps)
 >
-> An end-to-end example on how to build and register the backend manually (with CUDA) is at the end of this readme
+> An end-to-end example for dev setups on how to build and register the backend manually (with CUDA) is at the end of this readme
 >
 > See the [NC Admin docs](https://docs.nextcloud.com/server/latest/admin_manual/ai/app_context_chat.html) for requirements and known limitations.
 
-## Simple Install
+## Install
 
 Install the given apps for Context Chat to work as desired **in the given order**:
 - [AppAPI from the Apps page](https://apps.nextcloud.com/apps/app_api)
@@ -32,25 +32,26 @@ Install the given apps for Context Chat to work as desired **in the given order*
 > [!IMPORTANT]
 > To avoid task processing execution delay, setup at 4 background job workers in the main server (where Nextcloud is installed). The setup process is documented here: https://docs.nextcloud.com/server/latest/admin_manual/ai/overview.html#improve-ai-task-pickup-speed
 
-## Complex Install (without docker)
+## Dev Install (without docker)
 
-0. Install the required apps from [Simple Install](#simple-install) other than Context Chat Backend and setup background job workers
+0. Install the required apps from [Install](#install) other than Context Chat Backend and setup background job workers
 1. `python -m venv .venv`
 2. `. .venv/bin/activate`
 3. `pip install --upgrade pip setuptools wheel`
 4. Install requirements `pip install -r requirements.txt`
 5. Copy example.env to .env and fill in the variables
-6. Ensure the config file at `persistent_storage/config.yaml` points to the correct config file (cpu vs gpu). If you're unsure, delete it. It will be recreated upon launching the application. The default is to point to the gpu config.
-7. Configure `persistent_storage/config.yaml` for the model name, model type and its parameters (which also includes model file's path and model id as per requirements, see example config)
+6. Ensure the config file at `persistent_storage/config.yaml` points to the correct config file (cpu vs gpu). If you're unsure, delete it. It will be recreated upon launching the application.
+7. Configure `persistent_storage/config.yaml`
 8. Setup postgresql externally or use `dockerfile_scripts/pgsql/install.sh` to install it on a Debian-family system.
 9. Set the env var `EXTERNAL_DB` or the `connection` key in the `pgvector` config to the postgresql connection string if you're using an external database.
 10. Start the database (see `dockerfile_scripts/pgsql/setup.sh` for an example)
-11. `./main.py`
-12. [Follow the below steps to register the app in the app ecosystem](#register-as-an-ex-app)
+11. `./main_em.py` in one terminal/tmux pane
+12. `./main.py` in another
+13. [Follow the below steps to register the app in the app ecosystem](#register-as-an-ex-app)
 
-## Complex Install (with docker)
+## Dev Install (with docker)
 
-0. Install the required apps from [Simple Install](#simple-install) other than Context Chat Backend and setup background job workers
+0. Install the required apps from [Install](#install) other than Context Chat Backend and setup background job workers
 1. Build the image
     *(this is a good place to edit the example.env file before building the container)*
     `docker build -t context_chat_backend . -f Dockerfile`
@@ -67,12 +68,12 @@ occ app_api:daemon:register --net host manual_install "Manual Install" manual-in
 ```
 `host` will be `localhost` if nextcloud can access localhost or `host.docker.internal` if nextcloud is inside a docker container and the backend app is on localhost.
 
-If nextcloud is inside a container, `--add-host` option would be required by your nextcloud container. [See example above, pt. 2](#complex-install-with-docker)
+If nextcloud is inside a container, `--add-host` option would be required by your nextcloud container. [See example above, pt. 2](#dev-install-with-docker)
 
 **2. Register the app using the deploy daemon (be mindful of the port number and the app's version):**
 ```
 occ app_api:app:register context_chat_backend manual_install --json-info \
-"{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"4.4.1\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" \
+"{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"4.5.0\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" \
 --force-scopes --wait-finish
 ```
 The command to unregister is given below (force is used to also remove apps whose container has been removed)
@@ -167,7 +168,6 @@ Adjust the example.env to your needs so that it fits your environment
 docker run \
     -v ./config.yaml:/app/config.yaml \
     -v ./context_chat_backend:/app/context_chat_backend \
-    -v /var/run/docker.sock:/var/run/docker.sock \
     --env-file example.env \
     -p 10034:10034 \
     -e CUDA_VISIBLE_DEVICES=0 \
@@ -186,10 +186,6 @@ docker run \
 
 	Mounts the context_chat_backend into the docker image
 
-	`-v /var/run/docker.sock:/var/run/docker.sock`
-
-	Mounts the Docker socket file from the host into the container. This is done to allow the Docker client running inside the container to communicate with the Docker daemon on the host, essentially controlling Docker and GPU from within the container.
-
 	`-v persistent_storage:/app/persistent_storage`
 
 	Mounts the persistent storage into the docker instance to keep downloaded models stored for the future.
@@ -229,11 +225,9 @@ sudo -u www-data php occ app_api:app:register \
     --json-info "{\"appid\":\"context_chat_backend\",\
                   \"name\":\"Context Chat Backend\",\
                   \"daemon_config_name\":\"manual_install\",\
-                  \"version\":\"4.4.1\",\
+                  \"version\":\"4.5.0\",\
                   \"secret\":\"12345\",\
-                  \"port\":10034,\
-                  \"scopes\":[],\
-                  \"system_app\":0}" \
+                  \"port\":10034}" \
     --force-scopes \
     --wait-finish
 ```

appinfo/info.xml

@@ -20,7 +20,7 @@ Install the given apps for Context Chat to work as desired **in the given order*
 
 Setup background job workers as described here: https://docs.nextcloud.com/server/latest/admin_manual/ai/overview.html#improve-ai-task-pickup-speed
 ]]></description>
-	<version>4.4.1</version>
+	<version>4.5.0</version>
 	<licence>agpl</licence>
 	<author mail="kyteinsky@gmail.com" homepage="https://github.com/kyteinsky">Anupam Kumar</author>
 	<namespace>Context Chat</namespace>
@@ -29,13 +29,13 @@ Setup background job workers as described here: https://docs.nextcloud.com/serve
 	<bugs>https://github.com/nextcloud/context_chat_backend/issues</bugs>
 	<repository type="git">https://github.com/nextcloud/context_chat_backend.git</repository>
 	<dependencies>
-		<nextcloud min-version="30" max-version="32"/>
+		<nextcloud min-version="30" max-version="33"/>
 	</dependencies>
 	<external-app>
 		<docker-install>
 			<registry>ghcr.io</registry>
 			<image>nextcloud/context_chat_backend</image>
-			<image-tag>4.4.1</image-tag>
+			<image-tag>4.5.0</image-tag>
 		</docker-install>
 		<routes>
 			<route>

changelog.md

@@ -9,6 +9,56 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
 
+## 4.5.0 - 2025-09-23
+
+### Changed
+- stable release of the 4.5.0-beta series
+- bump max NC version to 33
+- store worker logs and present them better (#200) @kyteinsky
+
+### Fixed
+- better parsing of version string for repairs (#209) @kyteinsky
+- suppress tokenizer max context length warning (#200) @kyteinsky
+- fix nc_py_api to 0.20.2 (#200) @kyteinsky
+
+### Added
+- add app version to file logs (#201) @kyteinsky
+
+
+## 4.5.0-beta.3 - 2025-09-12
+
+### Changed
+- actually bump max NC version to 33
+
+
+## 4.5.0-beta.2 - 2025-09-12
+
+### Changed
+- bump max NC version to 33
+
+### Fixed
+- let empty but set title headers pass (#211) @kyteinsky
+
+
+## 4.5.0-beta.1 - 2025-09-09
+
+### Fixed
+- better parsing of version string for repairs (#209) @kyteinsky
+
+
+## 4.5.0-beta.0 - 2025-08-29
+
+### Fixed
+- suppress tokenizer max context length warning (#200) @kyteinsky
+- fix nc_py_api to 0.20.2 (#200) @kyteinsky
+
+### Changed
+- store worker logs and present them better (#200) @kyteinsky
+
+### Added
+- add app version to file logs (#201) @kyteinsky
+
+
 ## 4.4.1 - 2025-07-31
 
 ### Fixed

context_chat_backend/chain/query_proc.py

@@ -3,12 +3,23 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 #
 import logging
+from sys import maxsize as SYS_MAXSIZE
 
 from langchain.llms.base import LLM
+from transformers import GPT2Tokenizer
 
 from ..types import TConfig
 
 logger = logging.getLogger('ccb.chain')
+TOKENIZER = GPT2Tokenizer.from_pretrained('gpt2')
+
+
+def get_num_tokens(text: str) -> int:
+	'''
+	Returns the number of tokens in the text using the fast GPT2 tokenizer.
+	'''
+	return len(TOKENIZER.encode(text, max_length=SYS_MAXSIZE, truncation=True))
+
 
 def get_pruned_query(llm: LLM, config: TConfig, query: str, template: str, text_chunks: list[str]) -> str:
 	'''
@@ -36,16 +47,16 @@ def get_pruned_query(llm: LLM, config: TConfig, query: str, template: str, text_
 		) \
 		or 4096
 
-	query_tokens = llm.get_num_tokens(query)
-	template_tokens = llm.get_num_tokens(template.format(context='', question=''))
+	query_tokens = get_num_tokens(query)
+	template_tokens = get_num_tokens(template.format(context='', question=''))
 
 	# remaining tokens after the template, query and 'to be' generated tokens
 	remaining_tokens = n_ctx - template_tokens - query_tokens - n_gen
 
 	# If the query is too long to fit in the context, truncate it (keeping the template)
 	if remaining_tokens <= 0:
 		new_remaining_tokens = n_ctx - template_tokens - n_gen
-		while query and llm.get_num_tokens(query) > new_remaining_tokens:
+		while query and get_num_tokens(query) > new_remaining_tokens:
 			query = ' '.join(query.split()[:-10])
 
 		if not query:
@@ -57,7 +68,7 @@ def get_pruned_query(llm: LLM, config: TConfig, query: str, template: str, text_
 
 	while text_chunks and remaining_tokens > 0:
 		context = text_chunks.pop(0)
-		context_tokens = llm.get_num_tokens(context)
+		context_tokens = get_num_tokens(context)
 
 		if context_tokens <= remaining_tokens:
 			accepted_chunks.append(context)

context_chat_backend/controller.py

@@ -354,7 +354,7 @@ def _(sources: list[UploadFile]):
 
 		if not (
 			value_of(source.headers.get('userIds'))
-			and value_of(source.headers.get('title'))
+			and source.headers.get('title', None) is not None
 			and value_of(source.headers.get('type'))
 			and value_of(source.headers.get('modified'))
 			and source.headers['modified'].isdigit()

context_chat_backend/logger.py

@@ -12,8 +12,11 @@
 import os
 from time import gmtime
 
+from dotenv import load_dotenv
 from ruamel.yaml import YAML
 
+load_dotenv()
+
 __all__ = ['JSONFormatter', 'setup_logging']
 
 LOG_RECORD_BUILTIN_ATTRS = {
@@ -62,6 +65,7 @@ def _prepare_log_dict(self, record: logging.LogRecord):
 			"timestamp": dt.datetime.fromtimestamp(
 				record.created, tz=dt.UTC,
 			).isoformat(),
+			"version": os.getenv("APP_VERSION", "unknown"),
 		}
 		if record.exc_info is not None:
 			always_fields["exc_info"] = self.formatException(record.exc_info)

context_chat_backend/repair/runner.py

@@ -16,10 +16,19 @@ def get_previous_version(version_info_path: str) -> tuple[int, bool]:
 		return (0, False)
 
 	with open(version_info_path) as f:
-		version_string = f.read()
+		version_string = f.read().strip()
 
-	major, minor, patch = version_string.split('.')
-	repairs_pending = not (patch.endswith('+') and version_string.rstrip('+') == os.environ['APP_VERSION'])
+	if not version_string:
+		return (0, False)
+
+	splits = version_string.split('.')
+	major = splits[0]
+	minor = splits[1] if len(splits) > 1 else '0'
+
+	repairs_pending = not (
+		version_string.endswith('+')
+		and version_string.rstrip('+') == os.environ['APP_VERSION']
+	)
 
 	return (int(major + minor.zfill(3)), repairs_pending)
 

example.env

@@ -18,7 +18,7 @@ AA_VERSION=3.0.0
 APP_SECRET=12345
 APP_ID=context_chat_backend
 APP_DISPLAY_NAME=Context Chat Backend
-APP_VERSION=4.4.1
+APP_VERSION=4.5.0
 APP_HOST=0.0.0.0
 APP_PORT=10034
 APP_PERSISTENT_STORAGE=persistent_storage

main_em.py

@@ -82,6 +82,7 @@
 			'model_files',
 			em_conf.llama['model'],
 		)
+		logger.debug(f'Trying model path: {em_conf.llama["model"]}')
 
 		# if the model file is still not found, raise an error
 		if not os.path.isfile(em_conf.llama['model']):

pyproject.toml

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 [project]
 name = "context_chat_backend"
-version = "4.4.1"
+version = "4.5.0"
 requires-python = ">=3.11"
 authors = [
 	{ name = "Anupam Kumar", email = "kyteinsky@gmail.com" }