Add tag-based document indexing allowlist

appinfo/info.xml

@@ -47,6 +47,7 @@ Refer to the [Context Chat Backend's readme](https://github.com/nextcloud/contex
 		<command>OCA\ContextChat\Command\Search</command>
 		<command>OCA\ContextChat\Command\Statistics</command>
 		<command>OCA\ContextChat\Command\Reindex</command>
+		<command>OCA\ContextChat\Command\CleanupUntagged</command>
 	</commands>
     <repair-steps>
         <install>

lib/BackgroundJobs/UntaggedCleanupCrawlJob.php

@@ -0,0 +1,83 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\ContextChat\BackgroundJobs;
+
+use OCA\ContextChat\Logger;
+use OCA\ContextChat\Service\FsEventService;
+use OCA\ContextChat\Service\IndexCompletionService;
+use OCA\ContextChat\Service\StorageService;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\IJobList;
+use OCP\BackgroundJob\QueuedJob;
+use OCP\Files\IRootFolder;
+
+/**
+ * For a single mount, walks all files in batches and removes from the
+ * index any file that does not have the "AI knowledge" tag. Mirrors
+ * StorageCrawlJob's batching/self-rescheduling shape, but for cleanup
+ * instead of indexing.
+ */
+class UntaggedCleanupCrawlJob extends QueuedJob {
+	public const BATCH_SIZE = 2000;
+	public const JOB_INTERVAL = 60;
+
+	public function __construct(
+		ITimeFactory $timeFactory,
+		private Logger $logger,
+		private IJobList $jobList,
+		private StorageService $storageService,
+		private FsEventService $fsEventService,
+		private IRootFolder $rootFolder,
+		private IndexCompletionService $indexCompletionService,
+	) {
+		parent::__construct($timeFactory);
+	}
+
+	/**
+	 * @param array{storage_id:int, root_id:int, last_file_id:int|null} $argument
+	 * @return void
+	 */
+	protected function run($argument): void {
+		$storageId = $argument['storage_id'];
+		$rootId = $argument['root_id'];
+		$lastFileId = ($argument['last_file_id'] ?? null) === null ? 0 : $argument['last_file_id'];
+
+		// Remove current iteration
+		$this->jobList->remove(self::class, $argument);
+
+		$mountFilesCount = 0;
+		$lastFileIdSeen = $lastFileId;
+		foreach ($this->storageService->getFilesInMount($storageId, $rootId, $lastFileId, self::BATCH_SIZE) as $fileId) {
+			$lastFileIdSeen = $fileId;
+			$mountFilesCount++;
+
+			$node = $this->rootFolder->getFirstNodeById($fileId);
+			if ($node === null) {
+				continue;
+			}
+
+			if (!$this->fsEventService->hasAiKnowledgeTag($node)) {
+				$this->fsEventService->onDelete($node, false);
+			}
+		}
+
+		if ($mountFilesCount > 0) {
+			// Schedule next batch
+			$this->jobList->scheduleAfter(self::class, $this->time->getTime() + self::JOB_INTERVAL, [
+				'storage_id' => $storageId,
+				'root_id' => $rootId,
+				'last_file_id' => $lastFileIdSeen,
+			]);
+		} else {
+			$this->logger->info('[UntaggedCleanupCrawlJob] Finished mount storage_id=' . $storageId . ' root_id=' . $rootId);
+			$this->indexCompletionService->checkAndMarkComplete();
+		}
+	}
+}

lib/BackgroundJobs/UntaggedCleanupSchedulerJob.php

@@ -0,0 +1,48 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\ContextChat\BackgroundJobs;
+
+use OCA\ContextChat\Logger;
+use OCA\ContextChat\Service\StorageService;
+use OCP\AppFramework\Services\IAppConfig;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\IJobList;
+use OCP\BackgroundJob\QueuedJob;
+
+/**
+ * Discovers all mounts and schedules an UntaggedCleanupCrawlJob for each,
+ * mirroring SchedulerJob's shape.
+ */
+class UntaggedCleanupSchedulerJob extends QueuedJob {
+	public function __construct(
+		ITimeFactory $timeFactory,
+		private Logger $logger,
+		private IJobList $jobList,
+		private StorageService $storageService,
+		private IAppConfig $appConfig,
+	) {
+		parent::__construct($timeFactory);
+	}
+
+	protected function run($argument): void {
+		// Mirror SchedulerJob: a fresh cleanup run means the initial-indexing
+		// status is no longer settled until this run finishes.
+		$this->appConfig->setAppValueInt('last_indexed_time', 0, lazy: true);
+		foreach ($this->storageService->getMounts() as $mount) {
+			$this->logger->debug('[UntaggedCleanupSchedulerJob] Scheduling cleanup storage_id=' . $mount['storage_id'] . ' root_id=' . $mount['root_id']);
+			$this->jobList->add(UntaggedCleanupCrawlJob::class, [
+				'storage_id' => $mount['storage_id'],
+				'root_id' => $mount['root_id'],
+				'last_file_id' => 0,
+			]);
+		}
+		$this->jobList->remove(self::class);
+	}
+}

lib/Command/CleanupUntagged.php

@@ -0,0 +1,43 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCA\ContextChat\Command;
+
+use OCA\ContextChat\BackgroundJobs\UntaggedCleanupSchedulerJob;
+use OCP\BackgroundJob\IJobList;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+/**
+ * Schedule removal from the index of all files that do not have the
+ * "AI knowledge" tag. Used when switching to the tag-only indexing mode.
+ */
+class CleanupUntagged extends Command {
+	public function __construct(
+		private IJobList $jobList,
+	) {
+		parent::__construct();
+	}
+
+	protected function configure() {
+		$this->setName('context_chat:cleanup-untagged')
+			->setDescription('Schedule removal from the index of all files that do not have the "AI knowledge" tag.');
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output): int {
+		if ($this->jobList->has(UntaggedCleanupSchedulerJob::class, null)) {
+			$output->writeln('<comment>A cleanup is already scheduled; nothing to do.</comment>');
+			return 0;
+		}
+		$this->jobList->add(UntaggedCleanupSchedulerJob::class);
+		$output->writeln('<info>Scheduled removal of untagged files from the index.</info>');
+		return 0;
+	}
+}

lib/Controller/ConfigController.php

@@ -7,10 +7,14 @@
 
 namespace OCA\ContextChat\Controller;
 
+use OCA\ContextChat\BackgroundJobs\SchedulerJob;
+use OCA\ContextChat\BackgroundJobs\UntaggedCleanupSchedulerJob;
 use OCP\AppFramework\Controller;
 use OCP\AppFramework\Http\Attribute\NoAdminRequired;
 use OCP\AppFramework\Http\DataResponse;
 use OCP\AppFramework\Services\IAppConfig;
+use OCP\BackgroundJob\IJobList;
+use OCP\IAppConfig as ICoreAppConfig;
 use OCP\IRequest;
 use OCP\PreConditionNotMetException;
 
@@ -20,6 +24,8 @@ public function __construct(
 		string $appName,
 		IRequest $request,
 		private IAppConfig $appConfig,
+		private IJobList $jobList,
+		private ICoreAppConfig $coreAppConfig,
 		private ?string $userId,
 	) {
 		parent::__construct($appName, $request);
@@ -43,16 +49,33 @@ public function setConfig(array $values): DataResponse {
 		return new DataResponse(1);
 	}
 
+	private function handleIndexModeChange(array $values): void {
+		if (!isset($values['index_mode'])) {
+			return;
+		}
+		$oldMode = $this->appConfig->getAppValueString('index_mode', 'all', lazy: true);
+		$newMode = $values['index_mode'];
+		if ($newMode === $oldMode) {
+			return;
+		}
+		if ($newMode === 'tag_only') {
+			$this->jobList->add(UntaggedCleanupSchedulerJob::class);
+		} else {
+			$this->jobList->add(SchedulerJob::class);
+		}
+	}
 	/**
 	 * Set admin config values
 	 *
 	 * @param array $values key/value pairs to store in app config
 	 * @return DataResponse
 	 */
 	public function setAdminConfig(array $values): DataResponse {
+		$this->handleIndexModeChange($values);
 		foreach ($values as $key => $value) {
 			$this->appConfig->setAppValueString($key, $value, lazy: true);
 		}
+		$this->coreAppConfig->clearCache();
 		return new DataResponse(1);
 	}
 }

lib/Controller/QueueController.php

@@ -10,12 +10,12 @@
 
 namespace OCA\ContextChat\Controller;
 
-use OCA\ContextChat\BackgroundJobs\StorageCrawlJob;
 use OCA\ContextChat\Db\QueueActionMapper;
 use OCA\ContextChat\Db\QueueContentItem;
 use OCA\ContextChat\Db\QueueContentItemMapper;
 use OCA\ContextChat\Db\QueueFile;
 use OCA\ContextChat\Db\QueueMapper;
+use OCA\ContextChat\Service\IndexCompletionService;
 use OCA\ContextChat\Service\ProviderConfigService;
 use OCA\ContextChat\Service\QueueService;
 use OCA\ContextChat\Service\StorageService;
@@ -39,7 +39,6 @@
 use Psr\Log\LoggerInterface;
 
 class QueueController extends OCSController {
-	private const INDEX_COMPLETION_THRESHOLD = 0.02; // 2%
 
 	public function __construct(
 		string $appName,
@@ -51,6 +50,7 @@ public function __construct(
 		private IJobList $jobList,
 		private ITimeFactory $timeFactory,
 		private QueueMapper $queueMapper,
+		private IndexCompletionService $indexCompletionService,
 		string $corsMethods = 'PUT, POST, GET, DELETE, PATCH',
 		string $corsAllowedHeaders = 'Authorization, Content-Type, Accept, OCS-APIRequest',
 		int $corsMaxAge = 1728000,
@@ -112,6 +112,7 @@ public function getDocumentsQueueItems(
 			$n = $maxN;
 		}
 		try {
+			$this->indexCompletionService->checkAndMarkComplete();
 			$files = [];
 			while (count($files) < $n) {
 				$limit = $n - count($files);
@@ -181,7 +182,7 @@ public function deleteDocumentsQueueItems(IDBConnection $db, QueueMapper $queueM
 		}
 
 		try {
-			$this->setInitialIndexCompletion();
+			$this->indexCompletionService->checkAndMarkComplete();
 		} catch (\Exception $e) {
 			$this->logger->warning('Could not check for initial index completion', ['exception' => $e]);
 		}
@@ -327,75 +328,4 @@ private function getContentItemSource(QueueContentItem $document) : Source {
 			strlen($document->getContent()),
 		);
 	}
-
-	/**
-	 * @template T of \OCP\BackgroundJob\Job
-	 * @psalm-param T::class $jobClass
-	 */
-	public function getJobCount(string $jobClass): int {
-		$countByClass = array_values(array_filter($this->jobList->countByClass(), fn ($row) => $row['class'] == $jobClass));
-		$jobCount = count($countByClass) > 0 ? $countByClass[0]['count'] : 0;
-		return $jobCount;
-	}
-
-	private function setInitialIndexCompletion(): void {
-		if ($this->appConfig->getAppValueInt('last_indexed_time', 0, lazy: true) !== 0) {
-			return;
-		}
-
-		try {
-			$crawlJobCount = $this->getJobCount(StorageCrawlJob::class);
-			if ($crawlJobCount > 0) {
-				$this->logger->debug('StorageCrawlJob\'s still scheduled for execution, intial indexing has not completed.');
-				return;
-			}
-		} catch (\Exception $e) {
-			$this->logger->warning('Could not get count of scheduled StorageCrawlJob jobs', ['exception' => $e]);
-			return;
-		}
-
-		try {
-			$lastEnqueuedDbId = $this->appConfig->getAppValueInt('last_enqueued_db_id', -1, lazy: true);
-			if ($lastEnqueuedDbId !== -1) {
-				$initiallyQueuedFilesExist = $this->queueMapper->existsQueueItemsUpToDbId($lastEnqueuedDbId);
-				if ($initiallyQueuedFilesExist) {
-					$this->logger->debug('Initially queued files still in the queue, intial indexing has not completed.');
-					return;
-				}
-				$this->logger->info('Initial index completion detected, setting last indexed time');
-				$this->appConfig->setAppValueInt('last_indexed_time', $this->timeFactory->getTime(), lazy: true);
-				return;
-			}
-		} catch (\Exception $e) {
-			$this->logger->warning('Could not get last enqueued file\'s DB id', ['exception' => $e]);
-		}
-
-		// last enqueued file's ID could not be retrieved, falling back to file counting method
-		try {
-			$queuedNewFilesCount = $this->queueService->countNewFiles();
-			$eligibleFilesCount = $this->storageService->countFiles();
-			// if the new files in the queue are less than 2% of the total eligible files, we consider the
-			// initial indexing complete this allows for some margin of error in case some files were
-			// added while we were indexing but still ensures that we have indexed the vast majority of
-			// files at least once
-			if (self::withinThreshold($queuedNewFilesCount, $eligibleFilesCount)) {
-				$this->logger->info('Initial index completion detected, setting last indexed time');
-				$this->appConfig->setAppValueInt('last_indexed_time', $this->timeFactory->getTime(), lazy: true);
-				return;
-			}
-		} catch (\OCP\DB\Exception $e) {
-			$this->logger->warning('Could not count queued new files or total eligible files', ['exception' => $e]);
-			return;
-		}
-
-		// we are still indexing files that were never indexed before.
-		$this->logger->debug('Initial indexing not completed yet', [
-			'queuedNewFilesCount' => $queuedNewFilesCount,
-			'eligibleFilesCount' => $eligibleFilesCount,
-		]);
-	}
-
-	private static function withinThreshold(int $current, int $total, float $threshold = self::INDEX_COMPLETION_THRESHOLD): bool {
-		return ((float)($total - $current) / (float)$total) < $threshold;
-	}
 }