diff --git a/src/services/code-index/processors/__tests__/file-preparation.spec.ts b/src/services/code-index/processors/__tests__/file-preparation.spec.ts new file mode 100644 index 0000000000..22020a9fdd --- /dev/null +++ b/src/services/code-index/processors/__tests__/file-preparation.spec.ts @@ -0,0 +1,327 @@ +import { createHash } from "crypto" +import path from "path" +import { FileType, Uri } from "vscode" +import { v5 as uuidv5 } from "uuid" +import type { CodeBlock, ICodeParser, IEmbedder } from "../../interfaces" +import { MAX_FILE_SIZE_BYTES, QDRANT_CODE_BLOCK_NAMESPACE } from "../../constants" +import { FilePreparation } from "../file-preparation" +import type { FilePreparationDependencies } from "../file-preparation-dependencies" + +// Keep path spies local to the modules under test, not Node or the test runner. +vi.mock("path", async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, default: { ...actual } } +}) + +describe.each(["posix", "win32"] as const)("FilePreparation (%s paths)", (platform) => { + beforeEach(() => { + vi.spyOn(path, "resolve").mockImplementation(path[platform].resolve) + vi.spyOn(path, "relative").mockImplementation(path[platform].relative) + vi.spyOn(path, "normalize").mockImplementation(path[platform].normalize) + }) + + afterEach(() => vi.restoreAllMocks()) + + const filePath = "/workspace/src/file.ts" + const relativeFilePath = path[platform].join("src", "file.ts") + const normalizedFilePath = path[platform].resolve(filePath) + const content = "test content" + const hash = createHash("sha256").update(content).digest("hex") + const block: CodeBlock = { + file_path: filePath, + identifier: null, + type: "function", + start_line: 2, + end_line: 5, + content, + fileHash: hash, + segmentHash: "segment", + } + + function setup() { + return { + workspacePath: "/workspace", + ignoreController: { + validateAccess: vi + .fn() + .mockReturnValue(true), + }, + ignoreInstance: { ignores: vi.fn<(path: string) => boolean>().mockReturnValue(false) }, + fileSystem: { + stat: vi + .fn() + .mockResolvedValue({ type: FileType.File, ctime: 0, mtime: 0, size: 100 }), + readFile: vi + .fn() + .mockResolvedValue(Buffer.from(content)), + }, + cacheManager: { getHash: vi.fn() }, + parser: { parseFile: vi.fn().mockResolvedValue([{ ...block }]) }, + embedder: { + createEmbeddings: vi + .fn() + .mockResolvedValue({ embeddings: [[0.1, 0.2]] }), + }, + } satisfies FilePreparationDependencies + } + + it.each([".git/config", ".hidden/file.ts", "node_modules/pkg/file.ts", "dist/file.js"])( + "skips excluded directory %s before access or file reads", + async (relativePath) => { + const dependencies = setup() + const path = `/workspace/${relativePath}` + expect(await new FilePreparation(dependencies).prepareFile(path)).toEqual({ + path, + status: "skipped", + reason: "File is in an ignored directory", + }) + expect(dependencies.ignoreController.validateAccess).not.toHaveBeenCalled() + expect(dependencies.ignoreInstance.ignores).not.toHaveBeenCalled() + expect(dependencies.fileSystem.stat).not.toHaveBeenCalled() + expect(dependencies.fileSystem.readFile).not.toHaveBeenCalled() + expect(dependencies.cacheManager.getHash).not.toHaveBeenCalled() + expect(dependencies.parser.parseFile).not.toHaveBeenCalled() + expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() + }, + ) + + it.each(["access", "gitignore"])("skips %s exclusions before reading", async (source) => { + const dependencies = setup() + dependencies.ignoreController.validateAccess.mockReturnValue(source !== "access") + dependencies.ignoreInstance.ignores.mockReturnValue(source === "gitignore") + expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ + path: filePath, + status: "skipped", + reason: "File is ignored by .rooignore or .gitignore", + }) + expect(dependencies.ignoreController.validateAccess).toHaveBeenCalledWith(filePath) + if (source === "access") { + expect(dependencies.ignoreInstance.ignores).not.toHaveBeenCalled() + } else { + expect(dependencies.ignoreInstance.ignores).toHaveBeenCalledWith(relativeFilePath) + } + expect(dependencies.fileSystem.stat).not.toHaveBeenCalled() + expect(dependencies.fileSystem.readFile).not.toHaveBeenCalled() + expect(dependencies.cacheManager.getHash).not.toHaveBeenCalled() + expect(dependencies.parser.parseFile).not.toHaveBeenCalled() + expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() + }) + + it("skips oversized files without reading them", async () => { + const dependencies = setup() + dependencies.fileSystem.stat.mockResolvedValue({ + type: FileType.File, + ctime: 0, + mtime: 0, + size: MAX_FILE_SIZE_BYTES + 1, + }) + expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ + path: filePath, + status: "skipped", + reason: "File is too large", + }) + expect(dependencies.fileSystem.readFile).not.toHaveBeenCalled() + expect(dependencies.cacheManager.getHash).not.toHaveBeenCalled() + expect(dependencies.parser.parseFile).not.toHaveBeenCalled() + expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() + }) + + it("skips unchanged content before parsing or embedding", async () => { + const dependencies = setup() + dependencies.cacheManager.getHash.mockReturnValue(hash) + expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ + path: filePath, + status: "skipped", + reason: "File has not changed", + }) + expect(dependencies.cacheManager.getHash).toHaveBeenCalledWith(filePath) + expect(dependencies.parser.parseFile).not.toHaveBeenCalled() + expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() + }) + + it("returns the hash and empty points when parsing produces no blocks", async () => { + const dependencies = setup() + dependencies.parser.parseFile.mockResolvedValue([]) + expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ + path: filePath, + status: "processed_for_batching", + newHash: hash, + pointsToUpsert: [], + }) + expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() + }) + + it("still parses and returns the hash without an embedder or gitignore", async () => { + const dependencies = setup() + expect( + await new FilePreparation({ ...dependencies, embedder: undefined, ignoreInstance: undefined }).prepareFile( + filePath, + ), + ).toEqual({ + path: filePath, + status: "processed_for_batching", + newHash: hash, + pointsToUpsert: [], + }) + expect(dependencies.parser.parseFile).toHaveBeenCalledWith(filePath, { content, fileHash: hash }) + }) + + it("preserves normalized paths, stable IDs and embedding order at the size limit", async () => { + const dependencies = setup() + dependencies.fileSystem.stat.mockResolvedValue({ + type: FileType.File, + ctime: 0, + mtime: 0, + size: MAX_FILE_SIZE_BYTES, + }) + dependencies.parser.parseFile.mockResolvedValue([ + { ...block, file_path: "src/../src/file.ts" }, + { ...block, start_line: 8, end_line: 10, content: "second" }, + ]) + dependencies.embedder.createEmbeddings.mockResolvedValue({ + embeddings: [ + [0.1, 0.2], + [0.3, 0.4], + ], + }) + expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ + path: filePath, + status: "processed_for_batching", + newHash: hash, + pointsToUpsert: [ + { + id: uuidv5(`${normalizedFilePath}:2`, QDRANT_CODE_BLOCK_NAMESPACE), + vector: [0.1, 0.2], + payload: { filePath: relativeFilePath, codeChunk: content, startLine: 2, endLine: 5 }, + }, + { + id: uuidv5(`${normalizedFilePath}:8`, QDRANT_CODE_BLOCK_NAMESPACE), + vector: [0.3, 0.4], + payload: { filePath: relativeFilePath, codeChunk: "second", startLine: 8, endLine: 10 }, + }, + ], + }) + expect(dependencies.fileSystem.stat).toHaveBeenCalledWith(Uri.file(filePath)) + expect(dependencies.fileSystem.readFile).toHaveBeenCalledWith(Uri.file(filePath)) + expect(dependencies.parser.parseFile).toHaveBeenCalledWith(filePath, { content, fileHash: hash }) + expect(dependencies.embedder.createEmbeddings).toHaveBeenCalledWith([content, "second"]) + }) + + it("calls service methods with their original receivers", async () => { + const dependencies = setup() + expect((await new FilePreparation(dependencies).prepareFile(filePath)).status).toBe("processed_for_batching") + expect(dependencies.ignoreController.validateAccess).toHaveBeenCalledOnce() + expect(dependencies.ignoreController.validateAccess.mock.contexts[0]).toBe(dependencies.ignoreController) + expect(dependencies.ignoreInstance.ignores).toHaveBeenCalledOnce() + expect(dependencies.ignoreInstance.ignores.mock.contexts[0]).toBe(dependencies.ignoreInstance) + expect(dependencies.fileSystem.stat).toHaveBeenCalledOnce() + expect(dependencies.fileSystem.stat.mock.contexts[0]).toBe(dependencies.fileSystem) + expect(dependencies.fileSystem.readFile).toHaveBeenCalledOnce() + expect(dependencies.fileSystem.readFile.mock.contexts[0]).toBe(dependencies.fileSystem) + expect(dependencies.cacheManager.getHash).toHaveBeenCalledOnce() + expect(dependencies.cacheManager.getHash.mock.contexts[0]).toBe(dependencies.cacheManager) + expect(dependencies.parser.parseFile).toHaveBeenCalledOnce() + expect(dependencies.parser.parseFile.mock.contexts[0]).toBe(dependencies.parser) + expect(dependencies.embedder.createEmbeddings).toHaveBeenCalledOnce() + expect(dependencies.embedder.createEmbeddings.mock.contexts[0]).toBe(dependencies.embedder) + }) + + it("does not treat a hidden workspace ancestor as an excluded directory", async () => { + const dependencies = setup() + dependencies.workspacePath = "/.hidden/workspace" + expect((await new FilePreparation(dependencies).prepareFile("/.hidden/workspace/src/file.ts")).status).toBe( + "processed_for_batching", + ) + expect(dependencies.ignoreInstance.ignores).toHaveBeenCalledWith(relativeFilePath) + }) + + describe.each(["Buffer", "Uint8Array"] as const)("UTF-8 decoding from %s", (representation) => { + it.each([ + { name: "ASCII", bytes: [65, 66], expectedContent: "AB" }, + { + name: "Cyrillic, Spanish, CJK and emoji", + bytes: [...Buffer.from("Привет, español 中文 😀", "utf-8")], + expectedContent: "Привет, español 中文 😀", + }, + { name: "preserved BOM", bytes: [0xef, 0xbb, 0xbf, 65, 66], expectedContent: "\uFEFFAB" }, + { name: "empty content", bytes: [], expectedContent: "" }, + { + name: "invalid and truncated UTF-8 sequences", + bytes: [65, 0xc3, 0x28, 0xff, 0xe2, 0x82], + expectedContent: "A\uFFFD(\uFFFD\uFFFD", + }, + { + name: "sliced view with nonzero byteOffset", + bytes: [0x58, 65, 0xc3, 0xb1, 0x59], + expectedContent: "Añ", + sliced: true, + }, + ])("decodes and hashes $name, then skips the cached content", async ({ bytes, expectedContent, sliced }) => { + const dependencies = setup() + const backing = representation === "Buffer" ? Buffer.from(bytes) : new Uint8Array(bytes) + const fileContent = sliced ? backing.subarray(1, backing.length - 1) : backing + if (sliced) { + expect(fileContent.byteOffset).toBeGreaterThan(0) + expect(fileContent.byteLength).toBeLessThan(backing.byteLength) + } + dependencies.fileSystem.readFile.mockResolvedValue(fileContent) + dependencies.parser.parseFile.mockResolvedValue([]) + // Hash the literal expected text, not the input bytes or the production decoder's output. + const expectedHash = createHash("sha256").update(expectedContent).digest("hex") + const preparation = new FilePreparation(dependencies) + + expect(await preparation.prepareFile(filePath)).toEqual({ + path: filePath, + status: "processed_for_batching", + newHash: expectedHash, + pointsToUpsert: [], + }) + expect(dependencies.parser.parseFile).toHaveBeenCalledWith(filePath, { + content: expectedContent, + fileHash: expectedHash, + }) + + dependencies.parser.parseFile.mockClear() + dependencies.cacheManager.getHash.mockReturnValue(expectedHash) + expect(await preparation.prepareFile(filePath)).toEqual({ + path: filePath, + status: "skipped", + reason: "File has not changed", + }) + expect(dependencies.cacheManager.getHash).toHaveBeenCalledWith(filePath) + expect(dependencies.parser.parseFile).not.toHaveBeenCalled() + expect(dependencies.embedder.createEmbeddings).not.toHaveBeenCalled() + }) + }) + + it.each(["access", "ignore", "stat", "read", "cache", "parse", "embed"])( + "returns the original %s error as a local error", + async (stage) => { + const dependencies = setup() + const error = new Error(`${stage} failed`) + const fail = () => { + throw error + } + if (stage === "access") dependencies.ignoreController.validateAccess.mockImplementation(fail) + if (stage === "ignore") dependencies.ignoreInstance.ignores.mockImplementation(fail) + if (stage === "stat") dependencies.fileSystem.stat.mockRejectedValue(error) + if (stage === "read") dependencies.fileSystem.readFile.mockRejectedValue(error) + if (stage === "cache") dependencies.cacheManager.getHash.mockImplementation(fail) + if (stage === "parse") dependencies.parser.parseFile.mockRejectedValue(error) + if (stage === "embed") dependencies.embedder.createEmbeddings.mockRejectedValue(error) + const result = await new FilePreparation(dependencies).prepareFile(filePath) + expect(result).toEqual({ path: filePath, status: "local_error", error }) + expect(result.error).toBe(error) + }, + ) + + it("does not wrap non-Error rejections", async () => { + const dependencies = setup() + dependencies.fileSystem.readFile.mockRejectedValue("read failed") + expect(await new FilePreparation(dependencies).prepareFile(filePath)).toEqual({ + path: filePath, + status: "local_error", + error: "read failed", + }) + }) +}) diff --git a/src/services/code-index/processors/__tests__/file-watcher.spec.ts b/src/services/code-index/processors/__tests__/file-watcher.spec.ts index fc61e687bd..736cca0806 100644 --- a/src/services/code-index/processors/__tests__/file-watcher.spec.ts +++ b/src/services/code-index/processors/__tests__/file-watcher.spec.ts @@ -3,6 +3,7 @@ import * as vscode from "vscode" import { FileWatcher } from "../file-watcher" +import { FilePreparation } from "../file-preparation" import { clearAllMocks } from "../../../../test-utils/reset" @@ -182,6 +183,38 @@ describe("FileWatcher", () => { vi.useRealTimers() }) + it("reuses constructor-created preparation for public processFile without writing points or cache", async () => { + const prepare = vi.spyOn(FilePreparation.prototype, "prepareFile") + try { + const preparation = fileWatcher["filePreparation"] + expect(preparation).toBeInstanceOf(FilePreparation) + expect(preparation["dependencies"].fileSystem).toBe(vscode.workspace.fs) + expect(preparation["dependencies"].cacheManager).toBe(mockCacheManager) + expect(preparation["dependencies"].ignoreController).toBe(fileWatcher["ignoreController"]) + const path = "/mock/workspace/src/file.ts" + const result = await fileWatcher.processFile(path) + expect(prepare).toHaveBeenNthCalledWith(1, path) + expect(result).toBe(await prepare.mock.results[0].value) + expect(result.status).toBe("processed_for_batching") + expect(result.pointsToUpsert).toHaveLength(1) + expect(vscode.workspace.fs.stat).toHaveBeenCalledWith(vscode.Uri.file(path)) + expect(vscode.workspace.fs.readFile).toHaveBeenCalledWith(vscode.Uri.file(path)) + const secondPath = "/mock/workspace/src/second.ts" + const secondResult = await fileWatcher.processFile(secondPath) + expect(prepare).toHaveBeenCalledTimes(2) + expect(prepare).toHaveBeenNthCalledWith(2, secondPath) + expect(prepare.mock.contexts[0]).toBe(preparation) + expect(prepare.mock.contexts[1]).toBe(preparation) + expect(secondResult).toBe(await prepare.mock.results[1].value) + expect(secondResult.status).toBe("processed_for_batching") + expect(mockVectorStore.upsertPoints).not.toHaveBeenCalled() + expect(mockCacheManager.updateHash).not.toHaveBeenCalled() + expect(mockCacheManager.deleteHash).not.toHaveBeenCalled() + } finally { + prepare.mockRestore() + } + }) + describe("file filtering", () => { it("should ignore files in hidden directories on create events", async () => { // Initialize the file watcher diff --git a/src/services/code-index/processors/file-preparation-dependencies.ts b/src/services/code-index/processors/file-preparation-dependencies.ts new file mode 100644 index 0000000000..001da428d7 --- /dev/null +++ b/src/services/code-index/processors/file-preparation-dependencies.ts @@ -0,0 +1,15 @@ +import type { Ignore } from "ignore" +import type { FileSystem } from "vscode" +import type { RooIgnoreController } from "../../../core/ignore/RooIgnoreController" +import type { CacheManager } from "../cache-manager" +import type { ICodeParser, IEmbedder } from "../interfaces" + +export interface FilePreparationDependencies { + workspacePath: string + ignoreController: Pick + ignoreInstance?: Pick + fileSystem: Pick + cacheManager: Pick + parser: ICodeParser + embedder?: Pick +} diff --git a/src/services/code-index/processors/file-preparation.ts b/src/services/code-index/processors/file-preparation.ts new file mode 100644 index 0000000000..a99ff6b56a --- /dev/null +++ b/src/services/code-index/processors/file-preparation.ts @@ -0,0 +1,100 @@ +import { createHash } from "crypto" +import { Uri } from "vscode" +import { v5 as uuidv5 } from "uuid" +import type { CodeBlock, FileProcessingResult, PointStruct } from "../interfaces" +import type { FilePreparationDependencies } from "./file-preparation-dependencies" +import { MAX_FILE_SIZE_BYTES, QDRANT_CODE_BLOCK_NAMESPACE } from "../constants" +import { generateNormalizedAbsolutePath, generateRelativeFilePath } from "../shared/get-relative-path" +import { isPathInIgnoredDirectory } from "../../glob/ignore-utils" + +/** Prepares one file for batching without writing points or mutating the hash cache. */ +export class FilePreparation { + constructor(private readonly dependencies: FilePreparationDependencies) {} + + public async prepareFile(filePath: string): Promise { + const dependencies = this.dependencies + try { + const skipReason = await this.getSkipReason(filePath) + if (skipReason !== undefined) { + return this.skippedResult(filePath, skipReason) + } + + const fileContent = await dependencies.fileSystem.readFile(Uri.file(filePath)) + const content = Buffer.from(fileContent).toString("utf-8") + const newHash = createHash("sha256").update(content).digest("hex") + + if (dependencies.cacheManager.getHash(filePath) === newHash) { + return this.skippedResult(filePath, "File has not changed") + } + + const blocks = await dependencies.parser.parseFile(filePath, { content, fileHash: newHash }) + const pointsToUpsert = await this.preparePoints(blocks) + + return { + path: filePath, + status: "processed_for_batching", + newHash, + pointsToUpsert, + } + } catch (error) { + return { + path: filePath, + status: "local_error", + error: error as Error, + } + } + } + + private async getSkipReason(filePath: string): Promise { + const dependencies = this.dependencies + // Use relative paths so ignored directories outside the workspace do not exclude the file. + const relativeFilePath = generateRelativeFilePath(filePath, dependencies.workspacePath) + if (isPathInIgnoredDirectory(relativeFilePath)) { + return "File is in an ignored directory" + } + + if ( + !dependencies.ignoreController.validateAccess(filePath) || + dependencies.ignoreInstance?.ignores(relativeFilePath) + ) { + return "File is ignored by .rooignore or .gitignore" + } + + const fileStat = await dependencies.fileSystem.stat(Uri.file(filePath)) + if (fileStat.size > MAX_FILE_SIZE_BYTES) { + return "File is too large" + } + return undefined + } + + private skippedResult(filePath: string, reason: string): FileProcessingResult { + return { path: filePath, status: "skipped", reason } + } + + private async preparePoints(blocks: CodeBlock[]): Promise { + const dependencies = this.dependencies + if (!dependencies.embedder || blocks.length === 0) { + return [] + } + + const texts = blocks.map((block) => block.content) + const { embeddings } = await dependencies.embedder.createEmbeddings(texts) + + return blocks.map((block, index) => { + const normalizedAbsolutePath = generateNormalizedAbsolutePath(block.file_path, dependencies.workspacePath) + const stableName = `${normalizedAbsolutePath}:${block.start_line}` + const pointId = uuidv5(stableName, QDRANT_CODE_BLOCK_NAMESPACE) + + return { + id: pointId, + vector: embeddings[index], + payload: { + filePath: generateRelativeFilePath(normalizedAbsolutePath, dependencies.workspacePath), + codeChunk: block.content, + startLine: block.start_line, + endLine: block.end_line, + }, + } + }) + } +} diff --git a/src/services/code-index/processors/file-watcher.ts b/src/services/code-index/processors/file-watcher.ts index a6a3122c36..933947bbe2 100644 --- a/src/services/code-index/processors/file-watcher.ts +++ b/src/services/code-index/processors/file-watcher.ts @@ -1,14 +1,6 @@ import * as vscode from "vscode" -import { - QDRANT_CODE_BLOCK_NAMESPACE, - MAX_FILE_SIZE_BYTES, - BATCH_SEGMENT_THRESHOLD, - MAX_BATCH_RETRIES, - INITIAL_RETRY_DELAY_MS, -} from "../constants" -import { createHash } from "crypto" +import { BATCH_SEGMENT_THRESHOLD, MAX_BATCH_RETRIES, INITIAL_RETRY_DELAY_MS } from "../constants" import { RooIgnoreController } from "../../../core/ignore/RooIgnoreController" -import { v5 as uuidv5 } from "uuid" import { Ignore } from "ignore" import { scannerExtensions } from "../shared/supported-extensions" import { @@ -20,9 +12,8 @@ import { BatchProcessingSummary, } from "../interfaces" import { codeParser } from "./parser" +import { FilePreparation } from "./file-preparation" import { CacheManager } from "../cache-manager" -import { generateNormalizedAbsolutePath, generateRelativeFilePath } from "../shared/get-relative-path" -import { isPathInIgnoredDirectory } from "../../glob/ignore-utils" import { TelemetryService } from "@roo-code/telemetry" import { TelemetryEventName } from "@roo-code/types" import { sanitizeErrorMessage } from "../shared/validation-helpers" @@ -32,6 +23,7 @@ import { Package } from "../../../shared/package" * Implementation of the file watcher interface */ export class FileWatcher implements IFileWatcher { + private readonly filePreparation: FilePreparation private ignoreInstance?: Ignore private fileWatcher?: vscode.FileSystemWatcher private ignoreController: RooIgnoreController @@ -86,6 +78,15 @@ export class FileWatcher implements IFileWatcher { if (ignoreInstance) { this.ignoreInstance = ignoreInstance } + this.filePreparation = new FilePreparation({ + workspacePath: this.workspacePath, + ignoreController: this.ignoreController, + ignoreInstance: this.ignoreInstance, + fileSystem: vscode.workspace.fs, + cacheManager: this.cacheManager, + parser: codeParser, + embedder: this.embedder, + }) // Get the configurable batch size from VSCode settings, fallback to default // If not provided in constructor, try to get from VSCode settings if (batchSegmentThreshold !== undefined) { @@ -507,97 +508,6 @@ export class FileWatcher implements IFileWatcher { * @returns Promise resolving to processing result */ async processFile(filePath: string): Promise { - try { - // Get relative path for ignore checks - const relativeFilePath = generateRelativeFilePath(filePath, this.workspacePath) - - // Check if file is in an ignored directory - // Use relative path to avoid matching parent directories outside the workspace - if (isPathInIgnoredDirectory(relativeFilePath)) { - return { - path: filePath, - status: "skipped" as const, - reason: "File is in an ignored directory", - } - } - - // Check if file should be ignored - if ( - !this.ignoreController.validateAccess(filePath) || - (this.ignoreInstance && this.ignoreInstance.ignores(relativeFilePath)) - ) { - return { - path: filePath, - status: "skipped" as const, - reason: "File is ignored by .rooignore or .gitignore", - } - } - - // Check file size - const fileStat = await vscode.workspace.fs.stat(vscode.Uri.file(filePath)) - if (fileStat.size > MAX_FILE_SIZE_BYTES) { - return { - path: filePath, - status: "skipped" as const, - reason: "File is too large", - } - } - - // Read file content - const fileContent = await vscode.workspace.fs.readFile(vscode.Uri.file(filePath)) - const content = fileContent.toString() - - // Calculate hash - const newHash = createHash("sha256").update(content).digest("hex") - - // Check if file has changed - if (this.cacheManager.getHash(filePath) === newHash) { - return { - path: filePath, - status: "skipped" as const, - reason: "File has not changed", - } - } - - // Parse file - const blocks = await codeParser.parseFile(filePath, { content, fileHash: newHash }) - - // Prepare points for batch processing - let pointsToUpsert: PointStruct[] = [] - if (this.embedder && blocks.length > 0) { - const texts = blocks.map((block) => block.content) - const { embeddings } = await this.embedder.createEmbeddings(texts) - - pointsToUpsert = blocks.map((block, index) => { - const normalizedAbsolutePath = generateNormalizedAbsolutePath(block.file_path, this.workspacePath) - const stableName = `${normalizedAbsolutePath}:${block.start_line}` - const pointId = uuidv5(stableName, QDRANT_CODE_BLOCK_NAMESPACE) - - return { - id: pointId, - vector: embeddings[index], - payload: { - filePath: generateRelativeFilePath(normalizedAbsolutePath, this.workspacePath), - codeChunk: block.content, - startLine: block.start_line, - endLine: block.end_line, - }, - } - }) - } - - return { - path: filePath, - status: "processed_for_batching" as const, - newHash, - pointsToUpsert, - } - } catch (error) { - return { - path: filePath, - status: "local_error" as const, - error: error as Error, - } - } + return this.filePreparation.prepareFile(filePath) } }