feat: implement file locking mechanism and enhance extraction process

src/modules/extraction/engine/extract-sections.ts

@@ -10,6 +10,7 @@ import type { ProjectMetadata } from './extract-project-metadata'
 import type { ScannedFile } from './file-scan'
 import { initTreeSitterWithSelectedGrammars } from './grammar-loader'
 import persistPreparedFileSections from './persist-prepared-file-sections'
+import type { PreparedFile } from './prepare-file-sections'
 import prepareFileSections from './prepare-file-sections'
 import rebuildModuleArtifacts, {
     type ModuleArtifactKey,
@@ -19,6 +20,7 @@ import rebuildModuleArtifacts, {
 import {
     clearExtractedSections,
     clearExtractedSectionsForPaths,
+    recordExtractedSuppressionsForPaths,
 } from './section-cleanup'
 import TreeSitterEngine from './tree-sitter-engine'
 
@@ -95,13 +97,127 @@ export default async function extractSections(
         phase: 'database',
         status: 'progress',
     })
+    if (options.mode === 'changed') {
+        const preparedFiles: PreparedFile[] = []
+        await recordExtractedSuppressionsForPaths(affectedPaths)
+        if (files.length > 0) {
+            progress?.({
+                message: `Extracting ${files.length} files`,
+                phase: 'sections',
+                status: 'start',
+                total: files.length,
+            })
+            await yieldToProgressRenderer()
+        }
+
+        for (const [fileIndex, file] of files.entries()) {
+            const preparedFile = await prepareFileSections({
+                context,
+                engine,
+                file,
+            })
+            preparedFiles.push(preparedFile)
+
+            if (
+                preparedFile.parserEngine === 'tree_sitter' &&
+                preparedFile.parserStatus === 'ok'
+            ) {
+                parserUsedFiles += 1
+            } else if (
+                preparedFile.sections.some(section => section.kind === 'code')
+            ) {
+                parserFallbackFiles += 1
+            }
+
+            if (preparedFile.truncated) {
+                filesTruncatedBySectionLimit += 1
+            }
+
+            sectionCount += preparedFile.sections.length
+            progress?.({
+                current: fileIndex + 1,
+                message: `Extracted ${file.path} (${preparedFile.sections.length} sections)`,
+                path: file.path,
+                phase: 'sections',
+                sectionCount: sectionCount,
+                status: 'progress',
+                total: files.length,
+            })
+            await yieldToProgressRenderer()
+        }
+
+        if (files.length > 0) {
+            progress?.({
+                current: files.length,
+                message: `Extracted ${sectionCount} sections`,
+                parserCount: loadedParserCount,
+                phase: 'sections',
+                sectionCount: sectionCount,
+                status: 'done',
+                total: files.length,
+            })
+        }
+        progress?.({
+            message: 'Rebuilding module artifacts and retrieval index',
+            phase: 'modules',
+            status: 'start',
+        })
+
+        await withTransaction(async () => {
+            await clearExtractedSectionsForPaths(affectedPaths)
+            const rootNode = await upsertNode({
+                name: 'Project Files',
+                summary: 'Files extracted from the current project.',
+            })
+            rootNodeId = rootNode.id
+
+            for (const preparedFile of preparedFiles) {
+                if (preparedFile.sections.length > 0) {
+                    await persistPreparedFileSections({
+                        extractedAt,
+                        preparedFile,
+                        rootNodeId,
+                    })
+                }
+            }
+
+            const currentModuleKeys =
+                await queryModuleArtifactKeysForPaths(changedPaths)
+            await rebuildChangedModuleArtifacts(extractedAt, {
+                canUpdateEmbeddings: options.canUpdateEmbeddings ?? false,
+                metadata: options.metadata,
+                metadataChanged: hasMetadataRelevantPath(affectedPaths),
+                moduleKeys: dedupeModuleKeys([
+                    ...previousModuleKeys,
+                    ...currentModuleKeys,
+                ]),
+            })
+
+            await appendMemoryEvent({
+                actor: 'cli',
+                eventType: 'project_extracted',
+                id: `event_${contentHash(`${context.projectRoot}:${extractedAt}`).slice(0, 32)}`,
+                subjectType: 'project',
+                summary: `Extracted ${files.length} files into ${sectionCount} sections.`,
+            })
+        })
+        progress?.({
+            message: 'Module artifacts and retrieval index ready',
+            phase: 'modules',
+            status: 'done',
+        })
+
+        return {
+            filesTruncatedBySectionLimit,
+            loadedParserCount,
+            parserFallbackFiles,
+            parserUsedFiles,
+            sectionCount: sectionCount,
+        }
+    }
+
     await withTransaction(async () => {
-        if (options.mode === 'changed') {
-            await clearExtractedSectionsForPaths([
-                ...files.map(file => file.path),
-                ...(options.deletedPaths ?? []),
-            ])
-        } else if (options.mode !== 'resume') {
+        if (options.mode !== 'resume') {
             await clearExtractedSections()
         }
         const rootNode = await upsertNode({
@@ -180,25 +296,9 @@ export default async function extractSections(
         phase: 'modules',
         status: 'start',
     })
-    const currentModuleKeys =
-        options.mode === 'changed'
-            ? await queryModuleArtifactKeysForPaths(changedPaths)
-            : []
     await withTransaction(async () => {
-        if (options.mode === 'changed') {
-            await rebuildChangedModuleArtifacts(extractedAt, {
-                canUpdateEmbeddings: options.canUpdateEmbeddings ?? false,
-                metadata: options.metadata,
-                metadataChanged: hasMetadataRelevantPath(affectedPaths),
-                moduleKeys: dedupeModuleKeys([
-                    ...previousModuleKeys,
-                    ...currentModuleKeys,
-                ]),
-            })
-        } else {
-            await rebuildModuleArtifacts(extractedAt, options.metadata)
-            await reindexRetrievalDocumentFts()
-        }
+        await rebuildModuleArtifacts(extractedAt, options.metadata)
+        await reindexRetrievalDocumentFts()
 
         await appendMemoryEvent({
             actor: 'cli',

src/modules/extraction/engine/manifest.ts

@@ -1,4 +1,5 @@
-import { readFile, writeFile } from 'node:fs/promises'
+import { randomUUID } from 'node:crypto'
+import { readFile, rename, rm, writeFile } from 'node:fs/promises'
 import { join } from 'node:path'
 import { pathExists } from '@/modules/project/context'
 import type { ExtractionMode, LegacyExtractionMode } from '@/types/extraction'
@@ -65,10 +66,19 @@ export async function writeExtractionManifest(
     memoryDir: string,
     manifest: ExtractionManifest,
 ): Promise<void> {
-    await writeFile(
-        extractionManifestPath(memoryDir),
-        `${JSON.stringify(manifest, null, 2)}\n`,
+    const targetPath = extractionManifestPath(memoryDir)
+    const tempPath = join(
+        memoryDir,
+        `.extraction-manifest.${process.pid}.${Date.now()}.${randomUUID()}.tmp`,
     )
+
+    try {
+        await writeFile(tempPath, `${JSON.stringify(manifest, null, 2)}\n`)
+        await rename(tempPath, targetPath)
+    } catch (error) {
+        await rm(tempPath, { force: true }).catch(() => undefined)
+        throw error
+    }
 }
 
 export async function getExtractionFreshness(

src/modules/extraction/engine/section-cleanup.ts

@@ -135,6 +135,17 @@ where target_type = 'section'
         )
 }
 
+export async function recordExtractedSuppressionsForPaths(
+    paths: string[],
+): Promise<void> {
+    const uniquePaths = [...new Set(paths)].filter(Boolean)
+    if (uniquePaths.length === 0) {
+        return
+    }
+
+    await recordExtractedSuppressions(uniquePaths)
+}
+
 async function recordExtractedSuppressions(paths?: string[]): Promise<void> {
     const db = await getDb()
     await db.run(sql`

src/modules/memory/runtime.ts

@@ -1,7 +1,9 @@
+import { join } from 'node:path'
 import type { SaveOptions } from '@/database/services/save-memory'
 import { readExtractionManifest } from '@/modules/extraction/engine/manifest'
 import { extractProject } from '@/modules/extraction/extract-project'
 import { loadProjectContext } from '@/modules/project/context'
+import { acquireFileLock } from '@/support/file-lock'
 import type { EmbeddingProviderContract } from '@/types/embedding-provider'
 import type { LoadedProjectContext } from '@/types/project'
 
@@ -20,11 +22,27 @@ export async function updateChangedProjectMemorySilently(
         return undefined
     }
 
-    const result = await extractProject(context, 'changed', {
-        embeddingProvider,
+    const lock = await acquireFileLock({
+        lockDir: join(
+            context.memoryDir,
+            'locks',
+            'changed-project-memory.lock',
+        ),
+        operationName: 'changed_project_memory',
     })
-    return {
-        deletedFilePaths: result.deletedFilePaths,
-        updatedFilePaths: result.updatedFilePaths,
+    if (!lock.acquired) {
+        return undefined
+    }
+
+    try {
+        const result = await extractProject(context, 'changed', {
+            embeddingProvider,
+        })
+        return {
+            deletedFilePaths: result.deletedFilePaths,
+            updatedFilePaths: result.updatedFilePaths,
+        }
+    } finally {
+        await lock.release()
     }
 }