diff --git a/.claude/incremental-resolution/01-design-invalidation-tracking.md b/.claude/incremental-resolution/01-design-invalidation-tracking.md new file mode 100644 index 0000000000..f56f299285 --- /dev/null +++ b/.claude/incremental-resolution/01-design-invalidation-tracking.md @@ -0,0 +1,209 @@ +# Incremental Resolution: Invalidation Tracking Design + +## Goal + +Capture invalidation dependencies on each `BundlerResolution`, so that when file system changes occur, we can efficiently determine which resolutions need to be re-run. + +## Invalidation Categories + +### 1. Path Existence (`existenceInvalidations`) +A resolution is invalidated if the existence of a specific file or directory path changes (added or removed). + +**Sources:** +- `fileSystemLookup(path)` — the resolver probes many paths (with various extensions, platforms, etc.) and the resolution depends on which ones exist and which don't +- `doesFileExist(path)` — deprecated but still used, same as above +- `hierarchicalLookup()` — walks up directory tree looking for package.json; already has `invalidatedBy` pattern +- `resolveAsset()` — probes for asset files at various resolutions + +**What to capture:** Every path passed to `fileSystemLookup` / `doesFileExist`, because: +- If a **missing** path is later **added**, the resolution might change +- If an **existing** path is later **removed**, the resolution might change + +The existing `LookupResult` from TreeFS already returns the real path (for existing) or first missing path segment (for non-existing). We capture the real path for existing lookups and the `missing` path for non-existing lookups. + +### 2. File Content (`contentInvalidations`) +A resolution is invalidated if the content of a specific file changes. + +**Sources:** +- `getPackage(packageJsonPath)` — reads and parses package.json; if its content changes (e.g. `main` field, `exports` map, `browser` field), the resolution may change +- `getPackageForModule(absolutePath)` — finds closest package.json and reads it + +**What to capture:** The absolute path of any package.json that was read during resolution. + +### 3. Haste Name (`hasteInvalidations`) +A resolution is invalidated if the Haste mapping for a specific name changes — i.e., if any file providing that Haste name is added, removed, or changes which file wins. + +**Sources:** +- `resolveHasteModule(name)` — looks up a Haste module name +- `resolveHastePackage(name)` — looks up a Haste package name + +**What to capture:** The Haste name that was looked up. This is a compact representation — the Haste map itself tracks which files provide which names, so we just need the name. + +## Data Structure + +```flow +type ResolutionInvalidations = Readonly<{ + // Paths whose existence (added/removed) would invalidate this resolution. + // Includes both paths that exist (removal invalidates) and paths that + // don't exist (addition invalidates). + pathExistence: ReadonlySet, + + // Paths whose content change would invalidate this resolution + // (e.g. package.json files that were read). + fileContent: ReadonlySet, + + // Haste names that were looked up. Any change to which file provides + // this name (or whether it's provided at all) invalidates. + hasteNames: ReadonlySet, +}>; +``` + +## Updated BundlerResolution + +```flow +export type BundlerResolution = Readonly<{ + type: 'sourceFile', + filePath: string, + unstable_invalidations?: ResolutionInvalidations, +}>; +``` + +The `unstable_invalidations` field is optional: +- Present when `unstable_incrementalResolution` is enabled +- Absent when using the traditional resolution cache (no tracking needed) + +## Context Wrapping Strategy + +In `ModuleResolution.resolveDependency()`, when `unstable_incrementalResolution` is true, we wrap the context methods to collect invalidation data: + +``` +// Pseudocode +const pathExistence = new Set(); +const fileContent = new Set(); +const hasteNames = new Set(); + +wrappedContext = { + ...context, + fileSystemLookup: (path) => { + const result = context.fileSystemLookup(path); + pathExistence.add(result.exists ? result.realPath : path); + return result; + }, + doesFileExist: (path) => { + const result = context.doesFileExist(path); + pathExistence.add(path); // Simplified - ideally we'd get the realPath + return result; + }, + getPackage: (pkgPath) => { + const result = context.getPackage(pkgPath); + if (result != null) { + fileContent.add(pkgPath); + } + return result; + }, + resolveHasteModule: (name) => { + hasteNames.add(name); + return context.resolveHasteModule(name); + }, + resolveHastePackage: (name) => { + hasteNames.add(name); + return context.resolveHastePackage(name); + }, +}; +``` + +## Key Design Decisions + +1. **No public API change**: `unstable_invalidations` is an internal field on `BundlerResolution`, gated behind the existing `unstable_incrementalResolution` flag. + +2. **Efficient wrapping**: Only wrap when `unstable_incrementalResolution` is true. The wrapping overhead is minimal (Set.add per call). + +3. **Use TreeFS return values**: `fileSystemLookup` returns `realPath` for existing paths and we can use the input path for non-existing. This correctly handles symlinks. + +4. **Compact Haste representation**: Instead of tracking all files that could provide a Haste name, we just track the name. The invalidation check is: "did any file with this Haste name get added/removed/modified?" + +5. **hierarchicalLookup already supports this**: TreeFS's `hierarchicalLookup` has an `invalidatedBy: Set` parameter. We'll pass a real Set instead of null when incremental resolution is enabled. + +## Invalidation Check (Future) + +Given a `ChangeEvent` with added/modified/removed files: +``` +shouldInvalidate(resolution, changes): + for path in changes.addedFiles ∪ changes.removedFiles: + if path ∈ resolution.pathExistence → INVALIDATE + for path in changes.modifiedFiles: + if path ∈ resolution.fileContent → INVALIDATE + for name in resolution.hasteNames: + if name ∈ changes.affectedHasteNames → INVALIDATE + return KEEP +``` + +## Implementation Order + +1. ✅ Add `ResolutionInvalidations` type to `types.js` +2. ✅ Add `unstable_invalidations` to `BundlerResolution` +3. ✅ Implement context wrapping in `ResolutionTracker` class (ModuleResolution.js) +4. ✅ Wire up `hierarchicalLookup` invalidatedBy through PackageCache → DependencyGraph +5. Build invalidation checking in `DependencyGraph._onHasteChange()` + +## Implementation Notes + +### hierarchicalLookup wiring +TreeFS's `hierarchicalLookup` API was split from a single `invalidatedBy` set +into two separate parameters: +- `invalidatedByExistence: ?Set` — paths whose addition/removal + would invalidate the result (via explicit `.add()` calls in TreeFS) +- `invalidatedByModification: ?Set` — symlinks traversed during the + lookup, whose modification (target change) would invalidate the result + (via `collectLinkPaths` in `#lookupByNormalPath`) + +This separation is important because symlink modifications are **file content +changes** (the symlink still exists, but its target changed), not existence +changes. A regular file could also become a symlink or vice-versa at the same +path, which is simply a "modified file" either way. + +The call chain is: +``` +ResolutionTracker.getPackageForModule(path) + → ModuleResolution._getPackageForModule(path, pathExistence, fileContent) + → PackageCache.getPackageOf(path, pathExistence, fileContent) + → DependencyGraph._getClosestPackage(path, pathExistence, fileContent) + → TreeFS.hierarchicalLookup(path, 'package.json', { + invalidatedByExistence: pathExistence, + invalidatedByModification: fileContent, + }) +``` + +### PackageCache caching correctness +`PackageCache` caches the result of `hierarchicalLookup`. To correctly support +invalidation tracking with caching: + +1. **Cache miss (tracked)**: Allocate **fresh** sets for `hierarchicalLookup`, + then copy the fresh paths into the caller's sets AND store them in the cache. + This avoids caching the caller's pre-existing invalidation paths. + +2. **Cache hit (stored paths exist)**: Replay stored paths into the caller's sets. + +3. **Cache hit (no stored paths)**: If the initial miss was from a non-tracking + caller, re-run `hierarchicalLookup` to collect the paths, store for future hits. + +### `resolveAsset` tracking +The `resolveAsset` closure in `DependencyGraph._createModuleResolver()` captures +the *unwrapped* `fileSystemLookup` - so calls from `resolveAsset` to +`fileSystemLookup` don't go through our tracking wrapper in `ModuleResolution`. +We handle this by explicitly tracking in `trackedResolveAsset`: +- For found assets: add each result path to `pathExistence` +- For not-found assets: add the base path to `pathExistence` + +### `doesFileExist` tracking +`doesFileExist` is deprecated in favor of `fileSystemLookup`, but still used. +We track the raw file path (not a real path, since `doesFileExist` returns only +a boolean). This is slightly less precise than `fileSystemLookup` tracking +(which gives us the realPath for existing files), but sufficient for invalidation. + +### Empty module resolution +`_getEmptyModule()` resolves the empty module path and caches it. Since the +empty module path is a config constant, it doesn't need invalidation tracking. +The `invalidations` field will be present on it when `unstable_incrementalResolution` +is enabled, but it will only contain the paths probed during resolution of +the empty module itself. diff --git a/packages/metro-file-map/src/__tests__/changes-between-starts-test.js b/packages/metro-file-map/src/__tests__/changes-between-starts-test.js new file mode 100644 index 0000000000..f9bfa3ec36 --- /dev/null +++ b/packages/metro-file-map/src/__tests__/changes-between-starts-test.js @@ -0,0 +1,344 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @format + * @oncall react_native + */ + +import type {CacheData, FileData, FileMetadata} from '../flow-types'; +import type FileMapT from '../index'; + +import * as path from 'path'; + +jest.useRealTimers(); + +type MockCrawlResult = { + changedFiles: FileData, + removedFiles: Set, + clocks: Map, +}; + +let mockCrawlResult: MockCrawlResult; + +jest.mock('../crawlers/node', () => ({ + __esModule: true, + default: jest.fn(() => Promise.resolve(mockCrawlResult)), +})); + +let FileMap: Class; +let mockCacheContent: ?CacheData = null; +let mockCacheManager: { + read: JestMockFn, Promise>, + write: JestMockFn, Promise>, + end: JestMockFn, Promise>, +}; + +const ROOT_DIR = path.join('/', 'project'); +const FRUITS_DIR = path.join(ROOT_DIR, 'fruits'); + +const DEFAULT_HEALTH_CHECK_CONFIG = { + enabled: false, + interval: 10000, + timeout: 1000, + filePrefix: '.metro-file-map-health-check', +}; + +function createFileMetadata( + mtime: number = 32, + size: number = 42, +): FileMetadata { + return [ + mtime, // H.MTIME + size, // H.SIZE + 0, // H.VISITED + null, // H.SHA1 + 0, // H.SYMLINK + ]; +} + +describe('FileMap crawler backend integration', () => { + beforeEach(() => { + jest.resetModules(); + + mockCacheContent = null; + mockCacheManager = { + read: jest.fn().mockImplementation(async () => mockCacheContent), + write: jest.fn().mockImplementation(async getSnapshot => { + mockCacheContent = getSnapshot(); + }), + end: jest.fn(), + }; + + ({default: FileMap} = require('../')); + + mockCrawlResult = { + changedFiles: new Map(), + removedFiles: new Set(), + clocks: new Map([['fruits', 'c:clock:1']]), + }; + }); + + afterEach(async () => { + mockCacheContent = null; + }); + + describe('Cold cache and warm cache with changes', () => { + test('creates a file map on cold cache with all files new, then handles changes on rebuild', async () => { + mockCrawlResult = { + changedFiles: new Map([ + [path.join('fruits', 'Apple.js'), createFileMetadata()], + [path.join('fruits', 'Banana.js'), createFileMetadata()], + [path.join('fruits', 'Cherry.js'), createFileMetadata()], + ]), + removedFiles: new Set(), + clocks: new Map([['fruits', 'c:clock:1']]), + }; + + // Configure FileMap with no plugins and computeSha1: false + // so files don't need to be visited/read + const fileMap1 = new FileMap({ + extensions: ['js'], + rootDir: ROOT_DIR, + roots: [FRUITS_DIR], + cacheManagerFactory: () => mockCacheManager, + healthCheck: DEFAULT_HEALTH_CHECK_CONFIG, + maxWorkers: 1, + resetCache: false, + retainAllFiles: false, + useWatchman: false, + computeSha1: false, + plugins: [], + }); + + const {fileSystem: fileSystem1} = await fileMap1.build(); + + expect(fileSystem1.exists(path.join('fruits', 'Apple.js'))).toBe(true); + expect(fileSystem1.exists(path.join('fruits', 'Banana.js'))).toBe(true); + expect(fileSystem1.exists(path.join('fruits', 'Cherry.js'))).toBe(true); + + const allFiles1 = fileSystem1.getAllFiles(); + expect(allFiles1).toHaveLength(3); + expect(allFiles1).toContain(path.join(ROOT_DIR, 'fruits', 'Apple.js')); + expect(allFiles1).toContain(path.join(ROOT_DIR, 'fruits', 'Banana.js')); + expect(allFiles1).toContain(path.join(ROOT_DIR, 'fruits', 'Cherry.js')); + + expect(mockCacheManager.write).toHaveBeenCalledTimes(1); + + await fileMap1.end(); + + // Second build: crawler reports changes (modified, added, removed files) + mockCrawlResult = { + changedFiles: new Map([ + [path.join('fruits', 'Banana.js'), createFileMetadata(100, 50)], + [path.join('fruits', 'Date.js'), createFileMetadata(100, 30)], + ]), + removedFiles: new Set([path.join('fruits', 'Cherry.js')]), + clocks: new Map([['fruits', 'c:clock:2']]), + }; + + const fileMap2 = new FileMap({ + extensions: ['js'], + rootDir: ROOT_DIR, + roots: [FRUITS_DIR], + cacheManagerFactory: () => mockCacheManager, + healthCheck: DEFAULT_HEALTH_CHECK_CONFIG, + maxWorkers: 1, + resetCache: false, + retainAllFiles: false, + useWatchman: false, + computeSha1: false, + plugins: [], + }); + + const {fileSystem: fileSystem2} = await fileMap2.build(); + + expect(fileSystem2.exists(path.join('fruits', 'Apple.js'))).toBe(true); + expect(fileSystem2.exists(path.join('fruits', 'Banana.js'))).toBe(true); + expect(fileSystem2.exists(path.join('fruits', 'Cherry.js'))).toBe(false); + expect(fileSystem2.exists(path.join('fruits', 'Date.js'))).toBe(true); + + const allFiles2 = fileSystem2.getAllFiles(); + expect(allFiles2).toHaveLength(3); + expect(allFiles2).toContain(path.join(ROOT_DIR, 'fruits', 'Apple.js')); + expect(allFiles2).toContain(path.join(ROOT_DIR, 'fruits', 'Banana.js')); + expect(allFiles2).toContain(path.join(ROOT_DIR, 'fruits', 'Date.js')); + expect(allFiles2).not.toContain( + path.join(ROOT_DIR, 'fruits', 'Cherry.js'), + ); + + const bananaLookup = fileSystem2.lookup( + path.join(ROOT_DIR, 'fruits', 'Banana.js'), + ); + expect(bananaLookup.exists).toBe(true); + if (bananaLookup.exists && bananaLookup.type === 'f') { + expect(bananaLookup.metadata[0]).toBe(100); + expect(bananaLookup.metadata[1]).toBe(50); + } + + const dateLookup = fileSystem2.lookup( + path.join(ROOT_DIR, 'fruits', 'Date.js'), + ); + expect(dateLookup.exists).toBe(true); + if (dateLookup.exists && dateLookup.type === 'f') { + expect(dateLookup.metadata[0]).toBe(100); + expect(dateLookup.metadata[1]).toBe(30); + } + + expect(mockCacheManager.read).toHaveBeenCalledTimes(2); + expect(mockCacheManager.write).toHaveBeenCalledTimes(2); + + await fileMap2.end(); + }); + + test('handles multiple file additions and removals in a single rebuild', async () => { + mockCrawlResult = { + changedFiles: new Map([ + [path.join('fruits', 'File1.js'), createFileMetadata()], + [path.join('fruits', 'File2.js'), createFileMetadata()], + [path.join('fruits', 'File3.js'), createFileMetadata()], + [path.join('fruits', 'File4.js'), createFileMetadata()], + ]), + removedFiles: new Set(), + clocks: new Map([['fruits', 'c:clock:1']]), + }; + + const fileMap1 = new FileMap({ + extensions: ['js'], + rootDir: ROOT_DIR, + roots: [FRUITS_DIR], + cacheManagerFactory: () => mockCacheManager, + healthCheck: DEFAULT_HEALTH_CHECK_CONFIG, + maxWorkers: 1, + resetCache: false, + retainAllFiles: false, + useWatchman: false, + computeSha1: false, + plugins: [], + }); + + const {fileSystem: fileSystem1} = await fileMap1.build(); + expect(fileSystem1.getAllFiles()).toHaveLength(4); + await fileMap1.end(); + + // Second build: remove 3 files, modify 1, add 2 + mockCrawlResult = { + changedFiles: new Map([ + [path.join('fruits', 'File2.js'), createFileMetadata(200, 100)], + [path.join('fruits', 'File5.js'), createFileMetadata(200, 50)], + [path.join('fruits', 'File6.js'), createFileMetadata(200, 60)], + ]), + removedFiles: new Set([ + path.join('fruits', 'File1.js'), + path.join('fruits', 'File3.js'), + path.join('fruits', 'File4.js'), + ]), + clocks: new Map([['fruits', 'c:clock:2']]), + }; + + const fileMap2 = new FileMap({ + extensions: ['js'], + rootDir: ROOT_DIR, + roots: [FRUITS_DIR], + cacheManagerFactory: () => mockCacheManager, + healthCheck: DEFAULT_HEALTH_CHECK_CONFIG, + maxWorkers: 1, + resetCache: false, + retainAllFiles: false, + useWatchman: false, + computeSha1: false, + plugins: [], + }); + + const {fileSystem: fileSystem2} = await fileMap2.build(); + + expect(fileSystem2.exists(path.join('fruits', 'File1.js'))).toBe(false); + expect(fileSystem2.exists(path.join('fruits', 'File2.js'))).toBe(true); + expect(fileSystem2.exists(path.join('fruits', 'File3.js'))).toBe(false); + expect(fileSystem2.exists(path.join('fruits', 'File4.js'))).toBe(false); + expect(fileSystem2.exists(path.join('fruits', 'File5.js'))).toBe(true); + expect(fileSystem2.exists(path.join('fruits', 'File6.js'))).toBe(true); + + const allFiles2 = fileSystem2.getAllFiles(); + expect(allFiles2).toHaveLength(3); + + await fileMap2.end(); + }); + + test('correctly updates FileSystem lookup results after changes', async () => { + mockCrawlResult = { + changedFiles: new Map([ + [path.join('fruits', 'Original.js'), createFileMetadata(50, 100)], + ]), + removedFiles: new Set(), + clocks: new Map([['fruits', 'c:clock:1']]), + }; + + const fileMap1 = new FileMap({ + extensions: ['js'], + rootDir: ROOT_DIR, + roots: [FRUITS_DIR], + cacheManagerFactory: () => mockCacheManager, + healthCheck: DEFAULT_HEALTH_CHECK_CONFIG, + maxWorkers: 1, + resetCache: false, + retainAllFiles: false, + useWatchman: false, + computeSha1: false, + plugins: [], + }); + + const {fileSystem: fileSystem1} = await fileMap1.build(); + + const lookup1 = fileSystem1.lookup( + path.join(ROOT_DIR, 'fruits', 'Original.js'), + ); + expect(lookup1.exists).toBe(true); + if (lookup1.exists && lookup1.type === 'f') { + expect(lookup1.metadata[0]).toBe(50); + expect(lookup1.metadata[1]).toBe(100); + } + + await fileMap1.end(); + + // Second build: crawler reports the file was modified with new metadata + mockCrawlResult = { + changedFiles: new Map([ + [path.join('fruits', 'Original.js'), createFileMetadata(999, 500)], + ]), + removedFiles: new Set(), + clocks: new Map([['fruits', 'c:clock:2']]), + }; + + const fileMap2 = new FileMap({ + extensions: ['js'], + rootDir: ROOT_DIR, + roots: [FRUITS_DIR], + cacheManagerFactory: () => mockCacheManager, + healthCheck: DEFAULT_HEALTH_CHECK_CONFIG, + maxWorkers: 1, + resetCache: false, + retainAllFiles: false, + useWatchman: false, + computeSha1: false, + plugins: [], + }); + + const {fileSystem: fileSystem2} = await fileMap2.build(); + + const lookup2 = fileSystem2.lookup( + path.join(ROOT_DIR, 'fruits', 'Original.js'), + ); + expect(lookup2.exists).toBe(true); + if (lookup2.exists && lookup2.type === 'f') { + expect(lookup2.metadata[0]).toBe(999); + expect(lookup2.metadata[1]).toBe(500); + } + + await fileMap2.end(); + }); + }); +}); diff --git a/packages/metro-file-map/src/__tests__/index-test.js b/packages/metro-file-map/src/__tests__/index-test.js index 4475df9068..30bf604d30 100644 --- a/packages/metro-file-map/src/__tests__/index-test.js +++ b/packages/metro-file-map/src/__tests__/index-test.js @@ -12,6 +12,8 @@ import type {InputOptions} from '..'; import type { BuildResult, + CanonicalPath, + ChangedFileMetadata, ChangeEvent, ChangeEventMetadata, FileData, @@ -19,6 +21,7 @@ import type { FileSystem, HasteMap, MockMap, + ReadonlyFileSystemChanges, WatcherBackendOptions, WorkerSetupArgs, } from '../flow-types'; @@ -1448,7 +1451,10 @@ describe('FileMap', () => { test('distributes work across workers', async () => { const jestWorker = require('jest-worker').Worker; const path = require('path'); - const dependencyExtractor = path.join(__dirname, 'dependencyExtractor.js'); + const dependencyExtractor = path.resolve( + __dirname, + '../plugins/dependencies/__tests__/mockDependencyExtractor.js', + ); await buildNewFileMap( { maxWorkers: 4, @@ -1634,14 +1640,59 @@ describe('FileMap', () => { }); describe('file system changes processing', () => { - function waitForItToChange( - fileMap: FileMap, - ): Promise<{eventsQueue: ChangeEvent}> { + function waitForItToChange(fileMap: FileMap): Promise { return new Promise(resolve => { fileMap.once('change', resolve); }); } + type ChangeEntry = [CanonicalPath, ChangedFileMetadata]; + + function expectChanges( + changes: ReadonlyFileSystemChanges, + expected: Readonly<{ + addedFiles?: ReadonlyArray, + modifiedFiles?: ReadonlyArray, + removedFiles?: ReadonlyArray, + addedDirectories?: ReadonlyArray, + removedDirectories?: ReadonlyArray, + }>, + ): void { + const sortByPath = (a: ChangeEntry, b: ChangeEntry): number => + a[0].localeCompare(b[0]); + + const toSortedArray = ( + iterable: Iterable>, + ): Array => + [...iterable].map(([p, m]): ChangeEntry => [p, m]).sort(sortByPath); + + expect(toSortedArray(changes.addedFiles)).toEqual( + (expected.addedFiles ?? []).slice().sort(sortByPath), + ); + expect(toSortedArray(changes.modifiedFiles)).toEqual( + (expected.modifiedFiles ?? []).slice().sort(sortByPath), + ); + expect(toSortedArray(changes.removedFiles)).toEqual( + (expected.removedFiles ?? []).slice().sort(sortByPath), + ); + expect([...changes.addedDirectories].sort()).toEqual( + (expected.addedDirectories ?? []).slice().sort(), + ); + expect([...changes.removedDirectories].sort()).toEqual( + (expected.removedDirectories ?? []).slice().sort(), + ); + } + + function countFileChanges( + changes: ReadonlyFileSystemChanges, + ): number { + return ( + [...changes.addedFiles].length + + [...changes.modifiedFiles].length + + [...changes.removedFiles].length + ); + } + function mockDeleteFile(root: string, relativePath: string) { const e = mockEmitters[root]; e.emitFileEvent({event: 'delete', relativePath}); @@ -1693,18 +1744,16 @@ describe('FileMap', () => { expect(hasteMap.getModule('Banana')).toBe(filePath); mockDeleteFile(path.join('/', 'project', 'fruits'), 'Banana.js'); mockDeleteFile(path.join('/', 'project', 'fruits'), 'Banana.js'); - const {eventsQueue} = await waitForItToChange(fileMap); - expect(eventsQueue).toHaveLength(1); - const deletedBanana = { - filePath, - metadata: { - modifiedTime: null, - size: null, - type: 'f', - }, - type: 'delete', - }; - expect(eventsQueue).toEqual([deletedBanana]); + const {changes} = await waitForItToChange(fileMap); + expect(countFileChanges(changes)).toBe(1); + expectChanges(changes, { + removedFiles: [ + [ + path.join('fruits', 'Banana.js'), + {isSymlink: false, modifiedTime: 32}, + ], + ], + }); // Verify that the initial result has been updated expect(fileSystem.exists(filePath)).toBe(false); expect(hasteMap.getModuleNameByPath(filePath)).toBeNull(); @@ -1718,24 +1767,12 @@ describe('FileMap', () => { size: 55, }; - const MOCK_DELETE_FILE: ChangeEventMetadata = { - type: 'f', - modifiedTime: null, - size: null, - }; - const MOCK_CHANGE_LINK: ChangeEventMetadata = { type: 'l', modifiedTime: 46, size: 5, }; - const MOCK_DELETE_LINK: ChangeEventMetadata = { - type: 'l', - modifiedTime: null, - size: null, - }; - const MOCK_CHANGE_FOLDER: ChangeEventMetadata = { type: 'd', modifiedTime: 45, @@ -1758,24 +1795,35 @@ describe('FileMap', () => { relativePath: 'Tomato.js', metadata: MOCK_CHANGE_FILE, }); + e.emitFileEvent({ + event: 'touch', + relativePath: 'Banana.js', + metadata: MOCK_CHANGE_FILE, + }); e.emitFileEvent({ event: 'touch', relativePath: 'Pear.js', metadata: MOCK_CHANGE_FILE, }); - const {eventsQueue} = await waitForItToChange(fileMap); - expect(eventsQueue).toEqual([ - { - filePath: path.join('/', 'project', 'fruits', 'Tomato.js'), - metadata: MOCK_CHANGE_FILE, - type: 'add', - }, - { - filePath: path.join('/', 'project', 'fruits', 'Pear.js'), - metadata: MOCK_CHANGE_FILE, - type: 'change', - }, - ]); + const {changes} = await waitForItToChange(fileMap); + expectChanges(changes, { + addedFiles: [ + [ + path.join('fruits', 'Tomato.js'), + {isSymlink: false, modifiedTime: 45}, + ], + ], + modifiedFiles: [ + [ + path.join('fruits', 'Banana.js'), + {isSymlink: false, modifiedTime: 45}, + ], + [ + path.join('fruits', 'Pear.js'), + {isSymlink: false, modifiedTime: 45}, + ], + ], + }); expect( fileSystem.exists(path.join('/', 'project', 'fruits', 'Tomato.js')), ).toBe(true); @@ -1801,8 +1849,8 @@ describe('FileMap', () => { relativePath: 'Tomato.js', metadata: MOCK_CHANGE_FILE, }); - const {eventsQueue} = await waitForItToChange(fileMap); - expect(eventsQueue).toHaveLength(1); + const {changes} = await waitForItToChange(fileMap); + expect(countFileChanges(changes)).toBe(1); }); fm_it( @@ -1847,8 +1895,8 @@ describe('FileMap', () => { expect(fileSystem.getSha1(bananaPath)).toBe(originalHash); expect(hasteMap.getModule('Banana')).toBe(bananaPath); - const {eventsQueue} = await waitForItToChange(fileMap); - expect(eventsQueue).toHaveLength(1); + const {changes} = await waitForItToChange(fileMap); + expect(countFileChanges(changes)).toBe(1); // After the 'change' event is emitted, we should have new data expect(fileSystem.linkStats(bananaPath)).toEqual({ @@ -1879,14 +1927,15 @@ describe('FileMap', () => { relativePath: 'LinkToStrawberry.js', metadata: MOCK_CHANGE_LINK, }); - const {eventsQueue} = await waitForItToChange(fileMap); - expect(eventsQueue).toEqual([ - { - filePath: path.join(fruitsRoot, 'Strawberry.js'), - metadata: MOCK_CHANGE_FILE, - type: 'change', - }, - ]); + const {changes} = await waitForItToChange(fileMap); + expectChanges(changes, { + modifiedFiles: [ + [ + path.join('fruits', 'Strawberry.js'), + {isSymlink: false, modifiedTime: 45}, + ], + ], + }); expect( fileSystem.linkStats(path.join(fruitsRoot, 'LinkToStrawberry.js')), ).toBeNull(); @@ -1909,19 +1958,19 @@ describe('FileMap', () => { relativePath: 'LinkToStrawberry.js', metadata: MOCK_CHANGE_LINK, }); - const {eventsQueue} = await waitForItToChange(fileMap); - expect(eventsQueue).toEqual([ - { - filePath: path.join(fruitsRoot, 'Strawberry.js'), - metadata: MOCK_CHANGE_FILE, - type: 'change', - }, - { - filePath: path.join(fruitsRoot, 'LinkToStrawberry.js'), - metadata: MOCK_CHANGE_LINK, - type: 'change', - }, - ]); + const {changes} = await waitForItToChange(fileMap); + expectChanges(changes, { + modifiedFiles: [ + [ + path.join('fruits', 'LinkToStrawberry.js'), + {isSymlink: true, modifiedTime: 46}, + ], + [ + path.join('fruits', 'Strawberry.js'), + {isSymlink: false, modifiedTime: 45}, + ], + ], + }); expect( fileSystem.linkStats(path.join(fruitsRoot, 'LinkToStrawberry.js')), ).toEqual({fileType: 'l', modifiedTime: 46, size: 5}); @@ -1934,12 +1983,15 @@ describe('FileMap', () => { async ({fileMap}) => { const {fileSystem} = await fileMap.build(); const e = mockEmitters[path.join('/', 'project', 'fruits')]; + mockFs[ + path.join('/', 'project', 'fruits', 'node_modules', 'apple.js') + ] = ''; e.emitFileEvent({ event: 'touch', relativePath: path.join('node_modules', 'apple.js'), metadata: MOCK_CHANGE_FILE, }); - const {eventsQueue} = await waitForItToChange(fileMap); + const {changes} = await waitForItToChange(fileMap); const filePath = path.join( '/', 'project', @@ -1947,14 +1999,47 @@ describe('FileMap', () => { 'node_modules', 'apple.js', ); - expect(eventsQueue).toHaveLength(1); - expect(eventsQueue).toEqual([ - {filePath, metadata: MOCK_CHANGE_FILE, type: 'add'}, - ]); + expect(countFileChanges(changes)).toBe(1); + expectChanges(changes, { + addedFiles: [ + [ + path.join('fruits', 'node_modules', 'apple.js'), + {isSymlink: false, modifiedTime: 45}, + ], + ], + addedDirectories: [path.join('fruits', 'node_modules')], + }); expect(fileSystem.exists(filePath)).toBe(true); }, ); + fm_it( + 'emits directory removed when removing the last file from a directory', + async ({fileMap}) => { + await fileMap.build(); + const e = mockEmitters[path.join('/', 'project', 'fruits')]; + e.emitFileEvent({ + event: 'delete', + relativePath: 'lonely.js', + }); + const {changes} = await waitForItToChange(fileMap); + expectChanges(changes, { + removedFiles: [ + [ + path.join('fruits', 'lonely.js'), + {isSymlink: false, modifiedTime: 32}, + ], + ], + removedDirectories: [path.join('fruits')], + }); + }, + { + mockFs: { + [path.join('/', 'project', 'fruits', 'lonely.js')]: '// lonely', + }, + }, + ); + fm_it( 'does not emit changes for regular files with unwatched extensions', async ({fileMap}) => { @@ -1972,12 +2057,17 @@ describe('FileMap', () => { relativePath: 'Banana.unwatched', metadata: MOCK_CHANGE_FILE, }); - const {eventsQueue} = await waitForItToChange(fileMap); + const {changes} = await waitForItToChange(fileMap); const filePath = path.join('/', 'project', 'fruits', 'Banana.js'); - expect(eventsQueue).toHaveLength(1); - expect(eventsQueue).toEqual([ - {filePath, metadata: MOCK_CHANGE_FILE, type: 'change'}, - ]); + expect(countFileChanges(changes)).toBe(1); + expectChanges(changes, { + modifiedFiles: [ + [ + path.join('fruits', 'Banana.js'), + {isSymlink: false, modifiedTime: 45}, + ], + ], + }); expect(fileSystem.exists(filePath)).toBe(true); }, ); @@ -1997,12 +2087,17 @@ describe('FileMap', () => { event: 'delete', relativePath: 'Unknown.ext', }); - const {eventsQueue} = await waitForItToChange(fileMap); + const {changes} = await waitForItToChange(fileMap); const filePath = path.join('/', 'project', 'fruits', 'Banana.js'); - expect(eventsQueue).toHaveLength(1); - expect(eventsQueue).toEqual([ - {filePath, metadata: MOCK_DELETE_FILE, type: 'delete'}, - ]); + expect(countFileChanges(changes)).toBe(1); + expectChanges(changes, { + removedFiles: [ + [ + path.join('fruits', 'Banana.js'), + {isSymlink: false, modifiedTime: 32}, + ], + ], + }); expect(fileSystem.exists(filePath)).toBe(false); expect(console.warn).not.toHaveBeenCalled(); expect(console.error).not.toHaveBeenCalled(); @@ -2022,17 +2117,22 @@ describe('FileMap', () => { relativePath: 'LinkToStrawberry.ext', metadata: MOCK_CHANGE_LINK, }); - const {eventsQueue} = await waitForItToChange(fileMap); + const {changes} = await waitForItToChange(fileMap); const filePath = path.join( '/', 'project', 'fruits', 'LinkToStrawberry.ext', ); - expect(eventsQueue).toHaveLength(1); - expect(eventsQueue).toEqual([ - {filePath, metadata: MOCK_CHANGE_LINK, type: 'add'}, - ]); + expect(countFileChanges(changes)).toBe(1); + expectChanges(changes, { + addedFiles: [ + [ + path.join('fruits', 'LinkToStrawberry.ext'), + {isSymlink: true, modifiedTime: 46}, + ], + ], + }); const linkStats = fileSystem.linkStats(filePath); expect(linkStats).toEqual({ fileType: 'l', @@ -2074,12 +2174,17 @@ describe('FileMap', () => { event: 'delete', relativePath: 'LinkToStrawberry.js', }); - const {eventsQueue} = await waitForItToChange(fileMap); + const {changes} = await waitForItToChange(fileMap); - expect(eventsQueue).toHaveLength(1); - expect(eventsQueue).toEqual([ - {filePath: symlinkPath, metadata: MOCK_DELETE_LINK, type: 'delete'}, - ]); + expect(countFileChanges(changes)).toBe(1); + expectChanges(changes, { + removedFiles: [ + [ + path.join('fruits', 'LinkToStrawberry.js'), + {isSymlink: true, modifiedTime: 32}, + ], + ], + }); // Symlink is deleted without affecting the Haste module or real file. expect(fileSystem.exists(symlinkPath)).toBe(false); @@ -2107,20 +2212,20 @@ describe('FileMap', () => { relativePath: 'Orange.android.js', metadata: MOCK_CHANGE_FILE, }); - const {eventsQueue} = await waitForItToChange(fileMap); - expect(eventsQueue).toHaveLength(2); - expect(eventsQueue).toEqual([ - { - filePath: path.join('/', 'project', 'fruits', 'Orange.ios.js'), - metadata: MOCK_CHANGE_FILE, - type: 'change', - }, - { - filePath: path.join('/', 'project', 'fruits', 'Orange.android.js'), - metadata: MOCK_CHANGE_FILE, - type: 'change', - }, - ]); + const {changes} = await waitForItToChange(fileMap); + expect(countFileChanges(changes)).toBe(2); + expectChanges(changes, { + modifiedFiles: [ + [ + path.join('fruits', 'Orange.android.js'), + {isSymlink: false, modifiedTime: 45}, + ], + [ + path.join('fruits', 'Orange.ios.js'), + {isSymlink: false, modifiedTime: 45}, + ], + ], + }); expect( hasteMap.getModuleNameByPath( path.join('/', 'project', 'fruits', 'Orange.ios.js'), @@ -2174,25 +2279,28 @@ describe('FileMap', () => { metadata: MOCK_CHANGE_FILE, }); - const {eventsQueue} = await waitForItToChange(fileMap); + const {changes} = await waitForItToChange(fileMap); // No duplicate warnings or errors should be printed. expect(console.warn).not.toHaveBeenCalled(); expect(console.error).not.toHaveBeenCalled(); - expect(eventsQueue).toHaveLength(2); - expect(eventsQueue).toEqual([ - { - filePath: path.join('/', 'project', 'vegetables', 'Melon.js'), - metadata: MOCK_DELETE_FILE, - type: 'delete', - }, - { - filePath: path.join('/', 'project', 'fruits', 'Melon.js'), - metadata: MOCK_CHANGE_FILE, - type: 'add', - }, - ]); + expect(countFileChanges(changes)).toBe(2); + expectChanges(changes, { + addedFiles: [ + [ + path.join('fruits', 'Melon.js'), + {isSymlink: false, modifiedTime: 45}, + ], + ], + removedFiles: [ + [ + path.join('vegetables', 'Melon.js'), + {isSymlink: false, modifiedTime: 32}, + ], + ], + removedDirectories: [path.join('vegetables')], + }); expect(hasteMap.getModule('Melon')).toEqual(newPath); }, ); @@ -2369,8 +2477,8 @@ describe('FileMap', () => { relativePath: path.join('tomato.js', 'index.js'), metadata: MOCK_CHANGE_FILE, }); - const {eventsQueue} = await waitForItToChange(fileMap); - expect(eventsQueue).toHaveLength(1); + const {changes} = await waitForItToChange(fileMap); + expect(countFileChanges(changes)).toBe(1); }, ); }); diff --git a/packages/metro-file-map/src/cache/__tests__/DiskCacheManager-test.js b/packages/metro-file-map/src/cache/__tests__/DiskCacheManager-test.js index 7732a69a6c..a37b092593 100644 --- a/packages/metro-file-map/src/cache/__tests__/DiskCacheManager-test.js +++ b/packages/metro-file-map/src/cache/__tests__/DiskCacheManager-test.js @@ -105,7 +105,7 @@ describe('cacheManager', () => { let pluginCacheKey = 'foo'; const plugin: FileMapPlugin<> = { name: 'foo', - bulkUpdate() {}, + onChanged() {}, async initialize() {}, assertValid() {}, getSerializableSnapshot() { @@ -114,8 +114,6 @@ describe('cacheManager', () => { getWorker() { return null; }, - onNewOrModifiedFile() {}, - onRemovedFile() {}, getCacheKey() { return pluginCacheKey; }, diff --git a/packages/metro-file-map/src/flow-types.js b/packages/metro-file-map/src/flow-types.js index d40543f486..798bdae3e0 100644 --- a/packages/metro-file-map/src/flow-types.js +++ b/packages/metro-file-map/src/flow-types.js @@ -90,10 +90,29 @@ export type CacheManagerWriteOptions = Readonly<{ // - Real (no symlinks in path, though the path itself may be a symlink) export type CanonicalPath = string; -export type ChangeEvent = { +/** + * An object passed to TreeFS methods that captures file system observations + * relevant to incremental invalidation. TreeFS will populate the `existence` + * and `modification` sets with canonical (root-relative) paths. The `haste` + * set is not written by TreeFS but is included so that a single object can + * capture all invalidation data for a resolution. + */ +export type InvalidationData = Readonly<{ + existence: Set, + modification: Set, + haste: Set, +}>; + +export type ChangedFileMetadata = Readonly<{ + isSymlink: boolean, + modifiedTime?: ?number, +}>; + +export type ChangeEvent = Readonly<{ logger: ?RootPerfLogger, - eventsQueue: EventsQueue, -}; + changes: ReadonlyFileSystemChanges>, + rootDir: string, +}>; export type ChangeEventMetadata = { modifiedTime: ?number, // Epoch ms @@ -150,17 +169,6 @@ export type WatcherStatus = export type DuplicatesSet = Map; export type DuplicatesIndex = Map>; -export type EventsQueue = Array<{ - filePath: Path, - metadata: ChangeEventMetadata, - type: string, -}>; - -export type FileMapDelta = Readonly<{ - removed: Iterable<[CanonicalPath, T]>, - addedOrModified: Iterable<[CanonicalPath, T]>, -}>; - export type FileMapPluginInitOptions< SerializableState, PerFileData = void, @@ -213,10 +221,8 @@ export interface FileMapPlugin< initOptions: FileMapPluginInitOptions, ): Promise; assertValid(): void; - bulkUpdate(delta: FileMapDelta): void; + onChanged(changes: ReadonlyFileSystemChanges): void; getSerializableSnapshot(): SerializableState; - onRemovedFile(relativeFilePath: string, pluginData: ?PerFileData): void; - onNewOrModifiedFile(relativeFilePath: string, pluginData: ?PerFileData): void; getCacheKey(): string; getWorker(): ?FileMapPluginWorker; } @@ -292,8 +298,10 @@ export interface FileSystem { * X = dirname(X) * while X !== dirname(X) * - * If opts.invalidatedBy is given, collects all absolute, real paths that if - * added or removed may invalidate this result. + * If opts.invalidatedBy is given, collects canonical (root-relative) paths + * into its sets: + * - existence: paths whose addition or removal may invalidate this result + * - modification: symlinks traversed, whose target change may invalidate * * Useful for finding the closest package scope (subpath: package.json, * type f, breakOnSegment: node_modules) or closest potential package root @@ -304,7 +312,7 @@ export interface FileSystem { subpath: string, opts: { breakOnSegment: ?string, - invalidatedBy: ?Set, + invalidatedBy: ?InvalidationData, subpathType: 'f' | 'd', }, ): ?{ @@ -322,7 +330,7 @@ export interface FileSystem { * Return information about the given path, whether a directory or file. * Always follow symlinks, and return a real path if it exists. */ - lookup(mixedPath: Path): LookupResult; + lookup(mixedPath: Path, invalidatedBy?: InvalidationData): LookupResult; matchFiles(opts: { /* Filter relative paths against a pattern. */ @@ -356,16 +364,9 @@ export type LookupResult = // could indicate an unwatched path, or a directory containing no watched // files). exists: false, - // The real, normal, absolute paths of any symlinks traversed. - links: ReadonlySet, - // The real, normal, absolute path of the first path segment - // encountered that does not exist, or cannot be navigated through. - missing: string, } | { exists: true, - // The real, normal, absolute paths of any symlinks traversed. - links: ReadonlySet, // The real, normal, absolute path of the directory. realPath: string, // Currently lookup always follows symlinks, so can only return @@ -374,8 +375,6 @@ export type LookupResult = } | { exists: true, - // The real, normal, absolute paths of any symlinks traversed. - links: ReadonlySet, // The real, normal, absolute path of the file. realPath: string, // Currently lookup always follows symlinks, so can only return @@ -423,10 +422,39 @@ export type HasteMapItem = { }; export type HasteMapItemMetadata = [/* path */ string, /* type */ number]; +export interface FileSystemListener { + directoryAdded(canonicalPath: CanonicalPath): void; + directoryRemoved(canonicalPath: CanonicalPath): void; + + fileAdded(canonicalPath: CanonicalPath, data: FileMetadata): void; + fileModified( + canonicalPath: CanonicalPath, + oldData: FileMetadata, + newData: FileMetadata, + ): void; + fileRemoved(canonicalPath: CanonicalPath, data: FileMetadata): void; +} + +export interface ReadonlyFileSystemChanges<+T = FileMetadata> { + +addedDirectories: Iterable; + +removedDirectories: Iterable; + + +addedFiles: Iterable>; + +modifiedFiles: Iterable>; + +removedFiles: Iterable>; +} + export interface MutableFileSystem extends FileSystem { - remove(filePath: Path): ?FileMetadata; - addOrModify(filePath: Path, fileMetadata: FileMetadata): void; - bulkAddOrModify(addedOrModifiedFiles: FileData): void; + remove(filePath: Path, listener?: FileSystemListener): ?FileMetadata; + addOrModify( + filePath: Path, + fileMetadata: FileMetadata, + listener?: FileSystemListener, + ): void; + bulkAddOrModify( + addedOrModifiedFiles: FileData, + listener?: FileSystemListener, + ): void; } export type Path = string; diff --git a/packages/metro-file-map/src/index.js b/packages/metro-file-map/src/index.js index 2f1c6507c5..98f62d0c19 100644 --- a/packages/metro-file-map/src/index.js +++ b/packages/metro-file-map/src/index.js @@ -17,12 +17,12 @@ import type { CacheManagerFactory, CacheManagerFactoryOptions, CanonicalPath, + ChangedFileMetadata, ChangeEvent, ChangeEventClock, ChangeEventMetadata, Console, CrawlerOptions, - EventsQueue, FileData, FileMapPlugin, FileMapPluginWorker, @@ -31,6 +31,7 @@ import type { HasteMapData, HasteMapItem, HType, + InvalidationData, MutableFileSystem, Path, PerfLogger, @@ -44,6 +45,7 @@ import {DiskCacheManager} from './cache/DiskCacheManager'; import H from './constants'; import checkWatchmanCapabilities from './lib/checkWatchmanCapabilities'; import {FileProcessor} from './lib/FileProcessor'; +import {FileSystemChangeAggregator} from './lib/FileSystemChangeAggregator'; import normalizePathSeparatorsToPosix from './lib/normalizePathSeparatorsToPosix'; import normalizePathSeparatorsToSystem from './lib/normalizePathSeparatorsToSystem'; import {RootPathUtils} from './lib/RootPathUtils'; @@ -52,7 +54,6 @@ import {Watcher} from './Watcher'; import EventEmitter from 'events'; import {promises as fsPromises} from 'fs'; import invariant from 'invariant'; -import nullthrows from 'nullthrows'; import * as path from 'path'; import {performance} from 'perf_hooks'; @@ -69,6 +70,7 @@ export type { FileSystem, HasteMapData, HasteMapItem, + InvalidationData, }; export type InputOptions = Readonly<{ @@ -117,6 +119,19 @@ type IndexedPlugin = Readonly<{ plugin: AnyFileMapPlugin, dataIdx: ?number, }>; +type InternalEnqueuedEvent = Readonly< + | { + clock: ?ChangeEventClock, + relativeFilePath: string, + metadata: FileMetadata, + type: 'touch', + } + | { + clock: ?ChangeEventClock, + relativeFilePath: string, + type: 'delete', + }, +>; export {DiskCacheManager} from './cache/DiskCacheManager'; export {default as DependencyPlugin} from './plugins/DependencyPlugin'; @@ -421,7 +436,7 @@ export default class FileMap extends EventEmitter { }; }, fileIterator: opts => - mapIterator( + mapIterable( fileSystem.metadataIterator(opts), ({baseName, canonicalPath, metadata}) => ({ baseName, @@ -568,25 +583,20 @@ export default class FileMap extends EventEmitter { removedFiles: ReadonlySet, clocks?: WatchmanClocks, }>, - ): Promise { + ): Promise { this.#startupPerfLogger?.point('applyFileDelta_start'); const {changedFiles, removedFiles} = delta; this.#startupPerfLogger?.point('applyFileDelta_preprocess_start'); - const missingFiles: Set = new Set(); - // Remove files first so that we don't mistake moved modules // modules as duplicates. this.#startupPerfLogger?.point('applyFileDelta_remove_start'); - const removed: Array<[string, FileMetadata]> = []; + const changeAggregator = new FileSystemChangeAggregator(); for (const relativeFilePath of removedFiles) { - const metadata = fileSystem.remove(relativeFilePath); - if (metadata) { - removed.push([relativeFilePath, metadata]); - } + fileSystem.remove(relativeFilePath, changeAggregator); } this.#startupPerfLogger?.point('applyFileDelta_remove_end'); - const readLinkPromises = []; + const readLinkPromises: Array> = []; const readLinkErrors: Array<{ normalFilePath: string, error: Error & {code?: string}, @@ -606,9 +616,9 @@ export default class FileMap extends EventEmitter { const maybeReadLink = this.#maybeReadLink(normalFilePath, fileData); if (maybeReadLink) { readLinkPromises.push( - maybeReadLink.catch(error => - readLinkErrors.push({normalFilePath, error}), - ), + maybeReadLink.catch(error => { + readLinkErrors.push({normalFilePath, error}); + }), ); } } @@ -647,38 +657,32 @@ export default class FileMap extends EventEmitter { /* $FlowFixMe[incompatible-type] Error exposed after improved typing of * Array.{includes,indexOf,lastIndexOf} */ if (['ENOENT', 'EACCESS'].includes(error.code)) { - missingFiles.add(normalFilePath); + changedFiles.delete(normalFilePath); + fileSystem.remove(normalFilePath, changeAggregator); } else { // Anything else is fatal. throw error; } } - for (const relativeFilePath of missingFiles) { - changedFiles.delete(relativeFilePath); - const metadata = fileSystem.remove(relativeFilePath); - if (metadata) { - removed.push([relativeFilePath, metadata]); - } - } + this.#startupPerfLogger?.point('applyFileDelta_missing_end'); this.#startupPerfLogger?.point('applyFileDelta_add_start'); - fileSystem.bulkAddOrModify(changedFiles); + fileSystem.bulkAddOrModify(changedFiles, changeAggregator); this.#startupPerfLogger?.point('applyFileDelta_add_end'); this.#startupPerfLogger?.point('applyFileDelta_updatePlugins_start'); - plugins.forEach(({plugin, dataIdx}) => { - const mapFn: ([CanonicalPath, FileMetadata]) => [CanonicalPath, unknown] = - dataIdx != null - ? ([relativePath, fileData]) => [relativePath, fileData[dataIdx]] - : ([relativePath, fileData]) => [relativePath, null]; - plugin.bulkUpdate({ - addedOrModified: mapIterator(changedFiles.entries(), mapFn), - removed: mapIterator(removed.values(), mapFn), - }); + this.#plugins.forEach(({plugin, dataIdx}) => { + plugin.onChanged( + changeAggregator.getMappedView( + dataIdx != null ? metadata => metadata[dataIdx] : () => null, + ), + ); }); this.#startupPerfLogger?.point('applyFileDelta_updatePlugins_end'); this.#startupPerfLogger?.point('applyFileDelta_end'); + + return changeAggregator; } /** @@ -743,20 +747,68 @@ export default class FileMap extends EventEmitter { const hasWatchedExtension = (filePath: string) => this.#options.extensions.some(ext => filePath.endsWith(ext)); - let changeQueue: Promise = Promise.resolve(); let nextEmit: ?{ - eventsQueue: EventsQueue, + events: Array, firstEventTimestamp: number, firstEnqueuedTimestamp: number, } = null; const emitChange = () => { - if (nextEmit == null || nextEmit.eventsQueue.length === 0) { + if (nextEmit == null) { // Nothing to emit return; } - const {eventsQueue, firstEventTimestamp, firstEnqueuedTimestamp} = - nextEmit; + const {events, firstEventTimestamp, firstEnqueuedTimestamp} = nextEmit; + + const changeAggregator = new FileSystemChangeAggregator(); + + // Process a sequence of events. Note that preserving ordering is + // important here - a file may be both removed and added in the same + // batch. + // `changeAggregator` flattens this over time into the net change from + // this sequence. + for (const event of events) { + const {relativeFilePath, clock} = event; + if (event.type === 'delete') { + fileSystem.remove(relativeFilePath, changeAggregator); + } else { + fileSystem.addOrModify( + relativeFilePath, + event.metadata, + changeAggregator, + ); + } + this.#updateClock(clocks, clock); + } + + const changeSize = changeAggregator.getSize(); + + if (changeSize === 0) { + // We had events, but they've exactly cancelled each other out, reset + // so that timers are correct for the next change. + nextEmit = null; + return; + } + + const _netChange = changeAggregator.getView(); + this.#plugins.forEach(({plugin, dataIdx}) => { + plugin.onChanged( + changeAggregator.getMappedView( + dataIdx != null ? metadata => metadata[dataIdx] : () => null, + ), + ); + }); + + const toPublicMetadata = ( + metadata: Readonly, + ): ChangedFileMetadata => ({ + isSymlink: metadata[H.SYMLINK] !== 0, + modifiedTime: metadata[H.MTIME] ?? null, + }); + + const changesWithMetadata = + changeAggregator.getMappedView(toPublicMetadata); + const hmrPerfLogger = this.#options.perfLoggerFactory?.('HMR', { key: this.#getNextChangeID(), }); @@ -766,19 +818,20 @@ export default class FileMap extends EventEmitter { timestamp: firstEnqueuedTimestamp, }); hmrPerfLogger.point('waitingForChangeInterval_end'); - hmrPerfLogger.annotate({ - int: {eventsQueueLength: eventsQueue.length}, - }); + hmrPerfLogger.annotate({int: {changeSize}}); hmrPerfLogger.point('fileChange_start'); } const changeEvent: ChangeEvent = { - eventsQueue, + changes: changesWithMetadata, logger: hmrPerfLogger, + rootDir: this.#options.rootDir, }; this.emit('change', changeEvent); nextEmit = null; }; + let changeQueue: Promise = Promise.resolve(); + const onChange = (change: WatcherBackendChangeEvent) => { if ( change.metadata && @@ -806,73 +859,38 @@ export default class FileMap extends EventEmitter { const relativeFilePath = this.#pathUtils.absoluteToNormal(absoluteFilePath); - const linkStats = fileSystem.linkStats(relativeFilePath); - - // The file has been accessed, not modified. If the modified time is - // null, then it is assumed that the watcher does not have capabilities - // to detect modified time, and change processing proceeds. - if ( - change.event === 'touch' && - linkStats != null && - change.metadata.modifiedTime != null && - linkStats.modifiedTime === change.metadata.modifiedTime - ) { - return; - } - - // Emitted events, unlike memoryless backend events, specify 'add' or - // 'change' instead of 'touch'. - const eventTypeToEmit = - change.event === 'touch' - ? linkStats == null - ? 'add' - : 'change' - : 'delete'; const onChangeStartTime = performance.timeOrigin + performance.now(); + const enqueueEvent = (event: InternalEnqueuedEvent) => { + nextEmit ??= { + events: [], + firstEnqueuedTimestamp: performance.timeOrigin + performance.now(), + firstEventTimestamp: onChangeStartTime, + }; + nextEmit.events.push(event); + }; + changeQueue = changeQueue .then(async () => { // If we get duplicate events for the same file, ignore them. if ( nextEmit != null && - nextEmit.eventsQueue.find( + nextEmit.events.find( event => - event.type === eventTypeToEmit && - event.filePath === absoluteFilePath && + event.type === change.event && + event.relativeFilePath === relativeFilePath && ((!event.metadata && !change.metadata) || (event.metadata && change.metadata && - event.metadata.modifiedTime != null && + event.metadata[H.MTIME] != null && change.metadata.modifiedTime != null && - event.metadata.modifiedTime === - change.metadata.modifiedTime)), + event.metadata[H.MTIME] === change.metadata.modifiedTime)), ) ) { return null; } - const linkStats = fileSystem.linkStats(relativeFilePath); - - const enqueueEvent = (metadata: ChangeEventMetadata) => { - const event = { - filePath: absoluteFilePath, - metadata, - type: eventTypeToEmit, - }; - if (nextEmit == null) { - nextEmit = { - eventsQueue: [event], - firstEnqueuedTimestamp: - performance.timeOrigin + performance.now(), - firstEventTimestamp: onChangeStartTime, - }; - } else { - nextEmit.eventsQueue.push(event); - } - return null; - }; - // If the file was added or modified, // parse it and update the file map. if (change.event === 'touch') { @@ -902,17 +920,12 @@ export default class FileMap extends EventEmitter { }, ); } - fileSystem.addOrModify(relativeFilePath, fileMetadata); - this.#updateClock(clocks, change.clock); - plugins.forEach(({plugin, dataIdx}) => - dataIdx != null - ? plugin.onNewOrModifiedFile( - relativeFilePath, - fileMetadata[dataIdx], - ) - : plugin.onNewOrModifiedFile(relativeFilePath), - ); - enqueueEvent(change.metadata); + enqueueEvent({ + clock: change.clock, + relativeFilePath, + metadata: fileMetadata, + type: change.event, + }); } catch (e) { if (!['ENOENT', 'EACCESS'].includes(e.code)) { throw e; @@ -925,25 +938,10 @@ export default class FileMap extends EventEmitter { // point. } } else if (change.event === 'delete') { - if (linkStats == null) { - // Don't emit deletion events for files we weren't retaining. - // This is expected for deletion of an ignored file. - return null; - } - // We've already checked linkStats != null above, so the file - // exists in the file map and remove should always return metadata. - const metadata = nullthrows(fileSystem.remove(relativeFilePath)); - this.#updateClock(clocks, change.clock); - plugins.forEach(({plugin, dataIdx}) => - dataIdx != null - ? plugin.onRemovedFile(relativeFilePath, metadata[dataIdx]) - : plugin.onRemovedFile(relativeFilePath), - ); - enqueueEvent({ - modifiedTime: null, - size: null, - type: linkStats.fileType, + clock: change.clock, + relativeFilePath, + type: 'delete', }); } else { throw new Error( @@ -1055,11 +1053,9 @@ export default class FileMap extends EventEmitter { } // TODO: Replace with it.map() from Node 22+ -const mapIterator: (Iterator, (T) => S) => Iterable = (it, fn) => - 'map' in it - ? it.map(fn) - : (function* mapped() { - for (const item of it) { - yield fn(item); - } - })(); +const mapIterable: (Iterable, (T) => S) => Iterator = (it, fn) => + (function* mapped() { + for (const item of it) { + yield fn(item); + } + })(); diff --git a/packages/metro-file-map/src/lib/FileSystemChangeAggregator.js b/packages/metro-file-map/src/lib/FileSystemChangeAggregator.js new file mode 100644 index 0000000000..0f00a49b7e --- /dev/null +++ b/packages/metro-file-map/src/lib/FileSystemChangeAggregator.js @@ -0,0 +1,143 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @format + * @oncall react_native + */ + +import type { + CanonicalPath, + FileMetadata, + FileSystemListener, + ReadonlyFileSystemChanges, +} from '../flow-types'; + +export class FileSystemChangeAggregator implements FileSystemListener { + // Mutually exclusive with removedDirectories + +#addedDirectories: Set = new Set(); + // Mutually exclusive with addedDirectories + +#removedDirectories: Set = new Set(); + + // Mutually exclusive with modified and removed files + +#addedFiles: Map = new Map(); + // Mutually exclusive with added and removed files + +#modifiedFiles: Map = new Map(); + // Mutually exclusive with added and modified files + +#removedFiles: Map = new Map(); + + // Removed files must be paired with the file's metadata the last time it was + // observable by consumers - ie, immediately *before* this batch. To report + // this accurately with minimal overhead, we'll note the current metadata of + // a file the first time it is modified or removed within a batch. If it is + // re-added, modified and removed again, we still have the initial metadata. + // This is particularly important if, say, a regular file is replaced by a + // symlink, or vice-versa. + +#initialMetadata: Map = new Map(); + + directoryAdded(canonicalPath: CanonicalPath): void { + // Only add to newDirectories if this directory wasn't previously removed + // (i.e., it's truly new). If it was removed and re-added, the net effect + // is no directory change. + if (!this.#removedDirectories.delete(canonicalPath)) { + this.#addedDirectories.add(canonicalPath); + } + } + + directoryRemoved(canonicalPath: CanonicalPath): void { + if (!this.#addedDirectories.delete(canonicalPath)) { + this.#removedDirectories.add(canonicalPath); + } + } + + fileAdded(canonicalPath: CanonicalPath, data: FileMetadata): void { + if (this.#removedFiles.delete(canonicalPath)) { + // File was removed then re-added in the same batch - treat as modification + this.#modifiedFiles.set(canonicalPath, data); + } else { + // New file + this.#addedFiles.set(canonicalPath, data); + } + } + + fileModified( + canonicalPath: CanonicalPath, + oldData: FileMetadata, + newData: FileMetadata, + ): void { + if (this.#addedFiles.has(canonicalPath)) { + // File did not exist before this batch. Further modification only + // updates metadata + this.#addedFiles.set(canonicalPath, newData); + } else { + if (!this.#initialMetadata.has(canonicalPath)) { + this.#initialMetadata.set(canonicalPath, oldData); + } + this.#modifiedFiles.set(canonicalPath, newData); + } + } + + fileRemoved(canonicalPath: CanonicalPath, data: FileMetadata): void { + // Check if this file was added in the same batch + if (!this.#addedFiles.delete(canonicalPath)) { + let initialData = this.#initialMetadata.get(canonicalPath); + if (!initialData) { + initialData = data; + this.#initialMetadata.set(canonicalPath, initialData); + } + + // File was not added in this batch, so add to removed with last metadata + this.#modifiedFiles.delete(canonicalPath); + this.#removedFiles.set(canonicalPath, initialData); + } + // else: File was added then removed in the same batch - no net change + } + + getSize(): number { + return ( + this.#addedDirectories.size + + this.#removedDirectories.size + + this.#addedFiles.size + + this.#modifiedFiles.size + + this.#removedFiles.size + ); + } + + getView(): ReadonlyFileSystemChanges { + return { + addedDirectories: this.#addedDirectories, + removedDirectories: this.#removedDirectories, + addedFiles: this.#addedFiles, + modifiedFiles: this.#modifiedFiles, + removedFiles: this.#removedFiles, + }; + } + + getMappedView( + metadataMapFn: (metadata: FileMetadata) => T, + ): ReadonlyFileSystemChanges { + return { + addedDirectories: this.#addedDirectories, + removedDirectories: this.#removedDirectories, + addedFiles: mapIterable(this.#addedFiles, metadataMapFn), + modifiedFiles: mapIterable(this.#modifiedFiles, metadataMapFn), + removedFiles: mapIterable(this.#removedFiles, metadataMapFn), + }; + } +} + +function mapIterable( + map: Map, + metadataMapFn: (metadata: FileMetadata) => T, +): Iterable> { + return { + *[Symbol.iterator](): Iterator> { + for (const [path, metadata] of map) { + yield [path, metadataMapFn(metadata)]; + } + }, + }; +} diff --git a/packages/metro-file-map/src/lib/TreeFS.js b/packages/metro-file-map/src/lib/TreeFS.js index aa3b7a2718..c2b107b672 100644 --- a/packages/metro-file-map/src/lib/TreeFS.js +++ b/packages/metro-file-map/src/lib/TreeFS.js @@ -13,6 +13,8 @@ import type { FileData, FileMetadata, FileStats, + FileSystemListener, + InvalidationData, LookupResult, MutableFileSystem, Path, @@ -250,25 +252,21 @@ export default class TreeFS implements MutableFileSystem { return result != null; } - lookup(mixedPath: Path): LookupResult { + lookup(mixedPath: Path, invalidatedBy?: ?InvalidationData): LookupResult { const normalPath = this.#normalizePath(mixedPath); - const links = new Set(); const result = this.#lookupByNormalPath(normalPath, { - collectLinkPaths: links, + invalidatedBy, followLeaf: true, }); if (!result.exists) { - const {canonicalMissingPath} = result; return { exists: false, - links, - missing: this.#pathUtils.normalToAbsolute(canonicalMissingPath), }; } const {canonicalPath, node} = result; const realPath = this.#pathUtils.normalToAbsolute(canonicalPath); if (isDirectory(node)) { - return {exists: true, links, realPath, type: 'd'}; + return {exists: true, realPath, type: 'd'}; } invariant( isRegularFile(node), @@ -276,7 +274,7 @@ export default class TreeFS implements MutableFileSystem { mixedPath, canonicalPath, ); - return {exists: true, links, realPath, type: 'f', metadata: node}; + return {exists: true, realPath, type: 'f', metadata: node}; } getAllFiles(): Array { @@ -378,11 +376,16 @@ export default class TreeFS implements MutableFileSystem { } } - addOrModify(mixedPath: Path, metadata: FileMetadata): void { + addOrModify( + mixedPath: Path, + metadata: FileMetadata, + changeListener?: FileSystemListener, + ): void { const normalPath = this.#normalizePath(mixedPath); // Walk the tree to find the *real* path of the parent node, creating // directories as we need. const parentDirNode = this.#lookupByNormalPath(path.dirname(normalPath), { + changeListener, makeDirectories: true, }); if (!parentDirNode.exists) { @@ -394,10 +397,13 @@ export default class TreeFS implements MutableFileSystem { const canonicalPath = this.#normalizePath( parentDirNode.canonicalPath + path.sep + path.basename(normalPath), ); - this.bulkAddOrModify(new Map([[canonicalPath, metadata]])); + this.bulkAddOrModify(new Map([[canonicalPath, metadata]]), changeListener); } - bulkAddOrModify(addedOrModifiedFiles: FileData): void { + bulkAddOrModify( + addedOrModifiedFiles: FileData, + changeListener?: FileSystemListener, + ): void { // Optimisation: Bulk FileData are typically clustered by directory, so we // optimise for that case by remembering the last directory we looked up. // Experiments with large result sets show this to be significantly (~30%) @@ -413,6 +419,7 @@ export default class TreeFS implements MutableFileSystem { if (directoryNode == null || dirname !== lastDir) { const lookup = this.#lookupByNormalPath(dirname, { + changeListener, followLeaf: false, makeDirectories: true, }); @@ -433,11 +440,26 @@ export default class TreeFS implements MutableFileSystem { lastDir = dirname; directoryNode = lookup.node; } + if (changeListener != null) { + const existingNode = directoryNode.get(basename); + if (existingNode != null) { + invariant( + !isDirectory(existingNode), + 'Detected addition or modification of file %s, but it is tracked as a non-empty directory', + normalPath, + ); + // File already exists - this is a modification + changeListener.fileModified(normalPath, existingNode, metadata); + } else { + // New file + changeListener.fileAdded(normalPath, metadata); + } + } directoryNode.set(basename, metadata); } } - remove(mixedPath: Path): ?FileMetadata { + remove(mixedPath: Path, changeListener?: FileSystemListener): ?FileMetadata { const normalPath = this.#normalizePath(mixedPath); const result = this.#lookupByNormalPath(normalPath, {followLeaf: false}); if (!result.exists) { @@ -451,6 +473,13 @@ export default class TreeFS implements MutableFileSystem { ); } if (parentNode != null) { + if (changeListener != null) { + if (isDirectory(node)) { + changeListener.directoryRemoved(canonicalPath); + } else { + changeListener.fileRemoved(canonicalPath, node); + } + } parentNode.delete(path.basename(canonicalPath)); if (parentNode.size === 0 && parentNode !== this.#rootNode) { // NB: This isn't the most efficient algorithm - in the case of @@ -458,7 +487,7 @@ export default class TreeFS implements MutableFileSystem { // that's not expected to be a case common enough to justify // implementation complexity, or slowing down more common uses of // _lookupByNormalPath. - this.remove(path.dirname(canonicalPath)); + this.remove(path.dirname(canonicalPath), changeListener); } } return isDirectory(node) ? null : node; @@ -488,9 +517,13 @@ export default class TreeFS implements MutableFileSystem { normalPath: string, segmentName: string, }>, - // Mutable Set into which absolute real paths of traversed symlinks will - // be added. Omit for performance if not needed. - collectLinkPaths?: ?Set, + // Mutable structure into which canonical paths of traversed symlinks + // will be added. Omit for performance if not needed. + invalidatedBy?: ?InvalidationData, + + // Low-level callbacks called on mutations of TreeFS data. + // Omit for performance if not needed. + changeListener?: FileSystemListener, // Like lstat vs stat, whether to follow a symlink at the basename of // the given path, or return the details of the symlink itself. @@ -541,7 +574,8 @@ export default class TreeFS implements MutableFileSystem { // null. let ancestorOfRootIdx: ?number = opts.start?.ancestorOfRootIdx ?? 0; - const collectAncestors = opts.collectAncestors; + const {collectAncestors, changeListener} = opts; + // Used only when collecting ancestors, to avoid double-counting nodes and // paths when traversing a symlink takes us back to rootNode and out again. // This tracks the first character of the first segment not already @@ -583,6 +617,12 @@ export default class TreeFS implements MutableFileSystem { } segmentNode = new Map(); if (opts.makeDirectories === true) { + if (changeListener != null) { + const canonicalPath = isLastSegment + ? targetNormalPath + : targetNormalPath.slice(0, fromIdx - 1); + changeListener.directoryAdded(canonicalPath); + } parentNode.set(segmentName, segmentNode); } } @@ -630,6 +670,9 @@ export default class TreeFS implements MutableFileSystem { : targetNormalPath.slice(0, fromIdx - 1); if (isRegularFile(segmentNode)) { + if (opts.invalidatedBy) { + opts.invalidatedBy.existence.add(currentPath); + } // Regular file in a directory path return { canonicalMissingPath: currentPath, @@ -643,10 +686,8 @@ export default class TreeFS implements MutableFileSystem { segmentNode, currentPath, ); - if (opts.collectLinkPaths) { - opts.collectLinkPaths.add( - this.#pathUtils.normalToAbsolute(currentPath), - ); + if (opts.invalidatedBy) { + opts.invalidatedBy.modification.add(currentPath); } const remainingTargetPath = isLastSegment @@ -761,8 +802,10 @@ export default class TreeFS implements MutableFileSystem { * X = dirname(X) * while X !== dirname(X) * - * If opts.invalidatedBy is given, collects all absolute, real paths that if - * added or removed may invalidate this result. + * If opts.invalidatedBy is given, collects canonical (root-relative) paths + * into its sets: + * - existence: paths whose addition or removal may invalidate this result + * - modification: symlinks traversed, whose target change may invalidate * * Useful for finding the closest package scope (subpath: package.json, * type f, breakOnSegment: node_modules) or closest potential package root @@ -773,7 +816,7 @@ export default class TreeFS implements MutableFileSystem { subpath: string, opts: { breakOnSegment: ?string, - invalidatedBy: ?Set, + invalidatedBy: ?InvalidationData, subpathType: 'f' | 'd', }, ): ?{ @@ -790,7 +833,7 @@ export default class TreeFS implements MutableFileSystem { const invalidatedBy = opts.invalidatedBy; const closestLookup = this.#lookupByNormalPath(normalPath, { collectAncestors: ancestorsOfInput, - collectLinkPaths: invalidatedBy, + invalidatedBy, }); if (closestLookup.exists && isDirectory(closestLookup.node)) { @@ -809,15 +852,13 @@ export default class TreeFS implements MutableFileSystem { } } else { if ( - invalidatedBy && + invalidatedBy != null && (!closestLookup.exists || !isDirectory(closestLookup.node)) ) { - invalidatedBy.add( - this.#pathUtils.normalToAbsolute( - closestLookup.exists - ? closestLookup.canonicalPath - : closestLookup.canonicalMissingPath, - ), + invalidatedBy.existence.add( + closestLookup.exists + ? closestLookup.canonicalPath + : closestLookup.canonicalMissingPath, ); } if ( @@ -954,7 +995,7 @@ export default class TreeFS implements MutableFileSystem { normalCandidatePath: string, subpath: string, subpathType: 'f' | 'd', - invalidatedBy: ?Set, + invalidatedBy: ?InvalidationData, start: ?{ ancestorOfRootIdx: ?number, node: DirectoryNode, @@ -965,7 +1006,7 @@ export default class TreeFS implements MutableFileSystem { this.#pathUtils.joinNormalToRelative(normalCandidatePath, subpath) .normalPath, { - collectLinkPaths: invalidatedBy, + invalidatedBy, }, ); if ( @@ -975,12 +1016,10 @@ export default class TreeFS implements MutableFileSystem { ) { return this.#pathUtils.normalToAbsolute(lookupResult.canonicalPath); } else if (invalidatedBy) { - invalidatedBy.add( - this.#pathUtils.normalToAbsolute( - lookupResult.exists - ? lookupResult.canonicalPath - : lookupResult.canonicalMissingPath, - ), + invalidatedBy.existence.add( + lookupResult.exists + ? lookupResult.canonicalPath + : lookupResult.canonicalMissingPath, ); } return null; diff --git a/packages/metro-file-map/src/lib/__tests__/FileSystemChangeAggregator-test.js b/packages/metro-file-map/src/lib/__tests__/FileSystemChangeAggregator-test.js new file mode 100644 index 0000000000..d5013807cd --- /dev/null +++ b/packages/metro-file-map/src/lib/__tests__/FileSystemChangeAggregator-test.js @@ -0,0 +1,89 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @format + * @oncall react_native + */ + +import type {FileMetadata} from '../../flow-types'; + +import {FileSystemChangeAggregator} from '../FileSystemChangeAggregator'; + +let aggregator: FileSystemChangeAggregator; + +beforeEach(() => { + aggregator = new FileSystemChangeAggregator(); +}); + +const FOO = 'foo.js'; + +test('removing, adding, modifying and removing a file records initial data', () => { + aggregator.fileRemoved(FOO, makeData(0)); + aggregator.fileAdded(FOO, makeData(1)); + aggregator.fileModified(FOO, makeData(1), makeData(2)); + aggregator.fileRemoved(FOO, makeData(2)); + const changes = getData(aggregator); + expect(changes.removedFiles.size).toBe(1); + expect(changes.removedFiles.get(FOO)).toEqual(makeData(0)); +}); + +test('modifying then removing a file records initial data', () => { + aggregator.fileModified(FOO, makeData(0), makeData(1)); + aggregator.fileRemoved(FOO, makeData(1)); + const changes = getData(aggregator); + expect(changes.removedFiles.size).toBe(1); + expect(changes.modifiedFiles.size).toBe(0); + expect(changes.removedFiles.get(FOO)).toEqual(makeData(0)); +}); + +test('adding, modifying then removing a file records empty changes', () => { + aggregator.fileAdded(FOO, makeData(0)); + aggregator.fileModified(FOO, makeData(0), makeData(1)); + aggregator.fileRemoved(FOO, makeData(1)); + const changes = getData(aggregator); + expect(changes.addedFiles.size).toBe(0); + expect(changes.modifiedFiles.size).toBe(0); + expect(changes.removedFiles.size).toBe(0); +}); + +afterEach(() => { + // assert mutual exclusivity + const changes = aggregator.getView(); + for (const dir of changes.addedDirectories) { + expect(changes.removedDirectories).not.toContain(dir); + } + for (const dir of changes.removedDirectories) { + expect(changes.addedDirectories).not.toContain(dir); + } + for (const file of changes.addedFiles) { + expect(changes.modifiedFiles).not.toContain(file); + expect(changes.removedFiles).not.toContain(file); + } + for (const file of changes.modifiedFiles) { + expect(changes.addedFiles).not.toContain(file); + expect(changes.removedFiles).not.toContain(file); + } + for (const file of changes.removedFiles) { + expect(changes.addedFiles).not.toContain(file); + expect(changes.modifiedFiles).not.toContain(file); + } +}); + +function makeData(mtime: number = 0): FileMetadata { + return [mtime, 1, 0, null, 0]; +} + +function getData(aggregator: FileSystemChangeAggregator) { + const view = aggregator.getView(); + return { + addedDirectories: new Set(view.addedDirectories), + removedDirectories: new Set(view.removedDirectories), + addedFiles: new Map(Array.from(view.addedFiles, ([k, v]) => [k, v])), + modifiedFiles: new Map(Array.from(view.modifiedFiles, ([k, v]) => [k, v])), + removedFiles: new Map(Array.from(view.removedFiles, ([k, v]) => [k, v])), + }; +} diff --git a/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js b/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js index 2309a3092f..9fd4045282 100644 --- a/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js +++ b/packages/metro-file-map/src/lib/__tests__/TreeFS-test.js @@ -9,7 +9,12 @@ * @oncall react_native */ -import type {CanonicalPath, FileData, FileMetadata} from '../../flow-types'; +import type { + CanonicalPath, + FileData, + FileMetadata, + FileSystemListener, +} from '../../flow-types'; import type TreeFSType from '../TreeFS'; import H from '../../constants'; @@ -361,132 +366,130 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { }); test.each([ - ['/A/B/C/a', '/A/B/C/a/package.json', '', []], - ['/A/B/C/a/b', '/A/B/C/a/package.json', 'b', ['/A/B/C/a/b/package.json']], + ['/A/B/C/a', '/A/B/C/a/package.json', '', [], []], + ['/A/B/C/a/b', '/A/B/C/a/package.json', 'b', ['a/b/package.json'], []], [ '/A/B/C/a/package.json', '/A/B/C/a/package.json', 'package.json', - ['/A/B/C/a/package.json'], + ['a/package.json'], + [], ], [ '/A/B/C/a/b/notexists', '/A/B/C/a/package.json', 'b/notexists', - ['/A/B/C/a/b/notexists', '/A/B/C/a/b/package.json'], + ['a/b/notexists', 'a/b/package.json'], + [], ], - ['/A/B/C/a/b/c', '/A/B/C/a/b/c/package.json', '', []], + ['/A/B/C/a/b/c', '/A/B/C/a/b/c/package.json', '', [], []], [ '/A/B/C/other', '/A/package.json', 'B/C/other', - ['/A/B/C/other', '/A/B/C/package.json', '/A/B/package.json'], + ['other', 'package.json', '../package.json'], + [], ], [ '/A/B/C', '/A/package.json', 'B/C', - ['/A/B/C/package.json', '/A/B/package.json'], + ['package.json', '../package.json'], + [], ], - ['/A/B', '/A/package.json', 'B', ['/A/B/package.json']], + ['/A/B', '/A/package.json', 'B', ['../package.json'], []], [ '/A/B/foo', '/A/package.json', 'B/foo', - - ['/A/B/foo', '/A/B/package.json'], + ['../foo', '../package.json'], + [], ], - ['/A/foo', '/A/package.json', 'foo', ['/A/foo']], - ['/foo', null, null, ['/foo', '/package.json']], + ['/A/foo', '/A/package.json', 'foo', ['../../foo'], []], + ['/foo', null, null, ['../../../foo', '../../../package.json'], []], [ '/A/B/C/a/b/c/d/link-to-C/foo.js', '/A/B/C/a/b/c/package.json', 'd/link-to-C/foo.js', - [ - '/A/B/C/a/b/c/d/link-to-C', - '/A/B/C/a/b/c/d/package.json', - '/A/B/C/foo.js', - '/A/B/C/package.json', - ], + ['a/b/c/d/package.json', 'foo.js', 'package.json'], + ['a/b/c/d/link-to-C'], ], [ '/A/B/C/a/b/c/d/link-to-B/C/foo.js', '/A/B/C/a/b/c/package.json', 'd/link-to-B/C/foo.js', - [ - '/A/B/C/a/b/c/d/link-to-B', - '/A/B/C/a/b/c/d/package.json', - '/A/B/C/foo.js', - '/A/B/C/package.json', - '/A/B/package.json', - ], + ['a/b/c/d/package.json', 'foo.js', 'package.json', '../package.json'], + ['a/b/c/d/link-to-B'], ], [ '/A/B/C/a/b/c/d/link-to-A/B/C/foo.js', '/A/package.json', 'B/C/foo.js', - [ - '/A/B/C/a/b/c/d/link-to-A', - '/A/B/C/foo.js', - '/A/B/C/package.json', - '/A/B/package.json', - ], + ['foo.js', 'package.json', '../package.json'], + ['a/b/c/d/link-to-A'], ], [ '/A/B/C/a/1/foo.js', '/A/B/C/a/1/real-package.json', 'foo.js', - ['/A/B/C/a/1/foo.js', '/A/B/C/a/1/package.json'], + ['a/1/foo.js'], + ['a/1/package.json'], ], [ '/A/B/C/a/2/foo.js', '/A/B/C/a/package.json', '2/foo.js', - [ - '/A/B/C/a/2/foo.js', - '/A/B/C/a/2/notexist-package.json', - '/A/B/C/a/2/package.json', - ], + ['a/2/foo.js', 'a/2/notexist-package.json'], + ['a/2/package.json'], ], [ '/A/B/C/a/n_m/pkg/notexist.js', '/A/B/C/a/n_m/pkg/package.json', 'notexist.js', - ['/A/B/C/a/n_m/pkg/notexist.js'], + ['a/n_m/pkg/notexist.js'], + [], ], [ '/A/B/C/a/n_m/pkg/subpath/notexist.js', '/A/B/C/a/n_m/pkg/subpath/package.json', 'notexist.js', - ['/A/B/C/a/n_m/pkg/subpath/notexist.js'], + ['a/n_m/pkg/subpath/notexist.js'], + [], ], [ '/A/B/C/a/n_m/pkg/otherpath/notexist.js', '/A/B/C/a/n_m/pkg/package.json', 'otherpath/notexist.js', - ['/A/B/C/a/n_m/pkg/otherpath'], + ['a/n_m/pkg/otherpath'], + [], ], // pkg3 does not exist, doesn't look beyond the containing n_m - ['/A/B/C/a/n_m/pkg3/foo.js', null, null, ['/A/B/C/a/n_m/pkg3']], + ['/A/B/C/a/n_m/pkg3/foo.js', null, null, ['a/n_m/pkg3'], []], // Does not look beyond n_m, if n_m does not exist - ['/A/B/C/a/b/n_m/pkg/foo', null, null, ['/A/B/C/a/b/n_m']], + ['/A/B/C/a/b/n_m/pkg/foo', null, null, ['a/b/n_m'], []], [ '/A/B/C/n_m/workspace/link-to-pkg/subpath', '/A/B/workspace-pkg/package.json', 'subpath', - ['/A/B/C/n_m/workspace/link-to-pkg', '/A/B/workspace-pkg/subpath'], + ['../workspace-pkg/subpath'], + ['n_m/workspace/link-to-pkg'], ], ])( - '%s => %s (relative %s, invalidatedBy %s)', + '%s => %s (relative %s, existence %s, modification %s)', ( startPath, expectedPath, expectedRelativeSubpath, - expectedInvalidatedBy, + expectedExistence, + expectedModification, ) => { const pathMap = (normalPosixPath: string) => mockPathModule.resolve(p('/A/B/C'), p(normalPosixPath)); - const invalidatedBy = new Set(); + const invalidatedBy = { + existence: new Set(), + modification: new Set(), + haste: new Set(), + }; expect( tfs.hierarchicalLookup(p(startPath), 'package.json', { breakOnSegment: 'n_m', @@ -501,7 +504,12 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { containerRelativePath: p(expectedRelativeSubpath), }, ); - expect(invalidatedBy).toEqual(new Set(expectedInvalidatedBy.map(p))); + expect(invalidatedBy.existence).toEqual( + new Set(expectedExistence.map(p)), + ); + expect(invalidatedBy.modification).toEqual( + new Set(expectedModification.map(p)), + ); }, ); }); @@ -970,4 +978,163 @@ describe.each([['win32'], ['posix']])('TreeFS on %s', platform => { expect(mockProcessFile).toHaveBeenCalledTimes(2); }); }); + + describe('change listener', () => { + let simpleTfs: TreeFSType; + const logChange = jest.fn(); + const listener: FileSystemListener = { + fileAdded: (...args) => logChange('fileAdded', ...args), + fileModified: (...args) => logChange('fileModified', ...args), + fileRemoved: (...args) => logChange('fileRemoved', ...args), + directoryAdded: (...args) => logChange('directoryAdded', ...args), + directoryRemoved: (...args) => logChange('directoryRemoved', ...args), + }; + + beforeEach(() => { + logChange.mockClear(); + simpleTfs = new TreeFS({ + rootDir: p('/project'), + files: new Map([ + [p('existing.js'), [123, 0, 0, '', 0]], + [p('dir/nested.js'), [456, 0, 0, '', 0]], + [p('mylink'), [0, 0, 0, '', p('./dir')]], + ]), + processFile: () => { + throw new Error('Not implemented'); + }, + }); + }); + + describe('addOrModify with listener', () => { + test('tracks added files when adding a new file', () => { + simpleTfs.addOrModify(p('new.js'), [789, 0, 0, '', 0], listener); + + expect(logChange.mock.calls).toEqual([ + ['fileAdded', p('new.js'), [789, 0, 0, '', 0]], + ]); + }); + + test('tracks modified files when modifying an existing file', () => { + simpleTfs.addOrModify(p('existing.js'), [999, 0, 0, '', 0], listener); + + expect(logChange.mock.calls).toEqual([ + [ + 'fileModified', + p('existing.js'), + [123, 0, 0, '', 0], + [999, 0, 0, '', 0], + ], + ]); + }); + + test('tracks new directories when adding a file in a new directory', () => { + simpleTfs.addOrModify( + p('newdir/file.js'), + [123, 0, 0, '', '', 0, null], + listener, + ); + + expect(logChange.mock.calls).toEqual([ + ['directoryAdded', p('newdir')], + ['fileAdded', p('newdir/file.js'), [123, 0, 0, '', '', 0, null]], + ]); + }); + + test('tracks multiple new directories for deeply nested paths', () => { + simpleTfs.addOrModify( + p('a/b/c/file.js'), + [123, 0, 0, '', '', 0, null], + listener, + ); + expect(logChange.mock.calls).toEqual([ + ['directoryAdded', p('a')], + ['directoryAdded', p('a/b')], + ['directoryAdded', p('a/b/c')], + ['fileAdded', p('a/b/c/file.js'), [123, 0, 0, '', '', 0, null]], + ]); + }); + + test('does not track existing directories as new', () => { + simpleTfs.addOrModify( + p('dir/another.js'), + [789, 0, 0, '', '', 0, null], + listener, + ); + + expect(logChange.mock.calls).toEqual([ + ['fileAdded', p('dir/another.js'), [789, 0, 0, '', '', 0, null]], + ]); + }); + }); + + describe('bulkAddOrModify with listener', () => { + test('tracks multiple added files', () => { + simpleTfs.bulkAddOrModify( + new Map([ + [p('file1.js'), [1, 0, 0, '', '', 0, null]], + [p('file2.js'), [2, 0, 0, '', '', 0, null]], + [p('file3.js'), [3, 0, 0, '', '', 0, null]], + ]), + listener, + ); + + expect(logChange.mock.calls).toEqual([ + ['fileAdded', p('file1.js'), [1, 0, 0, '', '', 0, null]], + ['fileAdded', p('file2.js'), [2, 0, 0, '', '', 0, null]], + ['fileAdded', p('file3.js'), [3, 0, 0, '', '', 0, null]], + ]); + }); + }); + + test('accumulates changes across multiple operations', () => { + simpleTfs.addOrModify(p('new1.js'), [1, 0, 0, '', 0], listener); + simpleTfs.addOrModify(p('new2/file.js'), [2, 0, 0, '', 0], listener); + simpleTfs.addOrModify(p('new2/file.js'), [3, 0, 0, '', 0], listener); + simpleTfs.addOrModify( + p('new3/nested/file.js'), + [3, 0, 0, '', 0], + listener, + ); + simpleTfs.remove(p('existing.js'), listener); + simpleTfs.remove(p('new2/file.js'), listener); + + expect(logChange.mock.calls).toEqual([ + ['fileAdded', p('new1.js'), [1, 0, 0, '', 0]], + ['directoryAdded', p('new2')], + ['fileAdded', p('new2/file.js'), [2, 0, 0, '', 0]], + ['fileModified', p('new2/file.js'), [2, 0, 0, '', 0], [3, 0, 0, '', 0]], + ['directoryAdded', p('new3')], + ['directoryAdded', p('new3/nested')], + ['fileAdded', p('new3/nested/file.js'), [3, 0, 0, '', 0]], + ['fileRemoved', p('existing.js'), [123, 0, 0, '', 0]], + ['fileRemoved', p('new2/file.js'), [3, 0, 0, '', 0]], + ['directoryRemoved', p('new2')], + ]); + }); + + describe('symlinks with listener', () => { + test('tracks added files when adding a symlink', () => { + simpleTfs.addOrModify( + p('link-to-existing'), + [0, 0, 0, '', p('./existing.js')], + listener, + ); + + expect(logChange.mock.calls).toEqual([ + [ + 'fileAdded', + p('link-to-existing'), + [0, 0, 0, '', p('./existing.js')], + ], + ]); + }); + + test('tracks removed symlinks with their metadata', () => { + simpleTfs.remove(p('mylink'), listener); + expect(logChange.mock.calls).toEqual([ + ['fileRemoved', p('mylink'), [0, 0, 0, '', p('./dir')]], + ]); + }); + }); + }); }); diff --git a/packages/metro-file-map/src/plugins/DependencyPlugin.js b/packages/metro-file-map/src/plugins/DependencyPlugin.js index 469624ce83..7ad2b57e22 100644 --- a/packages/metro-file-map/src/plugins/DependencyPlugin.js +++ b/packages/metro-file-map/src/plugins/DependencyPlugin.js @@ -10,7 +10,6 @@ */ import type { - FileMapDelta, FileMapPlugin, FileMapPluginInitOptions, FileMapPluginWorker, @@ -64,25 +63,11 @@ export default class DependencyPlugin return null; } - bulkUpdate(delta: FileMapDelta>): void { + onChanged(): void { // No-op: Worker already populated plugin data // Plugin data is write-only from worker } - onNewOrModifiedFile( - relativeFilePath: string, - pluginData: ?ReadonlyArray, - ): void { - // No-op: Dependencies already in plugin data - } - - onRemovedFile( - relativeFilePath: string, - pluginData: ?ReadonlyArray, - ): void { - // No-op - } - assertValid(): void { // No validation needed } diff --git a/packages/metro-file-map/src/plugins/HastePlugin.js b/packages/metro-file-map/src/plugins/HastePlugin.js index 89b9b56a0f..1aed548c3b 100644 --- a/packages/metro-file-map/src/plugins/HastePlugin.js +++ b/packages/metro-file-map/src/plugins/HastePlugin.js @@ -13,7 +13,6 @@ import type { Console, DuplicatesIndex, DuplicatesSet, - FileMapDelta, FileMapPlugin, FileMapPluginInitOptions, FileMapPluginWorker, @@ -24,6 +23,7 @@ import type { HTypeValue, Path, PerfLogger, + ReadonlyFileSystemChanges, } from '../flow-types'; import H from '../constants'; @@ -237,26 +237,26 @@ export default class HastePlugin ); } - bulkUpdate(delta: FileMapDelta): void { + onChanged(delta: ReadonlyFileSystemChanges): void { // Process removals first so that moves aren't treated as duplicates. - for (const [normalPath, maybeHasteId] of delta.removed) { - this.onRemovedFile(normalPath, maybeHasteId); + for (const [canonicalPath, maybeHasteId] of delta.removedFiles) { + this.#onRemovedFile(canonicalPath, maybeHasteId); } - for (const [normalPath, maybeHasteId] of delta.addedOrModified) { - this.onNewOrModifiedFile(normalPath, maybeHasteId); + for (const [canonicalPath, maybeHasteId] of delta.addedFiles) { + this.#onNewFile(canonicalPath, maybeHasteId); } } - onNewOrModifiedFile(relativeFilePath: string, id: ?string) { + #onNewFile(canonicalPath: string, id: ?string) { if (id == null) { // Not a Haste module or package return; } const module: HasteMapItemMetadata = [ - relativeFilePath, + canonicalPath, this.#enableHastePackages && - path.basename(relativeFilePath) === 'package.json' + path.basename(canonicalPath) === 'package.json' ? H.PACKAGE : H.MODULE, ]; @@ -324,14 +324,14 @@ export default class HastePlugin hasteMapItem[platform] = module; } - onRemovedFile(relativeFilePath: string, moduleName: ?string) { + #onRemovedFile(canonicalPath: string, moduleName: ?string) { if (moduleName == null) { // Not a Haste module or package return; } const platform = - getPlatformExtension(relativeFilePath, this.#platforms) || + getPlatformExtension(canonicalPath, this.#platforms) || H.GENERIC_PLATFORM; const hasteMapItem = this.#map.get(moduleName); @@ -344,7 +344,7 @@ export default class HastePlugin } } - this.#recoverDuplicates(moduleName, relativeFilePath); + this.#recoverDuplicates(moduleName, canonicalPath); } assertValid(): void { diff --git a/packages/metro-file-map/src/plugins/MockPlugin.js b/packages/metro-file-map/src/plugins/MockPlugin.js index bf0baaa995..305b317558 100644 --- a/packages/metro-file-map/src/plugins/MockPlugin.js +++ b/packages/metro-file-map/src/plugins/MockPlugin.js @@ -10,13 +10,13 @@ */ import type { - FileMapDelta, FileMapPlugin, FileMapPluginInitOptions, FileMapPluginWorker, MockMap as IMockMap, Path, RawMockMap, + ReadonlyFileSystemChanges, } from '../flow-types'; import normalizePathSeparatorsToPosix from '../lib/normalizePathSeparatorsToPosix'; @@ -79,15 +79,12 @@ export default class MockPlugin this.#raw = pluginState; } else { // Otherwise, traverse all files to rebuild - this.bulkUpdate({ - addedOrModified: [ - ...files.fileIterator({ - includeNodeModules: false, - includeSymlinks: false, - }), - ].map(({canonicalPath}) => [canonicalPath, null]), - removed: [], - }); + for (const {canonicalPath} of files.fileIterator({ + includeNodeModules: false, + includeSymlinks: false, + })) { + this.#onFileAdded(canonicalPath); + } } } @@ -102,24 +99,24 @@ export default class MockPlugin ); } - bulkUpdate(delta: FileMapDelta<>): void { + onChanged(delta: ReadonlyFileSystemChanges): void { // Process removals first so that moves aren't treated as duplicates. - for (const [relativeFilePath] of delta.removed) { - this.onRemovedFile(relativeFilePath); + for (const [canonicalPath] of delta.removedFiles) { + this.#onFileRemoved(canonicalPath); } - for (const [relativeFilePath] of delta.addedOrModified) { - this.onNewOrModifiedFile(relativeFilePath); + for (const [canonicalPath] of delta.addedFiles) { + this.#onFileAdded(canonicalPath); } } - onNewOrModifiedFile(relativeFilePath: Path): void { - const absoluteFilePath = this.#pathUtils.normalToAbsolute(relativeFilePath); + #onFileAdded(canonicalPath: Path): void { + const absoluteFilePath = this.#pathUtils.normalToAbsolute(canonicalPath); if (!this.#mocksPattern.test(absoluteFilePath)) { return; } const mockName = getMockName(absoluteFilePath); - const posixRelativePath = normalizePathSeparatorsToPosix(relativeFilePath); + const posixRelativePath = normalizePathSeparatorsToPosix(canonicalPath); const existingMockPosixPath = this.#raw.mocks.get(mockName); if (existingMockPosixPath != null) { @@ -141,16 +138,15 @@ export default class MockPlugin this.#raw.mocks.set(mockName, posixRelativePath); } - onRemovedFile(relativeFilePath: Path): void { - const absoluteFilePath = this.#pathUtils.normalToAbsolute(relativeFilePath); + #onFileRemoved(canonicalPath: Path): void { + const absoluteFilePath = this.#pathUtils.normalToAbsolute(canonicalPath); if (!this.#mocksPattern.test(absoluteFilePath)) { return; } const mockName = getMockName(absoluteFilePath); const duplicates = this.#raw.duplicates.get(mockName); if (duplicates != null) { - const posixRelativePath = - normalizePathSeparatorsToPosix(relativeFilePath); + const posixRelativePath = normalizePathSeparatorsToPosix(canonicalPath); duplicates.delete(posixRelativePath); if (duplicates.size === 1) { this.#raw.duplicates.delete(mockName); diff --git a/packages/metro-file-map/src/plugins/__tests__/DependencyPlugin-test.js b/packages/metro-file-map/src/plugins/dependencies/__tests__/DependencyPlugin-test.js similarity index 94% rename from packages/metro-file-map/src/plugins/__tests__/DependencyPlugin-test.js rename to packages/metro-file-map/src/plugins/dependencies/__tests__/DependencyPlugin-test.js index 0c8030bc09..bf00973127 100644 --- a/packages/metro-file-map/src/plugins/__tests__/DependencyPlugin-test.js +++ b/packages/metro-file-map/src/plugins/dependencies/__tests__/DependencyPlugin-test.js @@ -9,7 +9,7 @@ * @oncall react_native */ -import DependencyPlugin from '../DependencyPlugin'; +import DependencyPlugin from '../../DependencyPlugin'; import path from 'path'; describe('DependencyPlugin', () => { @@ -71,12 +71,9 @@ describe('DependencyPlugin', () => { }); test('returns different cache keys for different dependency extractors', () => { - const extractorPath = path.join( - __dirname, - '../../__tests__/dependencyExtractor.js', - ); + const extractorPath = path.join(__dirname, 'mockDependencyExtractor.js'); // $FlowFixMe[untyped-import] - const dependencyExtractor = require('../../__tests__/dependencyExtractor'); + const dependencyExtractor = require('./mockDependencyExtractor'); // Create plugin with cache key 'foo' dependencyExtractor.setCacheKey('foo'); @@ -103,10 +100,7 @@ describe('DependencyPlugin', () => { }); test('cache key includes extractor path', () => { - const extractorPath = path.join( - __dirname, - '../../__tests__/dependencyExtractor.js', - ); + const extractorPath = path.join(__dirname, 'mockDependencyExtractor.js'); plugin = new DependencyPlugin({ dependencyExtractor: extractorPath, computeDependencies: true, @@ -118,12 +112,9 @@ describe('DependencyPlugin', () => { }); test('handles extractor without getCacheKey method', () => { - const extractorPath = path.join( - __dirname, - '../../__tests__/dependencyExtractor.js', - ); + const extractorPath = path.join(__dirname, 'mockDependencyExtractor.js'); // $FlowFixMe[untyped-import] - const dependencyExtractor = require('../../__tests__/dependencyExtractor'); + const dependencyExtractor = require('./mockDependencyExtractor'); // Temporarily remove getCacheKey const originalGetCacheKey = dependencyExtractor.getCacheKey; diff --git a/packages/metro-file-map/src/__tests__/dependencyExtractor.js b/packages/metro-file-map/src/plugins/dependencies/__tests__/mockDependencyExtractor.js similarity index 100% rename from packages/metro-file-map/src/__tests__/dependencyExtractor.js rename to packages/metro-file-map/src/plugins/dependencies/__tests__/mockDependencyExtractor.js diff --git a/packages/metro-file-map/src/plugins/haste/__tests__/HastePlugin-test.js b/packages/metro-file-map/src/plugins/haste/__tests__/HastePlugin-test.js index b1d8fa8e28..c1b09c29f4 100644 --- a/packages/metro-file-map/src/plugins/haste/__tests__/HastePlugin-test.js +++ b/packages/metro-file-map/src/plugins/haste/__tests__/HastePlugin-test.js @@ -9,6 +9,10 @@ * @oncall react_native */ +import type { + CanonicalPath, + ReadonlyFileSystemChanges, +} from '../../../flow-types'; import type HasteMapType from '../../HastePlugin'; let mockPathModule; @@ -88,11 +92,16 @@ describe.each([['win32'], ['posix']])('HastePlugin on %s', platform => { expect(hasteMap.getModule('NameForFoo')).toEqual(p('/root/project/Foo.js')); }); - describe('onRemovedFile', () => { + describe('remove file', () => { let hasteMap: HasteMapType; + let removeFile: (path: CanonicalPath, name: ?string) => void; beforeEach(async () => { hasteMap = new HasteMap(opts); + removeFile = (canonicalPath, name) => + hasteMap.onChanged( + makeChanges({added: [], removed: [[canonicalPath, name]]}), + ); await hasteMap.initialize({ files: { fileIterator: jest.fn().mockReturnValue(INITIAL_FILES), @@ -104,7 +113,7 @@ describe.each([['win32'], ['posix']])('HastePlugin on %s', platform => { test('removes a module, without affecting others', () => { expect(hasteMap.getModule('NameForFoo')).not.toBeNull(); - hasteMap.onRemovedFile(p('project/Foo.js'), 'NameForFoo'); + removeFile(p('project/Foo.js'), 'NameForFoo'); expect(hasteMap.getModule('NameForFoo')).toBeNull(); expect(hasteMap.getModule('Bar')).not.toBeNull(); }); @@ -113,14 +122,14 @@ describe.each([['win32'], ['posix']])('HastePlugin on %s', platform => { expect(() => hasteMap.getModule('Duplicate')).toThrow( DuplicateHasteCandidatesError, ); - hasteMap.onRemovedFile(p('project/Duplicate.js'), 'Duplicate'); + removeFile(p('project/Duplicate.js'), 'Duplicate'); expect(hasteMap.getModule('Duplicate')).toBe( p('/root/project/other/Duplicate.js'), ); }); }); - describe('bulkUpdate', () => { + describe('onChanged', () => { let hasteMap: HasteMapType; beforeEach(async () => { @@ -134,27 +143,22 @@ describe.each([['win32'], ['posix']])('HastePlugin on %s', platform => { }); }); - test('removes a module, without affecting others', () => { - expect(hasteMap.getModule('NameForFoo')).not.toBeNull(); - hasteMap.onRemovedFile(p('project/Foo.js'), 'NameForFoo'); - expect(hasteMap.getModule('NameForFoo')).toBeNull(); - expect(hasteMap.getModule('Bar')).not.toBeNull(); - }); - test('fixes duplicates, adds and removes modules', () => { expect(() => hasteMap.getModule('Duplicate')).toThrow( DuplicateHasteCandidatesError, ); - hasteMap.bulkUpdate({ - removed: [ - [p('project/Duplicate.js'), 'Duplicate'], - [p('project/Foo.js'), 'NameForFoo'], - ], - addedOrModified: [ - [p('project/Baz.js'), 'Baz'], // New - [p('project/other/Bar.js'), 'Bar'], // New duplicate - ], - }); + hasteMap.onChanged( + makeChanges({ + added: [ + [p('project/Baz.js'), 'Baz'], // New + [p('project/other/Bar.js'), 'Bar'], // New duplicate + ], + removed: [ + [p('project/Duplicate.js'), 'Duplicate'], + [p('project/Foo.js'), 'NameForFoo'], + ], + }), + ); expect(hasteMap.getModule('Duplicate')).toBe( p('/root/project/other/Duplicate.js'), ); @@ -205,3 +209,19 @@ describe.each([['win32'], ['posix']])('HastePlugin on %s', platform => { }); }); }); + +function makeChanges({ + added, + removed, +}: Readonly<{ + added: ReadonlyArray<[CanonicalPath, ?string]>, + removed: ReadonlyArray<[CanonicalPath, ?string]>, +}>): ReadonlyFileSystemChanges { + return { + addedFiles: new Map(added), + removedFiles: new Map(removed), + modifiedFiles: new Map(), + addedDirectories: new Set(), + removedDirectories: new Set(), + }; +} diff --git a/packages/metro-file-map/src/plugins/mocks/__tests__/MockPlugin-test.js b/packages/metro-file-map/src/plugins/mocks/__tests__/MockPlugin-test.js index 6796c29430..43f2835646 100644 --- a/packages/metro-file-map/src/plugins/mocks/__tests__/MockPlugin-test.js +++ b/packages/metro-file-map/src/plugins/mocks/__tests__/MockPlugin-test.js @@ -21,6 +21,8 @@ describe.each([['win32'], ['posix']])('MockPlugin on %s', platform => { : filePath; let MockMap: Class; + let mockMap: MockMapType; + let onFileAdded: (filePath: string) => void; const opts = { console, @@ -33,20 +35,27 @@ describe.each([['win32'], ['posix']])('MockPlugin on %s', platform => { jest.resetModules(); mockPathModule = jest.requireActual<{}>('path')[platform]; MockMap = require('../../MockPlugin').default; + mockMap = new MockMap(opts); + onFileAdded = canonicalPath => + mockMap.onChanged({ + addedFiles: new Map([[canonicalPath, null]]), + modifiedFiles: new Map(), + removedFiles: new Map(), + addedDirectories: new Set(), + removedDirectories: new Set(), + }); jest.spyOn(console, 'warn').mockImplementation(() => {}); jest.clearAllMocks(); }); test('set and get a mock module', () => { - const mockMap = new MockMap(opts); - mockMap.onNewOrModifiedFile(p('__mocks__/foo.js')); + onFileAdded(p('__mocks__/foo.js')); expect(mockMap.getMockModule('foo')).toBe(p('/root/__mocks__/foo.js')); }); test('assertValid throws on duplicates', () => { - const mockMap = new MockMap(opts); - mockMap.onNewOrModifiedFile(p('__mocks__/foo.js')); - mockMap.onNewOrModifiedFile(p('other/__mocks__/foo.js')); + onFileAdded(p('__mocks__/foo.js')); + onFileAdded(p('other/__mocks__/foo.js')); expect(console.warn).toHaveBeenCalledTimes(1); expect(() => mockMap.assertValid()).toThrowError( @@ -59,9 +68,8 @@ Duplicate manual mock found for \`foo\`: }); test('recovers from duplicates', () => { - const mockMap = new MockMap(opts); - mockMap.onNewOrModifiedFile(p('__mocks__/foo.js')); - mockMap.onNewOrModifiedFile(p('other/__mocks__/foo.js')); + onFileAdded(p('__mocks__/foo.js')); + onFileAdded(p('other/__mocks__/foo.js')); expect(() => mockMap.assertValid()).toThrow(); @@ -79,7 +87,13 @@ Duplicate manual mock found for \`foo\`: version: 2, }); - mockMap.onRemovedFile(p('other/__mocks__/foo.js')); + mockMap.onChanged({ + addedFiles: new Map(), + modifiedFiles: new Map(), + removedFiles: new Map([[p('other/__mocks__/foo.js'), null]]), + addedDirectories: new Set(), + removedDirectories: new Set(), + }); expect(() => mockMap.assertValid()).not.toThrow(); @@ -94,7 +108,6 @@ Duplicate manual mock found for \`foo\`: }); test('loads from a snapshot', async () => { - const mockMap = new MockMap(opts); await mockMap.initialize({ files: { fileIterator: () => { @@ -130,17 +143,18 @@ Duplicate manual mock found for \`foo\`: ['foo', 'other/__mocks__/foo.js'], ]), duplicates: new Map([ - ['foo', new Set(['other/__mocks__/foo.js', '__mocks__/foo.js'])], + [ + 'foo', + new Set(['other/__mocks__/foo.js', '__mocks__/foo.js']), + ], ]), version: 2, }; - /* $FlowFixMe[incompatible-type] Natural Inference rollout. See - * https://fburl.com/workplace/6291gfvu */ - const mockMap = new MockMap({...opts, rawMockMap}); - expect(mockMap.getMockModule('bar')).toEqual( + const loadedMockMap = new MockMap({...opts, rawMockMap}); + expect(loadedMockMap.getMockModule('bar')).toEqual( p('/root/some/__mocks__/bar.js'), ); - expect(mockMap.getMockModule('foo')).toEqual( + expect(loadedMockMap.getMockModule('foo')).toEqual( p('/root/other/__mocks__/foo.js'), ); }); diff --git a/packages/metro-file-map/types/flow-types.d.ts b/packages/metro-file-map/types/flow-types.d.ts index edb5ccdfd5..3c089520d2 100644 --- a/packages/metro-file-map/types/flow-types.d.ts +++ b/packages/metro-file-map/types/flow-types.d.ts @@ -6,7 +6,7 @@ * * @noformat * @oncall react_native - * @generated SignedSource<<919ac912df195d04796dd62cb68839d2>> + * @generated SignedSource<<462548bb01970bd5a18c5edf1ee17187>> * * This file was translated from Flow by scripts/generateTypeScriptDefinitions.js * Original file: packages/metro-file-map/src/flow-types.js @@ -75,10 +75,27 @@ export type CacheManagerWriteOptions = Readonly<{ onWriteError: (error: Error) => void; }>; export type CanonicalPath = string; -export type ChangeEvent = { +/** + * An object passed to TreeFS methods that captures file system observations + * relevant to incremental invalidation. TreeFS will populate the `existence` + * and `modification` sets with canonical (root-relative) paths. The `haste` + * set is not written by TreeFS but is included so that a single object can + * capture all invalidation data for a resolution. + */ +export type InvalidationData = Readonly<{ + existence: Set; + modification: Set; + haste: Set; +}>; +export type ChangedFileMetadata = Readonly<{ + isSymlink: boolean; + modifiedTime?: null | undefined | number; +}>; +export type ChangeEvent = Readonly<{ logger: null | undefined | RootPerfLogger; - eventsQueue: EventsQueue; -}; + changes: ReadonlyFileSystemChanges>; + rootDir: string; +}>; export type ChangeEventMetadata = { modifiedTime: null | undefined | number; size: null | undefined | number; @@ -128,15 +145,6 @@ export type WatcherStatus = }; export type DuplicatesSet = Map; export type DuplicatesIndex = Map>; -export type EventsQueue = Array<{ - filePath: Path; - metadata: ChangeEventMetadata; - type: string; -}>; -export type FileMapDelta = Readonly<{ - removed: Iterable<[CanonicalPath, T]>; - addedOrModified: Iterable<[CanonicalPath, T]>; -}>; export type FileMapPluginInitOptions< SerializableState, PerFileData = void, @@ -183,16 +191,10 @@ export interface FileMapPlugin< initOptions: FileMapPluginInitOptions, ): Promise; assertValid(): void; - bulkUpdate(delta: FileMapDelta): void; - getSerializableSnapshot(): SerializableState; - onRemovedFile( - relativeFilePath: string, - pluginData: null | undefined | PerFileData, - ): void; - onNewOrModifiedFile( - relativeFilePath: string, - pluginData: null | undefined | PerFileData, + onChanged( + changes: ReadonlyFileSystemChanges, ): void; + getSerializableSnapshot(): SerializableState; getCacheKey(): string; getWorker(): null | undefined | FileMapPluginWorker; } @@ -260,8 +262,10 @@ export interface FileSystem { * X = dirname(X) * while X !== dirname(X) * - * If opts.invalidatedBy is given, collects all absolute, real paths that if - * added or removed may invalidate this result. + * If opts.invalidatedBy is given, collects canonical (root-relative) paths + * into its sets: + * - existence: paths whose addition or removal may invalidate this result + * - modification: symlinks traversed, whose target change may invalidate * * Useful for finding the closest package scope (subpath: package.json, * type f, breakOnSegment: node_modules) or closest potential package root @@ -272,7 +276,7 @@ export interface FileSystem { subpath: string, opts: { breakOnSegment: null | undefined | string; - invalidatedBy: null | undefined | Set; + invalidatedBy: null | undefined | InvalidationData; subpathType: 'f' | 'd'; }, ): null | undefined | {absolutePath: string; containerRelativePath: string}; @@ -342,10 +346,38 @@ export type HasteMapItem = { [platform: string]: HasteMapItemMetadata; }; export type HasteMapItemMetadata = [string, number]; +export interface FileSystemListener { + directoryAdded(canonicalPath: CanonicalPath): void; + directoryRemoved(canonicalPath: CanonicalPath): void; + fileAdded(canonicalPath: CanonicalPath, data: FileMetadata): void; + fileModified( + canonicalPath: CanonicalPath, + oldData: FileMetadata, + newData: FileMetadata, + ): void; + fileRemoved(canonicalPath: CanonicalPath, data: FileMetadata): void; +} +export interface ReadonlyFileSystemChanges { + readonly addedDirectories: Iterable; + readonly removedDirectories: Iterable; + readonly addedFiles: Iterable>; + readonly modifiedFiles: Iterable>; + readonly removedFiles: Iterable>; +} export interface MutableFileSystem extends FileSystem { - remove(filePath: Path): null | undefined | FileMetadata; - addOrModify(filePath: Path, fileMetadata: FileMetadata): void; - bulkAddOrModify(addedOrModifiedFiles: FileData): void; + remove( + filePath: Path, + listener?: FileSystemListener, + ): null | undefined | FileMetadata; + addOrModify( + filePath: Path, + fileMetadata: FileMetadata, + listener?: FileSystemListener, + ): void; + bulkAddOrModify( + addedOrModifiedFiles: FileData, + listener?: FileSystemListener, + ): void; } export type Path = string; export type ProcessFileFunction = ( diff --git a/packages/metro-file-map/types/lib/FileSystemChangeAggregator.d.ts b/packages/metro-file-map/types/lib/FileSystemChangeAggregator.d.ts new file mode 100644 index 0000000000..0a4b3ecd92 --- /dev/null +++ b/packages/metro-file-map/types/lib/FileSystemChangeAggregator.d.ts @@ -0,0 +1,40 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @noformat + * @oncall react_native + * @generated SignedSource<<5feda1b197530a9a5fdbc57200633ac5>> + * + * This file was translated from Flow by scripts/generateTypeScriptDefinitions.js + * Original file: packages/metro-file-map/src/lib/FileSystemChangeAggregator.js + * To regenerate, run: + * js1 build metro-ts-defs (internal) OR + * yarn run build-ts-defs (OSS) + */ + +import type { + CanonicalPath, + FileMetadata, + FileSystemListener, + ReadonlyFileSystemChanges, +} from '../flow-types'; + +export declare class FileSystemChangeAggregator implements FileSystemListener { + directoryAdded(canonicalPath: CanonicalPath): void; + directoryRemoved(canonicalPath: CanonicalPath): void; + fileAdded(canonicalPath: CanonicalPath, data: FileMetadata): void; + fileModified( + canonicalPath: CanonicalPath, + oldData: FileMetadata, + newData: FileMetadata, + ): void; + fileRemoved(canonicalPath: CanonicalPath, data: FileMetadata): void; + getSize(): number; + getView(): ReadonlyFileSystemChanges; + getMappedView( + metadataMapFn: (metadata: FileMetadata) => T, + ): ReadonlyFileSystemChanges; +} diff --git a/packages/metro-file-map/types/lib/TreeFS.d.ts b/packages/metro-file-map/types/lib/TreeFS.d.ts index 7764c9648c..ac86dbdeeb 100644 --- a/packages/metro-file-map/types/lib/TreeFS.d.ts +++ b/packages/metro-file-map/types/lib/TreeFS.d.ts @@ -5,7 +5,7 @@ * LICENSE file in the root directory of this source tree. * * @noformat - * @generated SignedSource<<1f36861cea798d8cc2a5dc61293ecb1b>> + * @generated SignedSource<<3dfb807bf32043b2d9e812418a5e112d>> * * This file was translated from Flow by scripts/generateTypeScriptDefinitions.js * Original file: packages/metro-file-map/src/lib/TreeFS.js @@ -19,6 +19,7 @@ import type { FileData, FileMetadata, FileStats, + FileSystemListener, LookupResult, MutableFileSystem, Path, @@ -122,9 +123,19 @@ declare class TreeFS implements MutableFileSystem { * for example: `a/b.js` -> `./a/b.js` */ matchFiles(opts: MatchFilesOptions): Iterable; - addOrModify(mixedPath: Path, metadata: FileMetadata): void; - bulkAddOrModify(addedOrModifiedFiles: FileData): void; - remove(mixedPath: Path): null | undefined | FileMetadata; + addOrModify( + mixedPath: Path, + metadata: FileMetadata, + changeListener?: FileSystemListener, + ): void; + bulkAddOrModify( + addedOrModifiedFiles: FileData, + changeListener?: FileSystemListener, + ): void; + remove( + mixedPath: Path, + changeListener?: FileSystemListener, + ): null | undefined | FileMetadata; /** * Given a start path (which need not exist), a subpath and type, and * optionally a 'breakOnSegment', performs the following: @@ -141,8 +152,10 @@ declare class TreeFS implements MutableFileSystem { * X = dirname(X) * while X !== dirname(X) * - * If opts.invalidatedBy is given, collects all absolute, real paths that if - * added or removed may invalidate this result. + * If opts.invalidatedBy is given, collects canonical (root-relative) paths + * into its sets: + * - existence: paths whose addition or removal may invalidate this result + * - modification: symlinks traversed, whose target change may invalidate * * Useful for finding the closest package scope (subpath: package.json, * type f, breakOnSegment: node_modules) or closest potential package root @@ -153,7 +166,7 @@ declare class TreeFS implements MutableFileSystem { subpath: string, opts: { breakOnSegment: null | undefined | string; - invalidatedBy: null | undefined | Set; + invalidatedBy: null | undefined | any; subpathType: 'f' | 'd'; }, ): null | undefined | {absolutePath: string; containerRelativePath: string}; diff --git a/packages/metro-file-map/types/plugins/DependencyPlugin.d.ts b/packages/metro-file-map/types/plugins/DependencyPlugin.d.ts index 8ac8355ec8..0fa9a41778 100644 --- a/packages/metro-file-map/types/plugins/DependencyPlugin.d.ts +++ b/packages/metro-file-map/types/plugins/DependencyPlugin.d.ts @@ -6,7 +6,7 @@ * * @noformat * @oncall react_native - * @generated SignedSource<<71361b3fd04f54f55665031c66465dd7>> + * @generated SignedSource<> * * This file was translated from Flow by scripts/generateTypeScriptDefinitions.js * Original file: packages/metro-file-map/src/plugins/DependencyPlugin.js @@ -16,7 +16,6 @@ */ import type { - FileMapDelta, FileMapPlugin, FileMapPluginInitOptions, FileMapPluginWorker, @@ -39,17 +38,7 @@ declare class DependencyPlugin initOptions: FileMapPluginInitOptions | null>, ): Promise; getSerializableSnapshot(): null; - bulkUpdate( - delta: FileMapDelta>, - ): void; - onNewOrModifiedFile( - relativeFilePath: string, - pluginData: null | undefined | ReadonlyArray, - ): void; - onRemovedFile( - relativeFilePath: string, - pluginData: null | undefined | ReadonlyArray, - ): void; + onChanged(): void; assertValid(): void; getCacheKey(): string; getWorker(): FileMapPluginWorker; diff --git a/packages/metro-file-map/types/plugins/HastePlugin.d.ts b/packages/metro-file-map/types/plugins/HastePlugin.d.ts index f39b950526..71d6cbc263 100644 --- a/packages/metro-file-map/types/plugins/HastePlugin.d.ts +++ b/packages/metro-file-map/types/plugins/HastePlugin.d.ts @@ -6,7 +6,7 @@ * * @noformat * @oncall react_native - * @generated SignedSource<> + * @generated SignedSource<<3d1462ab2325a09553e02b69b5de84eb>> * * This file was translated from Flow by scripts/generateTypeScriptDefinitions.js * Original file: packages/metro-file-map/src/plugins/HastePlugin.js @@ -17,7 +17,6 @@ import type { Console, - FileMapDelta, FileMapPlugin, FileMapPluginInitOptions, FileMapPluginWorker, @@ -27,6 +26,7 @@ import type { HTypeValue, Path, PerfLogger, + ReadonlyFileSystemChanges, } from '../flow-types'; export type HasteMapOptions = Readonly<{ @@ -59,16 +59,8 @@ declare class HastePlugin platform: null | undefined | string, _supportsNativePlatform?: null | undefined | boolean, ): null | undefined | Path; - bulkUpdate(delta: FileMapDelta): void; - onNewOrModifiedFile( - relativeFilePath: string, - id: null | undefined | string, - ): void; + onChanged(delta: ReadonlyFileSystemChanges): void; setModule(id: string, module: HasteMapItemMetadata): void; - onRemovedFile( - relativeFilePath: string, - moduleName: null | undefined | string, - ): void; assertValid(): void; computeConflicts(): Array; getCacheKey(): string; diff --git a/packages/metro-file-map/types/plugins/MockPlugin.d.ts b/packages/metro-file-map/types/plugins/MockPlugin.d.ts index 11b4d03a7c..9d5abf4521 100644 --- a/packages/metro-file-map/types/plugins/MockPlugin.d.ts +++ b/packages/metro-file-map/types/plugins/MockPlugin.d.ts @@ -6,7 +6,7 @@ * * @noformat * @oncall react_native - * @generated SignedSource<<81805d051693b746e75928fe6ed3dbca>> + * @generated SignedSource<> * * This file was translated from Flow by scripts/generateTypeScriptDefinitions.js * Original file: packages/metro-file-map/src/plugins/MockPlugin.js @@ -16,13 +16,13 @@ */ import type { - FileMapDelta, FileMapPlugin, FileMapPluginInitOptions, FileMapPluginWorker, MockMap as IMockMap, Path, RawMockMap, + ReadonlyFileSystemChanges, } from '../flow-types'; export declare const CACHE_VERSION: 2; @@ -39,9 +39,7 @@ declare class MockPlugin implements FileMapPlugin, IMockMap { constructor($$PARAM_0$$: MockMapOptions); initialize($$PARAM_0$$: FileMapPluginInitOptions): Promise; getMockModule(name: string): null | undefined | Path; - bulkUpdate(delta: FileMapDelta): void; - onNewOrModifiedFile(relativeFilePath: Path): void; - onRemovedFile(relativeFilePath: Path): void; + onChanged(delta: ReadonlyFileSystemChanges): void; getSerializableSnapshot(): RawMockMap; assertValid(): void; getCacheKey(): string; diff --git a/packages/metro/src/DeltaBundler/DeltaCalculator.js b/packages/metro/src/DeltaBundler/DeltaCalculator.js index fe8b64ba4d..f9e35dae95 100644 --- a/packages/metro/src/DeltaBundler/DeltaCalculator.js +++ b/packages/metro/src/DeltaBundler/DeltaCalculator.js @@ -10,7 +10,6 @@ */ import type {DeltaResult, Options} from './types'; -import type {RootPerfLogger} from 'metro-config'; import type {ChangeEvent} from 'metro-file-map'; import {Graph} from './Graph'; @@ -173,75 +172,66 @@ export default class DeltaCalculator extends EventEmitter { } _handleMultipleFileChanges = (changeEvent: ChangeEvent) => { - changeEvent.eventsQueue.forEach(eventInfo => { - this._handleFileChange(eventInfo, changeEvent.logger); - }); - }; - - /** - * Handles a single file change. To avoid doing any work before it's needed, - * the listener only stores the modified file, which will then be used later - * when the delta needs to be calculated. - */ - _handleFileChange = ( - {type, filePath, metadata}: ChangeEvent['eventsQueue'][number], - logger: ?RootPerfLogger, - ): unknown => { - debug('Handling %s: %s (type: %s)', type, filePath, metadata.type); - if ( - metadata.type === 'l' || - (this._options.unstable_enablePackageExports && - filePath.endsWith(path.sep + 'package.json')) - ) { - this._requiresReset = true; - this.emit('change', {logger}); - } - let state: void | 'deleted' | 'modified' | 'added'; - if (this._deletedFiles.has(filePath)) { - state = 'deleted'; - } else if (this._modifiedFiles.has(filePath)) { - state = 'modified'; - } else if (this._addedFiles.has(filePath)) { - state = 'added'; + const {changes, logger, rootDir} = changeEvent; + const enablePackageExports = this._options.unstable_enablePackageExports; + + // Process added files: deleted+added = modified, otherwise added + for (const [canonicalPath, metadata] of changes.addedFiles) { + debug('Handling add: %s', canonicalPath); + if ( + metadata.isSymlink || + (enablePackageExports && + canonicalPath.endsWith(path.sep + 'package.json')) + ) { + this._requiresReset = true; + } + const absolutePath = path.join(rootDir, canonicalPath); + if (this._deletedFiles.has(absolutePath)) { + this._deletedFiles.delete(absolutePath); + this._modifiedFiles.add(absolutePath); + } else { + this._addedFiles.add(absolutePath); + this._modifiedFiles.delete(absolutePath); + } } - let nextState: 'deleted' | 'modified' | 'added'; - if (type === 'delete') { - nextState = 'deleted'; - } else if (type === 'add') { - // A deleted+added file is modified - nextState = state === 'deleted' ? 'modified' : 'added'; - } else { - // type === 'change' - // An added+modified file is added - nextState = state === 'added' ? 'added' : 'modified'; + // Process modified files: added+modified stays added, otherwise modified + for (const [canonicalPath, metadata] of changes.modifiedFiles) { + debug('Handling change: %s', canonicalPath); + if ( + metadata.isSymlink || + (enablePackageExports && + canonicalPath.endsWith(path.sep + 'package.json')) + ) { + this._requiresReset = true; + } + const absolutePath = path.join(rootDir, canonicalPath); + if (!this._addedFiles.has(absolutePath)) { + this._modifiedFiles.add(absolutePath); + } + this._deletedFiles.delete(absolutePath); } - switch (nextState) { - case 'deleted': - this._deletedFiles.add(filePath); - this._modifiedFiles.delete(filePath); - this._addedFiles.delete(filePath); - break; - case 'added': - this._addedFiles.add(filePath); - this._deletedFiles.delete(filePath); - this._modifiedFiles.delete(filePath); - break; - case 'modified': - this._modifiedFiles.add(filePath); - this._deletedFiles.delete(filePath); - this._addedFiles.delete(filePath); - break; - default: - nextState as empty; + // Process removed files: added+deleted = no change, otherwise deleted + for (const [canonicalPath, metadata] of changes.removedFiles) { + debug('Handling delete: %s', canonicalPath); + if ( + metadata.isSymlink || + (enablePackageExports && + canonicalPath.endsWith(path.sep + 'package.json')) + ) { + this._requiresReset = true; + } + const absolutePath = path.resolve(rootDir, canonicalPath); + if (this._addedFiles.has(absolutePath)) { + this._addedFiles.delete(absolutePath); + } else { + this._deletedFiles.add(absolutePath); + this._modifiedFiles.delete(absolutePath); + } } - // Notify users that there is a change in some of the bundle files. This - // way the client can choose to refetch the bundle. - this.emit('change', { - logger, - }); + this.emit('change', {logger}); }; async _getChangedDependencies( diff --git a/packages/metro/src/DeltaBundler/__tests__/DeltaCalculator-context-test.js b/packages/metro/src/DeltaBundler/__tests__/DeltaCalculator-context-test.js index ab277e5731..03adaf4416 100644 --- a/packages/metro/src/DeltaBundler/__tests__/DeltaCalculator-context-test.js +++ b/packages/metro/src/DeltaBundler/__tests__/DeltaCalculator-context-test.js @@ -17,6 +17,7 @@ import type {Options, TransformResultDependency} from '../types'; import CountingSet from '../../lib/CountingSet'; import DeltaCalculator from '../DeltaCalculator'; import {Graph} from '../Graph'; +import {createEmitChange} from './test-utils'; const {EventEmitter} = require('events'); @@ -36,6 +37,7 @@ const markModifiedContextModules = jest.spyOn( describe('DeltaCalculator + require.context', () => { let deltaCalculator; let fileWatcher; + let emitChange; const options: Options<> = { unstable_allowRequireContext: true, @@ -62,6 +64,7 @@ describe('DeltaCalculator + require.context', () => { beforeEach(async () => { fileWatcher = new EventEmitter(); + emitChange = createEmitChange(fileWatcher, '/'); markModifiedContextModules.mockImplementation(function ( this: Graph, @@ -172,11 +175,7 @@ describe('DeltaCalculator + require.context', () => { // Initial build await deltaCalculator.getDelta({reset: false, shallow: false}); - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'delete', filePath: '/ctx/foo', metadata: {type: 'f'}}, - ], - }); + emitChange({removedFiles: ['ctx/foo']}); // Incremental build await deltaCalculator.getDelta({ @@ -199,11 +198,7 @@ describe('DeltaCalculator + require.context', () => { // Initial build await deltaCalculator.getDelta({reset: false, shallow: false}); - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'add', filePath: '/ctx/foo2', metadata: {type: 'f'}}, - ], - }); + emitChange({addedFiles: ['ctx/foo2']}); // Incremental build await deltaCalculator.getDelta({ @@ -223,11 +218,7 @@ describe('DeltaCalculator + require.context', () => { // Initial build await deltaCalculator.getDelta({reset: false, shallow: false}); - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'change', filePath: '/ctx/foo', metadata: {type: 'f'}}, - ], - }); + emitChange({modifiedFiles: ['ctx/foo']}); // Incremental build await deltaCalculator.getDelta({ @@ -247,11 +238,7 @@ describe('DeltaCalculator + require.context', () => { // Initial build await deltaCalculator.getDelta({reset: false, shallow: false}); - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'change', filePath: '/ctx/foo2', metadata: {type: 'f'}}, - ], - }); + emitChange({modifiedFiles: ['ctx/foo2']}); // Incremental build await deltaCalculator.getDelta({ @@ -266,17 +253,9 @@ describe('DeltaCalculator + require.context', () => { // Initial build await deltaCalculator.getDelta({reset: false, shallow: false}); - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'add', filePath: '/ctx/foo2', metadata: {type: 'f'}}, - ], - }); + emitChange({addedFiles: ['ctx/foo2']}); - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'change', filePath: '/ctx/foo2', metadata: {type: 'f'}}, - ], - }); + emitChange({modifiedFiles: ['ctx/foo2']}); // Incremental build await deltaCalculator.getDelta({ @@ -296,17 +275,9 @@ describe('DeltaCalculator + require.context', () => { // Initial build await deltaCalculator.getDelta({reset: false, shallow: false}); - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'add', filePath: '/ctx/foo2', metadata: {type: 'f'}}, - ], - }); + emitChange({addedFiles: ['ctx/foo2']}); - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'delete', filePath: '/ctx/foo2', metadata: {type: 'f'}}, - ], - }); + emitChange({removedFiles: ['ctx/foo2']}); // Incremental build await deltaCalculator.getDelta({ @@ -321,15 +292,9 @@ describe('DeltaCalculator + require.context', () => { // Initial build await deltaCalculator.getDelta({reset: false, shallow: false}); - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'delete', filePath: '/ctx/foo', metadata: {type: 'f'}}, - ], - }); + emitChange({removedFiles: ['ctx/foo']}); - fileWatcher.emit('change', { - eventsQueue: [{type: 'add', filePath: '/ctx/foo', metadata: {type: 'f'}}], - }); + emitChange({addedFiles: ['ctx/foo']}); // Incremental build await deltaCalculator.getDelta({ @@ -347,17 +312,9 @@ describe('DeltaCalculator + require.context', () => { // Initial build await deltaCalculator.getDelta({reset: false, shallow: false}); - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'change', filePath: '/ctx/foo', metadata: {type: 'f'}}, - ], - }); + emitChange({modifiedFiles: ['ctx/foo']}); - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'delete', filePath: '/ctx/foo', metadata: {type: 'f'}}, - ], - }); + emitChange({removedFiles: ['ctx/foo']}); // Incremental build await deltaCalculator.getDelta({ diff --git a/packages/metro/src/DeltaBundler/__tests__/DeltaCalculator-test.js b/packages/metro/src/DeltaBundler/__tests__/DeltaCalculator-test.js index 6aacc4449d..633974814a 100644 --- a/packages/metro/src/DeltaBundler/__tests__/DeltaCalculator-test.js +++ b/packages/metro/src/DeltaBundler/__tests__/DeltaCalculator-test.js @@ -18,6 +18,7 @@ import type { } from '../types'; import CountingSet from '../../lib/CountingSet'; +import {createEmitChange} from './test-utils'; import path from 'path'; jest.mock('../../Bundler'); @@ -33,6 +34,7 @@ describe.each(['linux', 'win32'])('DeltaCalculator (%s)', osPlatform => { let fileWatcher; let traverseDependencies; let initialTraverseDependencies; + let emitChange; const options: Options<> = { unstable_allowRequireContext: false, @@ -59,7 +61,11 @@ describe.each(['linux', 'win32'])('DeltaCalculator (%s)', osPlatform => { function p(posixPath: string): string { if (osPlatform === 'win32') { - return path.win32.join('C:\\', ...posixPath.split('/')); + if (path.posix.isAbsolute(posixPath)) { + return path.win32.join('C:\\', ...posixPath.split('/')); + } else { + return posixPath.replaceAll('/', '\\'); + } } return posixPath; @@ -211,6 +217,12 @@ describe.each(['linux', 'win32'])('DeltaCalculator (%s)', osPlatform => { fileWatcher, options, ); + + emitChange = createEmitChange( + fileWatcher, + p('/'), + osPlatform === 'win32' ? '\\' : '/', + ); }); afterEach(() => { @@ -293,9 +305,7 @@ describe.each(['linux', 'win32'])('DeltaCalculator (%s)', osPlatform => { test('should calculate a delta after a file addition', async () => { await deltaCalculator.getDelta({reset: false, shallow: false}); - fileWatcher.emit('change', { - eventsQueue: [{type: 'add', filePath: p('/foo'), metadata: {type: 'f'}}], - }); + emitChange({addedFiles: ['foo']}); traverseDependencies.mockResolvedValueOnce({ added: new Map([[p('/foo'), fooModule]]), @@ -322,11 +332,7 @@ describe.each(['linux', 'win32'])('DeltaCalculator (%s)', osPlatform => { test('should calculate a delta after a simple modification', async () => { await deltaCalculator.getDelta({reset: false, shallow: false}); - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'change', filePath: p('/foo'), metadata: {type: 'f'}}, - ], - }); + emitChange({modifiedFiles: ['foo']}); traverseDependencies.mockReturnValue( Promise.resolve({ @@ -355,11 +361,7 @@ describe.each(['linux', 'win32'])('DeltaCalculator (%s)', osPlatform => { // Get initial delta await deltaCalculator.getDelta({reset: false, shallow: false}); - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'change', filePath: p('/foo'), metadata: {type: 'f'}}, - ], - }); + emitChange({modifiedFiles: ['foo']}); traverseDependencies.mockReturnValue( Promise.resolve({ @@ -388,11 +390,7 @@ describe.each(['linux', 'win32'])('DeltaCalculator (%s)', osPlatform => { // Get initial delta await deltaCalculator.getDelta({reset: false, shallow: false}); - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'change', filePath: p('/foo'), metadata: {type: 'f'}}, - ], - }); + emitChange({modifiedFiles: ['foo']}); const quxModule: Module<$FlowFixMe> = { dependencies: new Map(), @@ -439,11 +437,7 @@ describe.each(['linux', 'win32'])('DeltaCalculator (%s)', osPlatform => { .getDelta({reset: false, shallow: false}) .then(() => { deltaCalculator.on('change', () => done()); - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'change', filePath: p('/foo'), metadata: {type: 'f'}}, - ], - }); + emitChange({modifiedFiles: ['foo']}); }) .catch(done); }); @@ -454,9 +448,7 @@ describe.each(['linux', 'win32'])('DeltaCalculator (%s)', osPlatform => { deltaCalculator.on('change', onChangeFile); - fileWatcher.emit('change', { - eventsQueue: [{type: 'add', filePath: p('/foo'), metadata: {type: 'f'}}], - }); + emitChange({addedFiles: ['foo']}); jest.runAllTimers(); @@ -469,11 +461,7 @@ describe.each(['linux', 'win32'])('DeltaCalculator (%s)', osPlatform => { deltaCalculator.on('delete', onChangeFile); - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'delete', filePath: p('/foo'), metadata: {type: 'f'}}, - ], - }); + emitChange({removedFiles: ['foo']}); jest.runAllTimers(); @@ -483,13 +471,9 @@ describe.each(['linux', 'win32'])('DeltaCalculator (%s)', osPlatform => { test('should retry to build the last delta after getting an error', async () => { await deltaCalculator.getDelta({reset: false, shallow: false}); - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'change', filePath: p('/foo'), metadata: {type: 'f'}}, - ], - }); + emitChange({modifiedFiles: ['foo']}); - traverseDependencies.mockReturnValue(Promise.reject(new Error())); + traverseDependencies.mockRejectedValue(new Error()); await expect( deltaCalculator.getDelta({reset: false, shallow: false}), @@ -505,18 +489,10 @@ describe.each(['linux', 'win32'])('DeltaCalculator (%s)', osPlatform => { await deltaCalculator.getDelta({reset: false, shallow: false}); // First modify the file - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'change', filePath: p('/foo'), metadata: {type: 'f'}}, - ], - }); + emitChange({modifiedFiles: ['foo']}); // Then delete that same file - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'delete', filePath: p('/foo'), metadata: {type: 'f'}}, - ], - }); + emitChange({removedFiles: ['foo']}); traverseDependencies.mockReturnValue( Promise.resolve({ @@ -543,18 +519,10 @@ describe.each(['linux', 'win32'])('DeltaCalculator (%s)', osPlatform => { await deltaCalculator.getDelta({reset: false, shallow: false}); // Delete a file - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'delete', filePath: p('/foo'), metadata: {type: 'f'}}, - ], - }); + emitChange({removedFiles: ['foo']}); // Delete a dependency of the deleted file - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'delete', filePath: p('/qux'), metadata: {type: 'f'}}, - ], - }); + emitChange({removedFiles: ['qux']}); traverseDependencies.mockReturnValue( Promise.resolve({ @@ -576,18 +544,10 @@ describe.each(['linux', 'win32'])('DeltaCalculator (%s)', osPlatform => { await deltaCalculator.getDelta({reset: false, shallow: false}); // First delete a file - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'delete', filePath: p('/foo'), metadata: {type: 'f'}}, - ], - }); + emitChange({removedFiles: ['foo']}); // Then add it again - fileWatcher.emit('change', { - eventsQueue: [ - {type: 'change', filePath: p('/foo'), metadata: {type: 'f'}}, - ], - }); + emitChange({modifiedFiles: ['foo']}); traverseDependencies.mockReturnValue( Promise.resolve({ @@ -612,11 +572,11 @@ describe.each(['linux', 'win32'])('DeltaCalculator (%s)', osPlatform => { deltaCalculator.once('change', resolve), ); - fileWatcher.emit('change', { - eventsQueue: [ - {type: eventType, filePath: p('/link'), metadata: {type: 'l'}}, - ], - }); + if (eventType === 'add') { + emitChange({addedFiles: [['link', {isSymlink: true}]]}); + } else { + emitChange({removedFiles: [['link', {isSymlink: true}]]}); + } // Any symlink change should trigger a 'change' event await changeEmitted; @@ -658,15 +618,7 @@ describe.each(['linux', 'win32'])('DeltaCalculator (%s)', osPlatform => { deltaCalculator.once('change', resolve), ); - fileWatcher.emit('change', { - eventsQueue: [ - { - type: 'change', - filePath: p('/node_modules/foo/package.json'), - metadata: {type: 'f'}, - }, - ], - }); + emitChange({modifiedFiles: ['node_modules/foo/package.json']}); // Any package.json change should trigger a 'change' event await changeEmitted; diff --git a/packages/metro/src/DeltaBundler/__tests__/test-utils.js b/packages/metro/src/DeltaBundler/__tests__/test-utils.js new file mode 100644 index 0000000000..f75b4c54a9 --- /dev/null +++ b/packages/metro/src/DeltaBundler/__tests__/test-utils.js @@ -0,0 +1,61 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @format + * @oncall react_native + */ + +import type {EventEmitter} from 'events'; + +export type FileEntry = + | string + | [string, {isSymlink?: boolean, modifiedTime?: number}]; + +export type ChangeEventInput = { + addedFiles?: ReadonlyArray, + modifiedFiles?: ReadonlyArray, + removedFiles?: ReadonlyArray, +}; + +/** + * Creates an emitChange helper function for DeltaCalculator tests. + * The helper emits change events with canonical paths relative to rootDir. + */ +export function createEmitChange( + fileWatcher: EventEmitter, + rootDir: string, + pathSeparator: string = '/', +): (changes: ChangeEventInput) => void { + return function emitChange(changes: ChangeEventInput): void { + const toEntry = ( + entry: FileEntry, + ): [string, {modifiedTime: ?number, isSymlink: boolean}] => { + const [file, opts] = typeof entry === 'string' ? [entry, {}] : entry; + // Convert forward slashes to platform-specific separators for canonical paths + const canonicalPath = + pathSeparator !== '/' ? file.replaceAll('/', '\\') : file; + return [ + canonicalPath, + { + modifiedTime: opts.modifiedTime ?? Date.now(), + isSymlink: opts.isSymlink ?? false, + }, + ]; + }; + fileWatcher.emit('change', { + changes: { + addedFiles: (changes.addedFiles ?? []).map(toEntry), + modifiedFiles: (changes.modifiedFiles ?? []).map(toEntry), + removedFiles: (changes.removedFiles ?? []).map(toEntry), + addedDirectories: [], + removedDirectories: [], + }, + rootDir, + logger: null, + }); + }; +} diff --git a/packages/metro/src/DeltaBundler/types.js b/packages/metro/src/DeltaBundler/types.js index d819ce149c..5e85e7b3e9 100644 --- a/packages/metro/src/DeltaBundler/types.js +++ b/packages/metro/src/DeltaBundler/types.js @@ -13,6 +13,7 @@ import type {RequireContext} from '../lib/contextModule'; import type {RequireContextParams} from '../ModuleGraph/worker/collectDependencies'; import type {ReadonlySourceLocation} from '../shared/types'; import type {Graph} from './Graph'; +import type {InvalidationData} from 'metro-file-map'; import type {JsTransformOptions} from 'metro-transform-worker'; import CountingSet from '../lib/CountingSet'; @@ -21,9 +22,7 @@ export type MixedOutput = { +data: unknown, +type: string, }; - export type AsyncDependencyType = 'async' | 'maybeSync' | 'prefetch' | 'weak'; - export type TransformResultDependency = Readonly<{ /** * The literal name provided to a require or import call. For example 'foo' in @@ -59,18 +58,15 @@ export type TransformResultDependency = Readonly<{ contextParams?: RequireContextParams, }>, }>; - export type ResolvedDependency = Readonly<{ absolutePath: string, data: TransformResultDependency, }>; - export type Dependency = | ResolvedDependency | Readonly<{ data: TransformResultDependency, }>; - export type Module = Readonly<{ dependencies: Map, inverseDependencies: CountingSet, @@ -79,7 +75,6 @@ export type Module = Readonly<{ getSource: () => Buffer, unstable_transformResultKey?: ?string, }>; - export type ModuleData = Readonly<{ dependencies: ReadonlyMap, resolvedContexts: ReadonlyMap, @@ -87,49 +82,40 @@ export type ModuleData = Readonly<{ getSource: () => Buffer, unstable_transformResultKey?: ?string, }>; - export type Dependencies = Map>; export type ReadOnlyDependencies = ReadonlyMap< string, Module, >; - export type TransformInputOptions = Omit< JsTransformOptions, 'inlinePlatform' | 'inlineRequires', >; - export type GraphInputOptions = Readonly<{ entryPoints: ReadonlySet, // Unused in core but useful for custom serializers / experimentalSerializerHook transformOptions: TransformInputOptions, }>; - export interface ReadOnlyGraph { +entryPoints: ReadonlySet; // Unused in core but useful for custom serializers / experimentalSerializerHook +transformOptions: Readonly; +dependencies: ReadOnlyDependencies; } - export type {Graph}; - export type TransformResult = Readonly<{ dependencies: ReadonlyArray, output: ReadonlyArray, unstable_transformResultKey?: ?string, }>; - export type TransformResultWithSource = Readonly<{ ...TransformResult, getSource: () => Buffer, }>; - export type TransformFn = ( string, ?RequireContext, ) => Promise>; - export type ResolveFn = ( from: string, dependency: TransformResultDependency, @@ -145,6 +131,12 @@ export type AllowOptionalDependencies = export type BundlerResolution = Readonly<{ type: 'sourceFile', filePath: string, + /** + * Present when `unstable_incrementalResolution` is enabled. Describes the + * file system observations that, if changed, would require re-resolving. + * All paths are canonical (root-relative). + */ + unstable_invalidations?: InvalidationData, }>; export type Options = Readonly<{ diff --git a/packages/metro/src/node-haste/DependencyGraph.js b/packages/metro/src/node-haste/DependencyGraph.js index 97761f3559..ed2d5ea7a9 100644 --- a/packages/metro/src/node-haste/DependencyGraph.js +++ b/packages/metro/src/node-haste/DependencyGraph.js @@ -22,6 +22,7 @@ import type { FileSystem, HasteMap, HealthCheckResult, + InvalidationData, WatcherStatus, default as MetroFileMap, } from 'metro-file-map'; @@ -29,6 +30,7 @@ import type {FileSystemLookup} from 'metro-resolver'; import createFileMap from './DependencyGraph/createFileMap'; import {ModuleResolver} from './DependencyGraph/ModuleResolution'; +import TrackedFileAccess from './DependencyGraph/TrackedFileAccess'; import {PackageCache} from './PackageCache'; import EventEmitter from 'events'; import fs from 'fs'; @@ -67,6 +69,7 @@ export default class DependencyGraph extends EventEmitter { _hasteMap: HasteMap; #dependencyPlugin: ?DependencyPlugin; _moduleResolver: ModuleResolver; + #trackedFileAccess: ?TrackedFileAccess; _resolutionCache: Map< // Custom resolver options string | symbol, @@ -150,10 +153,14 @@ export default class DependencyGraph extends EventEmitter { await this._initializedPromise; } - _onHasteChange({eventsQueue}: ChangeEvent) { + _onHasteChange({changes, rootDir}: ChangeEvent) { this._resolutionCache = new Map(); - eventsQueue.forEach(({filePath}) => - this.#packageCache.invalidate(filePath), + [ + ...changes.addedFiles, + ...changes.modifiedFiles, + ...changes.removedFiles, + ].forEach(([canonicalPath]) => + this.#packageCache.invalidate(path.join(rootDir, canonicalPath)), ); this._createModuleResolver(); this.emit('change'); @@ -172,6 +179,19 @@ export default class DependencyGraph extends EventEmitter { return {exists: false}; }; + const useTracking = this._config.resolver.unstable_incrementalResolution; + const trackedFileAccess = useTracking + ? new TrackedFileAccess( + this._fileSystem, + this._config.projectRoot, + this.#packageCache, + (name, platform) => this._hasteMap.getModule(name, platform, true), + (name, platform) => this._hasteMap.getPackage(name, platform, true), + this._config.resolver.assetResolutions, + ) + : null; + this.#trackedFileAccess = trackedFileAccess; + this._moduleResolver = new ModuleResolver({ assetExts: new Set(this._config.resolver.assetExts), dirExists: (filePath: string) => { @@ -182,33 +202,43 @@ export default class DependencyGraph extends EventEmitter { }, disableHierarchicalLookup: this._config.resolver.disableHierarchicalLookup, - doesFileExist: this.doesFileExist, + doesFileExist: trackedFileAccess?.doesFileExist ?? this.doesFileExist, emptyModulePath: this._config.resolver.emptyModulePath, extraNodeModules: this._config.resolver.extraNodeModules, - fileSystemLookup, - getHasteModulePath: (name, platform) => - this._hasteMap.getModule(name, platform, true), - getHastePackagePath: (name, platform) => - this._hasteMap.getPackage(name, platform, true), + fileSystemLookup: trackedFileAccess?.fileSystemLookup ?? fileSystemLookup, + getHasteModulePath: + trackedFileAccess != null + ? (name: string, _platform: ?string) => + trackedFileAccess.resolveHasteModule(name) + : (name: string, platform: ?string) => + this._hasteMap.getModule(name, platform, true), + getHastePackagePath: + trackedFileAccess != null + ? (name: string, _platform: ?string) => + trackedFileAccess.resolveHastePackage(name) + : (name: string, platform: ?string) => + this._hasteMap.getPackage(name, platform, true), mainFields: this._config.resolver.resolverMainFields, nodeModulesPaths: this._config.resolver.nodeModulesPaths, packageCache: this.#packageCache, preferNativePlatform: true, projectRoot: this._config.projectRoot, reporter: this._config.reporter, - resolveAsset: (dirPath: string, assetName: string, extension: string) => { - const basePath = dirPath + path.sep + assetName; - const assets = [ - basePath + extension, - ...this._config.resolver.assetResolutions.map( - resolution => basePath + '@' + resolution + 'x' + extension, - ), - ] - .map(assetPath => fileSystemLookup(assetPath).realPath) - .filter(Boolean); - - return assets.length ? assets : null; - }, + resolveAsset: + trackedFileAccess?.resolveAsset ?? + ((dirPath: string, assetName: string, extension: string) => { + const basePath = dirPath + path.sep + assetName; + const assets = [ + basePath + extension, + ...this._config.resolver.assetResolutions.map( + resolution => basePath + '@' + resolution + 'x' + extension, + ), + ] + .map(assetPath => fileSystemLookup(assetPath).realPath) + .filter(Boolean); + + return assets.length ? assets : null; + }), resolveRequest: this._config.resolver.resolveRequest, sourceExts: this._config.resolver.sourceExts, unstable_conditionNames: this._config.resolver.unstable_conditionNames, @@ -223,13 +253,14 @@ export default class DependencyGraph extends EventEmitter { _getClosestPackage( absoluteModulePath: string, + invalidatedBy: ?InvalidationData, ): ?{packageJsonPath: string, packageRelativePath: string} { const result = this._fileSystem.hierarchicalLookup( absoluteModulePath, 'package.json', { breakOnSegment: 'node_modules', - invalidatedBy: null, + invalidatedBy, subpathType: 'f', }, ); @@ -243,7 +274,8 @@ export default class DependencyGraph extends EventEmitter { _createPackageCache(): PackageCache { return new PackageCache({ - getClosestPackage: absolutePath => this._getClosestPackage(absolutePath), + getClosestPackage: (absolutePath, invalidatedBy) => + this._getClosestPackage(absolutePath, invalidatedBy), }); } @@ -309,38 +341,52 @@ export default class DependencyGraph extends EventEmitter { }, ): BundlerResolution { const to = dependency.name; - const isSensitiveToOriginFolder = - // Resolution is always relative to the origin folder unless we assume a flat node_modules - !assumeFlatNodeModules || - // Path requests are resolved relative to the origin folder - to.includes('/') || - to === '.' || - to === '..' || - // Preserve standard assumptions under node_modules - originModulePath.includes(path.sep + 'node_modules' + path.sep); - - // Compound key for the resolver cache - const resolverOptionsKey = - JSON.stringify(resolverOptions ?? {}, canonicalize) ?? ''; - const originKey = isSensitiveToOriginFolder - ? path.dirname(originModulePath) - : ''; - const targetKey = - to + (dependency.data.isESMImport === true ? '\0esm' : '\0cjs'); - const platformKey = platform ?? NULL_PLATFORM; - - // Traverse the resolver cache, which is a tree of maps - const mapByResolverOptions = this._resolutionCache; - const mapByOrigin = getOrCreateMap( - mapByResolverOptions, - resolverOptionsKey, - ); - const mapByTarget = getOrCreateMap(mapByOrigin, originKey); - const mapByPlatform = getOrCreateMap(mapByTarget, targetKey); - let resolution: ?BundlerResolution = mapByPlatform.get(platformKey); + let resolution: ?BundlerResolution; + let updateResolverCache: ?(BundlerResolution) => void; + + // Don't use the resolver cache under unstable_incrementalResolution, since + // this would bypass collection of invalidations. + if (!this._config.resolver.unstable_incrementalResolution) { + const isSensitiveToOriginFolder = + // Resolution is always relative to the origin folder unless we assume a flat node_modules + !assumeFlatNodeModules || + // Path requests are resolved relative to the origin folder + to.includes('/') || + to === '.' || + to === '..' || + // Preserve standard assumptions under node_modules + originModulePath.includes(path.sep + 'node_modules' + path.sep); + + // Compound key for the resolver cache + const resolverOptionsKey = + JSON.stringify(resolverOptions ?? {}, canonicalize) ?? ''; + const originKey = isSensitiveToOriginFolder + ? path.dirname(originModulePath) + : ''; + const targetKey = + to + (dependency.data.isESMImport === true ? '\0esm' : '\0cjs'); + const platformKey = platform ?? NULL_PLATFORM; + + // Traverse the resolver cache, which is a tree of maps + const mapByResolverOptions = this._resolutionCache; + const mapByOrigin = getOrCreateMap( + mapByResolverOptions, + resolverOptionsKey, + ); + const mapByTarget = getOrCreateMap(mapByOrigin, originKey); + const mapByPlatform = getOrCreateMap(mapByTarget, targetKey); + + updateResolverCache = (result: BundlerResolution) => { + mapByPlatform.set(platformKey, result); + }; + + // Check the cache + resolution = mapByPlatform.get(platformKey); + } if (!resolution) { try { + const invalidations = this.#trackedFileAccess?.startTracking(platform); resolution = this._moduleResolver.resolveDependency( originModulePath, dependency, @@ -348,6 +394,13 @@ export default class DependencyGraph extends EventEmitter { platform, resolverOptions, ); + if (invalidations != null) { + resolution = { + ...resolution, + unstable_invalidations: invalidations, + }; + } + updateResolverCache?.(resolution); } catch (error) { if (error instanceof DuplicateHasteCandidatesError) { throw new AmbiguousModuleResolutionError(originModulePath, error); @@ -363,7 +416,6 @@ export default class DependencyGraph extends EventEmitter { } } - mapByPlatform.set(platformKey, resolution); return resolution; } diff --git a/packages/metro/src/node-haste/DependencyGraph/ModuleResolution.js b/packages/metro/src/node-haste/DependencyGraph/ModuleResolution.js index 43bcbd0f68..8ce77f3a36 100644 --- a/packages/metro/src/node-haste/DependencyGraph/ModuleResolution.js +++ b/packages/metro/src/node-haste/DependencyGraph/ModuleResolution.js @@ -25,7 +25,7 @@ import type { } from 'metro-resolver'; import type {PackageForModule, PackageJson} from 'metro-resolver/private/types'; -import {codeFrameColumns} from '@babel/code-frame'; +import {codeFrameColumns} from '@babel/code-frame/lib'; import fs from 'fs'; import invariant from 'invariant'; import * as Resolver from 'metro-resolver'; diff --git a/packages/metro/src/node-haste/DependencyGraph/TrackedFileAccess.js b/packages/metro/src/node-haste/DependencyGraph/TrackedFileAccess.js new file mode 100644 index 0000000000..4233167c8c --- /dev/null +++ b/packages/metro/src/node-haste/DependencyGraph/TrackedFileAccess.js @@ -0,0 +1,198 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @flow strict-local + * @format + * @oncall react_native + */ + +import type {PackageCache} from '../PackageCache'; +import type {FileSystem, InvalidationData} from 'metro-file-map'; +import type { + DoesFileExist, + FileSystemLookup, + ResolveAsset, +} from 'metro-resolver'; +import type {PackageForModule, PackageJson} from 'metro-resolver/private/types'; + +import path from 'path'; + +/** + * Wraps resolver I/O to track file system observations for incremental + * resolution invalidation. Created once per ModuleResolver and reused across + * resolutions. Per-resolution state is managed via startTracking(), which + * returns a fresh InvalidationData that the instance writes to until the + * next call to startTracking() (or if tracking is not active, no-ops). + * + * By wrapping at this level (rather than downstream of DependencyGraph's + * fileSystemLookup wrapper), we have access to full LookupResult data from + * TreeFS, including `missing` (canonical path of the first missing segment) + * and `links` (symlinks traversed). + */ +export default class TrackedFileAccess { + #fileSystem: FileSystem; + #projectRoot: string; + #packageCache: PackageCache; + #getHasteModulePath: (name: string, platform: ?string) => ?string; + #getHastePackagePath: (name: string, platform: ?string) => ?string; + #assetResolutions: ReadonlyArray; + + #currentTarget: ?InvalidationData = null; + #platform: string | null = null; + + constructor( + fileSystem: FileSystem, + projectRoot: string, + packageCache: PackageCache, + getHasteModulePath: (name: string, platform: ?string) => ?string, + getHastePackagePath: (name: string, platform: ?string) => ?string, + assetResolutions: ReadonlyArray, + ) { + this.#fileSystem = fileSystem; + this.#projectRoot = projectRoot; + this.#packageCache = packageCache; + this.#getHasteModulePath = getHasteModulePath; + this.#getHastePackagePath = getHastePackagePath; + this.#assetResolutions = assetResolutions; + } + + /** + * Begin tracking for a new resolution. Returns a fresh InvalidationData + * that will be populated by subsequent calls to wrapped methods. + */ + startTracking(platform: string | null): InvalidationData { + const target: InvalidationData = { + existence: new Set(), + modification: new Set(), + haste: new Set(), + }; + this.#currentTarget = target; + this.#platform = platform; + return target; + } + + #toCanonical(absolutePath: string): string { + return path.relative(this.#projectRoot, absolutePath); + } + + doesFileExist: DoesFileExist = (filePath: string) => { + const result = this.#fileSystem.exists(filePath); + const target = this.#currentTarget; + if (target != null) { + target.existence.add(this.#toCanonical(filePath)); + } + return result; + }; + + fileSystemLookup: FileSystemLookup = ( + absoluteOrProjectRelativePath: string, + ) => { + const result = this.#fileSystem.lookup(absoluteOrProjectRelativePath); + const target = this.#currentTarget; + if (target != null) { + if (result.exists) { + target.existence.add(this.#toCanonical(result.realPath)); + for (const link of result.links) { + target.modification.add(this.#toCanonical(link)); + } + } else { + target.existence.add(this.#toCanonical(result.missing)); + for (const link of result.links) { + target.modification.add(this.#toCanonical(link)); + } + } + } + if (result.exists) { + return { + exists: true, + realPath: result.realPath, + type: result.type, + }; + } + return {exists: false}; + }; + + resolveAsset: ResolveAsset = ( + dirPath: string, + assetName: string, + extension: string, + ) => { + const basePath = dirPath + path.sep + assetName; + const assets = [ + basePath + extension, + ...this.#assetResolutions.map( + resolution => basePath + '@' + resolution + 'x' + extension, + ), + ] + .map(candidate => this.fileSystemLookup(candidate).realPath) + .filter(Boolean); + + return assets.length ? assets : null; + }; + + getPackage: (packageJsonPath: string) => ?PackageJson = ( + packageJsonPath: string, + ) => { + try { + const result = this.#packageCache.getPackage(packageJsonPath).read(); + const target = this.#currentTarget; + if (target != null && result != null) { + const canonical = this.#toCanonical(packageJsonPath); + target.modification.add(canonical); + target.existence.delete(canonical); + } + return result; + } catch (e) { + return null; + } + }; + + getPackageForModule: (absoluteModulePath: string) => ?PackageForModule = ( + absoluteModulePath: string, + ) => { + let result; + try { + result = this.#packageCache.getPackageOf( + absoluteModulePath, + this.#currentTarget, + ); + } catch (e) { + // Do nothing. + } + if (result != null) { + const target = this.#currentTarget; + if (target != null) { + const canonical = this.#toCanonical( + path.join(path.dirname(result.pkg.path), 'package.json'), + ); + target.modification.add(canonical); + target.existence.delete(canonical); + } + return { + packageJson: result.pkg.read(), + packageRelativePath: result.packageRelativePath, + rootPath: path.dirname(result.pkg.path), + }; + } + return null; + }; + + resolveHasteModule: (name: string) => ?string = (name: string) => { + const target = this.#currentTarget; + if (target != null) { + target.haste.add(name); + } + return this.#getHasteModulePath(name, this.#platform); + }; + + resolveHastePackage: (name: string) => ?string = (name: string) => { + const target = this.#currentTarget; + if (target != null) { + target.haste.add(name); + } + return this.#getHastePackagePath(name, this.#platform); + }; +} diff --git a/packages/metro/src/node-haste/PackageCache.js b/packages/metro/src/node-haste/PackageCache.js index 74323a719b..2ad7dda69d 100644 --- a/packages/metro/src/node-haste/PackageCache.js +++ b/packages/metro/src/node-haste/PackageCache.js @@ -9,9 +9,14 @@ * @oncall react_native */ +import type {InvalidationData} from 'metro-file-map'; + import Package from './Package'; -type GetClosestPackageFn = (absoluteFilePath: string) => ?{ +type GetClosestPackageFn = ( + absoluteFilePath: string, + invalidatedBy: ?InvalidationData, +) => ?{ packageJsonPath: string, packageRelativePath: string, }; @@ -28,6 +33,9 @@ export class PackageCache { [filePath: string]: ?{ packageJsonPath: string, packageRelativePath: string, + // Canonical paths observed during hierarchicalLookup. Stored so they + // can be replayed into the caller's InvalidationData on a cache hit. + storedInvalidatedBy: ?InvalidationData, }, __proto__: null, ... @@ -57,28 +65,79 @@ export class PackageCache { getPackageOf( absoluteModulePath: string, + invalidatedBy?: ?InvalidationData, ): ?{pkg: Package, packageRelativePath: string} { - let packagePathAndSubpath = - this._packagePathAndSubpathByModulePath[absoluteModulePath]; - if ( - packagePathAndSubpath && - this._packageCache[packagePathAndSubpath.packageJsonPath] - ) { + const cached = this._packagePathAndSubpathByModulePath[absoluteModulePath]; + if (cached && this._packageCache[cached.packageJsonPath]) { + // Cache hit: replay stored invalidation paths into the caller's sets. + if (invalidatedBy != null) { + const stored = cached.storedInvalidatedBy; + if (stored != null) { + for (const p of stored.existence) { + invalidatedBy.existence.add(p); + } + for (const p of stored.modification) { + invalidatedBy.modification.add(p); + } + } else { + // No stored paths (previous caller was not tracking). Re-run + // hierarchicalLookup to collect them, then store for future hits. + const freshInvalidatedBy = { + existence: new Set(), + modification: new Set(), + haste: new Set(), + }; + this._getClosestPackage(absoluteModulePath, freshInvalidatedBy); + cached.storedInvalidatedBy = freshInvalidatedBy; + for (const p of freshInvalidatedBy.existence) { + invalidatedBy.existence.add(p); + } + for (const p of freshInvalidatedBy.modification) { + invalidatedBy.modification.add(p); + } + } + } return { - pkg: this._packageCache[packagePathAndSubpath.packageJsonPath], - packageRelativePath: packagePathAndSubpath.packageRelativePath, + pkg: this._packageCache[cached.packageJsonPath], + packageRelativePath: cached.packageRelativePath, }; } - packagePathAndSubpath = this._getClosestPackage(absoluteModulePath); - if (!packagePathAndSubpath) { + // Cache miss: allocate fresh InvalidationData so we don't store the + // caller's pre-existing paths in the cache. + const isTracking = invalidatedBy != null; + let freshInvalidatedBy: ?InvalidationData = null; + if (isTracking) { + freshInvalidatedBy = { + existence: new Set(), + modification: new Set(), + haste: new Set(), + }; + } + const closestPackage = this._getClosestPackage( + absoluteModulePath, + freshInvalidatedBy, + ); + if (!closestPackage) { return null; } - const packagePath = packagePathAndSubpath.packageJsonPath; + // Copy fresh paths into the caller's sets. + if (invalidatedBy != null && freshInvalidatedBy != null) { + for (const p of freshInvalidatedBy.existence) { + invalidatedBy.existence.add(p); + } + for (const p of freshInvalidatedBy.modification) { + invalidatedBy.modification.add(p); + } + } - this._packagePathAndSubpathByModulePath[absoluteModulePath] = - packagePathAndSubpath; + const packagePath = closestPackage.packageJsonPath; + + this._packagePathAndSubpathByModulePath[absoluteModulePath] = { + ...closestPackage, + storedInvalidatedBy: freshInvalidatedBy, + }; const modulePaths = this._modulePathsByPackagePath[packagePath] ?? new Set(); modulePaths.add(absoluteModulePath); @@ -86,7 +145,7 @@ export class PackageCache { return { pkg: this.getPackage(packagePath), - packageRelativePath: packagePathAndSubpath.packageRelativePath, + packageRelativePath: closestPackage.packageRelativePath, }; } diff --git a/packages/metro/types/DeltaBundler/DeltaCalculator.d.ts b/packages/metro/types/DeltaBundler/DeltaCalculator.d.ts index 36f9b61dff..1f431c0fc5 100644 --- a/packages/metro/types/DeltaBundler/DeltaCalculator.d.ts +++ b/packages/metro/types/DeltaBundler/DeltaCalculator.d.ts @@ -6,7 +6,7 @@ * * @noformat * @oncall react_native - * @generated SignedSource<> + * @generated SignedSource<> * * This file was translated from Flow by scripts/generateTypeScriptDefinitions.js * Original file: packages/metro/src/DeltaBundler/DeltaCalculator.js @@ -16,7 +16,6 @@ */ import type {DeltaResult, Options} from './types'; -import type {RootPerfLogger} from 'metro-config'; import type {ChangeEvent} from 'metro-file-map'; import {Graph} from './Graph'; @@ -60,15 +59,6 @@ declare class DeltaCalculator extends EventEmitter { */ getGraph(): Graph; _handleMultipleFileChanges: (changeEvent: ChangeEvent) => void; - /** - * Handles a single file change. To avoid doing any work before it's needed, - * the listener only stores the modified file, which will then be used later - * when the delta needs to be calculated. - */ - _handleFileChange: ( - $$PARAM_0$$: ChangeEvent['eventsQueue'][number], - logger: null | undefined | RootPerfLogger, - ) => unknown; _getChangedDependencies( modifiedFiles: Set, deletedFiles: Set,