fix: not enough information to count sequential failures

packages/@aws-cdk-testing/cli-integ/tests/telemetry-integ-tests/cdk-deploy-sequential-errors.ts

@@ -0,0 +1,49 @@
+import * as path from 'path';
+import * as fs from 'fs-extra';
+import { integTest, withSpecificFixture } from '../../lib';
+
+integTest(
+  'multiple deploys track their failures',
+  withSpecificFixture('diagnose-app', async (fixture) => {
+    const telemetryFile = path.join(fixture.integTestDir, 'telemetry.json');
+
+    // Deploy a stack that will fail (IAM Policy without PolicyDocument)
+    await fixture.cdkDeploy('diagnose-deploy-fail', {
+      allowErrExit: true,
+      telemetryFile,
+    });
+
+    await fixture.cdkDeploy('diagnose-deploy-fail', {
+      allowErrExit: true,
+      telemetryFile,
+    });
+
+    // Should see a DEPLOY event with sequentialDeploymentFailures: 2
+    const json = fs.readJSONSync(telemetryFile);
+    expect(json).toEqual([
+      expect.objectContaining({
+        event: expect.objectContaining({
+          eventType: 'SYNTH',
+        }),
+      }),
+      expect.objectContaining({
+        event: expect.objectContaining({
+          eventType: 'ASSET',
+        }),
+      }),
+      expect.objectContaining({
+        event: expect.objectContaining({
+          eventType: 'DEPLOY',
+        }),
+        counters: expect.objectContaining({
+          sequentialDeploymentFailures: 2,
+        }),
+      }),
+      expect.objectContaining({
+        event: expect.objectContaining({
+          eventType: 'INVOKE',
+        }),
+      }),
+    ]);
+  }),
+);

packages/@aws-cdk-testing/cli-integ/tests/telemetry-integ-tests/cdk-deploy-telemetry.integtest.ts

@@ -56,6 +56,9 @@ integTest(
         identifiers: expect.objectContaining({
           eventId: expect.stringContaining(':3'),
         }),
+        counters: expect.objectContaining({
+          sequentialDeploymentFailures: 0,
+        }),
       }),
       expect.objectContaining({
         event: expect.objectContaining({

packages/aws-cdk/lib/cli/telemetry/installation-id.ts

@@ -4,21 +4,17 @@ import * as path from 'node:path';
 import type { IoHelper } from '../../api-private';
 import { cdkCacheDir } from '../../util';
 
-const INSTALLATION_ID_PATH = path.join(cdkCacheDir(), 'installation-id.json');
-
 /**
  * Get or create installation id
  */
 export async function getOrCreateInstallationId(ioHelper: IoHelper) {
-  try {
-    // Create the cache directory if it doesn't exist
-    if (!fs.existsSync(path.dirname(INSTALLATION_ID_PATH))) {
-      fs.mkdirSync(path.dirname(INSTALLATION_ID_PATH), { recursive: true });
-    }
+  // Do this quite lazily, so we can mock `cdkCacheDir` during tests
+  const installationIdPath = path.join(cdkCacheDir(), 'installation-id.json');
 
+  try {
     // Check if the installation ID file exists
-    if (fs.existsSync(INSTALLATION_ID_PATH)) {
-      const cachedId = fs.readFileSync(INSTALLATION_ID_PATH, 'utf-8').trim();
+    if (fs.existsSync(installationIdPath)) {
+      const cachedId = fs.readFileSync(installationIdPath, 'utf-8').trim();
 
       // Validate that the cached ID is a valid UUID
       const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
@@ -31,11 +27,12 @@ export async function getOrCreateInstallationId(ioHelper: IoHelper) {
     // Create a new installation ID
     const newId = randomUUID();
     try {
-      fs.writeFileSync(INSTALLATION_ID_PATH, newId);
+      await ensureCacheDir();
+      fs.writeFileSync(installationIdPath, newId);
     } catch (e: any) {
       // If we can't write the file, still return the generated ID
       // but log a trace message about the failure
-      await ioHelper.defaults.trace(`Failed to write installation ID to ${INSTALLATION_ID_PATH}: ${e}`);
+      await ioHelper.defaults.trace(`Failed to write installation ID to ${installationIdPath}: ${e}`);
     }
     return newId;
   } catch (e: any) {
@@ -45,3 +42,14 @@ export async function getOrCreateInstallationId(ioHelper: IoHelper) {
     return randomUUID();
   }
 }
+
+/**
+ * Make sure the cache dir exists
+ *
+ * Not part of `directories.ts` because that module is mocked in tests in order to mock the temporary directory
+ * to write to, and then I need to mock this function as well.
+ */
+export async function ensureCacheDir() {
+  await fs.promises.mkdir(cdkCacheDir(), { recursive: true });
+}
+

packages/aws-cdk/lib/cli/telemetry/session.ts

@@ -14,6 +14,7 @@ import { CLI_PRIVATE_SPAN } from '../telemetry/messages';
 import { isCI } from '../util/ci';
 import { versionNumber } from '../version';
 import { USER_INTERRUPTED_CODE } from './error';
+import { withTelemetryState } from './telemetry-state';
 
 const ABORTED_ERROR_MESSAGE = '__CDK-Toolkit__Aborted';
 
@@ -231,6 +232,10 @@ export class TelemetrySession {
     };
     this._nextEventCounters = undefined;
 
+    if (event.eventType == 'DEPLOY') {
+      await this.trackDeployStatistics(event.error === undefined, counters);
+    }
+
     return this.client.emit({
       event: {
         command: this.sessionInfo.event.command,
@@ -256,6 +261,26 @@ export class TelemetrySession {
     });
   }
 
+  /**
+   * This is a DEPLOY event, track some additional statistics about it
+   *
+   * We use this to measure things about deployment failures, such as the number of
+   * failed DEPLOY events in a sequence.
+   */
+  private async trackDeployStatistics(isSuccessful: boolean, counters: Record<string, number>) {
+    await withTelemetryState((state) => {
+      const recentFailures = state.sequentialDeploymentFailures ?? 0;
+
+      if (isSuccessful) {
+        state.sequentialDeploymentFailures = 0;
+      } else {
+        state.sequentialDeploymentFailures = recentFailures + 1;
+      }
+
+      counters.sequentialDeploymentFailures = state.sequentialDeploymentFailures;
+    });
+  }
+
   private get sessionInfo(): SessionSchema {
     if (!this._sessionInfo) {
       throw new ToolkitError('SessionNotInitialized', 'Session Info not initialized. Call begin() first.');

packages/aws-cdk/lib/cli/telemetry/telemetry-state.ts

@@ -0,0 +1,53 @@
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { ensureCacheDir } from './installation-id';
+import { cdkCacheDir } from '../../util';
+
+/**
+ * Telemetry state across CLI invocations
+ */
+export interface TelemetryState {
+  /**
+   * How many deployment failures we've seen since the last success.
+   */
+  sequentialDeploymentFailures?: number;
+}
+
+/**
+ * Run a function that can modify the given telemetry state.
+ */
+export async function withTelemetryState<A>(block: (x: TelemetryState) => A): Promise<A> {
+  const state = await loadTelemetryState();
+  const oldState = JSON.stringify(state);
+
+  const ret = block(state);
+
+  // Only write a file if the contents changed.
+  if (JSON.stringify(ret) !== oldState) {
+    await writeTelemetryState(state);
+  }
+
+  return ret;
+}
+
+/**
+ * Load telemetry state
+ */
+export async function loadTelemetryState(): Promise<TelemetryState> {
+  try {
+    return JSON.parse(await fs.readFile(TELEMETRY_STATE_PATH, 'utf-8'));
+  } catch (e: any) {
+    if (e.code === 'ENOENT') {
+      return {};
+    }
+    throw e;
+  }
+}
+
+export async function writeTelemetryState(state: TelemetryState) {
+  await ensureCacheDir();
+
+  await fs.writeFile(TELEMETRY_STATE_PATH, JSON.stringify(state, undefined, 2), 'utf-8');
+}
+
+const TELEMETRY_STATE_PATH = path.join(cdkCacheDir(), 'telemetry-state.json');

packages/aws-cdk/test/cli/telemetry/installation-id.test.ts

@@ -25,24 +25,9 @@ describe(getOrCreateInstallationId, () => {
   let mockIoHelper: IoHelper;
   let traceSpy: jest.Mock;
 
-  beforeAll(() => {
-    // Create the temp directory before any tests run
-    fs.mkdirSync(tempDir, { recursive: true });
-  });
-
   beforeEach(() => {
-    // Clean the temp directory for each test
-    if (fs.existsSync(tempDir)) {
-      const files = fs.readdirSync(tempDir);
-      for (const file of files) {
-        const filePath = path.join(tempDir, file);
-        if (fs.statSync(filePath).isDirectory()) {
-          fs.rmSync(filePath, { recursive: true, force: true });
-        } else {
-          fs.unlinkSync(filePath);
-        }
-      }
-    }
+    fs.rmSync(tempDir, { force: true, recursive: true });
+    fs.mkdirSync(tempDir, { recursive: true });
 
     // Mock randomUUID to return predictable values
     mockRandomUUID.mockReturnValue('12345678-1234-1234-1234-123456789abc');
@@ -130,7 +115,7 @@ describe(getOrCreateInstallationId, () => {
     expect(traceSpy).not.toHaveBeenCalled();
   });
 
-  test('creates cache directory if it does not exist', async() => {
+  test('creates cache directory if it does not exist', async () => {
     // GIVEN
     // Remove the temp directory to test directory creation
     fs.rmSync(tempDir, { recursive: true, force: true });
@@ -171,24 +156,24 @@ describe(getOrCreateInstallationId, () => {
   test('handles general error gracefully and returns temporary ID', async () => {
     // GIVEN
     // Mock fs.existsSync to throw an error
-    const originalExistsSync = fs.existsSync;
-    jest.spyOn(fs, 'existsSync').mockImplementation(() => {
+    const mockExistsSync = jest.spyOn(fs, 'existsSync').mockImplementation(() => {
       throw new Error('Filesystem error');
     });
-
-    // WHEN
-    const result = await getOrCreateInstallationId(mockIoHelper);
-
-    // THEN
-    expect(result).toBe('12345678-1234-1234-1234-123456789abc');
-    expect(mockRandomUUID).toHaveBeenCalledTimes(1);
-
-    // Should have logged a trace message about the general error
-    expect(traceSpy).toHaveBeenCalledWith(
-      expect.stringContaining('Error getting installation ID:'),
-    );
-
-    // Restore original function
-    (fs.existsSync as jest.Mock).mockImplementation(originalExistsSync);
+    try {
+      // WHEN
+      const result = await getOrCreateInstallationId(mockIoHelper);
+
+      // THEN
+      expect(result).toBe('12345678-1234-1234-1234-123456789abc');
+      expect(mockRandomUUID).toHaveBeenCalledTimes(1);
+
+      // Should have logged a trace message about the general error
+      expect(traceSpy).toHaveBeenCalledWith(
+        expect.stringContaining('Error getting installation ID:'),
+      );
+    } finally {
+      // Restore original function
+      mockExistsSync.mockRestore();
+    }
   });
 });