From 50953a06d15f5a52b3d13de7555e26e95a2effff Mon Sep 17 00:00:00 2001 From: fullsend-code <278716306+fullsend-ai-coder[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 10:39:21 +0000 Subject: [PATCH 1/3] feat(#4579): add GitHub AI adoption metric provider Add GithubAiAdoptionMetricProvider to the scorecard GitHub module. The provider calculates AI adoption rates for 7d, 30d, and 90d time ranges by analyzing commit trailers (Co-Authored-By, Co-authored-by, Assisted-by) for known AI tool identifiers (Claude, Cursor, Copilot, Codeium, Cody, Tabnine, Gemini, Amazon Q, Windsurf, Devin, Aider). Changes: - New GithubAiAdoptionMetricProvider with 3 metrics: github.aiAdoptionRate[7d], [30d], [90d] - New getCommitHistory method on GithubClient using GraphQL to fetch commit messages from the default branch - Merge commits are excluded from the ratio calculation - Provider registered in module.ts alongside GithubOpenPRsProvider - Config schema updated in config.d.ts for aiAdoption settings - Comprehensive test suite covering AI detection, merge commit filtering, time range calculation, and edge cases Closes #4579 --- .../add-github-ai-adoption-metric.md | 5 + .../config.d.ts | 6 + .../src/github/GithubClient.ts | 81 ++++ .../src/github/types.ts | 25 ++ .../GithubAiAdoptionMetricProvider.test.ts | 386 ++++++++++++++++++ .../GithubAiAdoptionMetricProvider.ts | 219 ++++++++++ .../src/module.ts | 2 + 7 files changed, 724 insertions(+) create mode 100644 workspaces/scorecard/.changeset/add-github-ai-adoption-metric.md create mode 100644 workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.test.ts create mode 100644 workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.ts diff --git a/workspaces/scorecard/.changeset/add-github-ai-adoption-metric.md b/workspaces/scorecard/.changeset/add-github-ai-adoption-metric.md new file mode 100644 index 00000000000..f79edf8c999 --- /dev/null +++ b/workspaces/scorecard/.changeset/add-github-ai-adoption-metric.md @@ -0,0 +1,5 @@ +--- +'@red-hat-developer-hub/backstage-plugin-scorecard-backend-module-github': minor +--- + +Add AI adoption rate metric provider to the GitHub scorecard module. diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/config.d.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/config.d.ts index ae0c7e1b17f..ca3c306773f 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/config.d.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/config.d.ts @@ -30,6 +30,12 @@ export interface Config { /** How github.openPRs metric values are categorized */ thresholds?: ThresholdConfig; }; + aiAdoption?: { + /** How often github.aiAdoption metrics will be calculated */ + schedule?: SchedulerServiceTaskScheduleDefinitionConfig; + /** How github.aiAdoption metric values are categorized */ + thresholds?: ThresholdConfig; + }; }; }; }; diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GithubClient.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GithubClient.ts index 849e0034051..3d1c4d6b6a7 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GithubClient.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GithubClient.ts @@ -23,6 +23,8 @@ import { import { graphql } from '@octokit/graphql'; import { Octokit } from '@octokit/rest'; import { + GithubCommit, + GithubCommitHistoryQueryResponse, GithubDeployment, GithubWorkflowRun, GithubPullRequest, @@ -408,4 +410,83 @@ export class GithubClient { // normalize to ASC for chronological processing (oldest -> newest). return workflowRuns.reverse(); } + + async getCommitHistory( + url: string, + repository: GithubRepository, + since: Date, + options?: { fetchItemsLimit?: number }, + ): Promise { + const fetchItemsLimit = + options?.fetchItemsLimit ?? DEFAULT_DEPLOYMENT_FETCH_ITEMS_LIMIT; + const octokit = await this.getOctokitClient(url); + const commits: GithubCommit[] = []; + const query = ` + query getCommitHistory($owner: String!, $repo: String!, $since: GitTimestamp!, $after: String) { + repository(owner: $owner, name: $repo) { + defaultBranchRef { + target { + ... on Commit { + history(since: $since, first: ${GITHUB_BATCH_SIZE}, after: $after) { + nodes { + message + committedDate + } + pageInfo { + hasNextPage + endCursor + } + totalCount + } + } + } + } + } + } + `; + + let after: string | null = null; + let hasMorePages = true; + + while (hasMorePages && commits.length < fetchItemsLimit) { + const response: GithubCommitHistoryQueryResponse = await octokit(query, { + owner: repository.owner, + repo: repository.repo, + since: since.toISOString(), + after, + }); + + if (!response.repository) { + throw new Error( + `GitHub repository '${repository.owner}/${repository.repo}' was not found or is inaccessible`, + ); + } + + const history = response.repository.defaultBranchRef?.target?.history; + const pageCommits = history?.nodes ?? []; + + if (pageCommits.length === 0) { + break; + } + + for (const commit of pageCommits) { + if (commits.length >= fetchItemsLimit) { + break; + } + if (commit) { + commits.push({ + message: commit.message, + committedDate: commit.committedDate, + }); + } + } + + hasMorePages = + commits.length < fetchItemsLimit && + Boolean(history?.pageInfo.hasNextPage); + after = history?.pageInfo.endCursor ?? null; + } + + return commits; + } } diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/types.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/types.ts index 3e7f045f05a..a4971c5074b 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/types.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/types.ts @@ -82,3 +82,28 @@ export type GithubCommitsPullRequestsQueryResponse = } | null > | null; }; + +export type GithubCommit = { + message: string; + committedDate: string; +}; + +export type GithubCommitHistoryQueryResponse = GraphQlQueryResponseData & { + repository: { + defaultBranchRef?: { + target?: { + history?: { + nodes: Array<{ + message: string; + committedDate: string; + } | null>; + pageInfo: { + hasNextPage: boolean; + endCursor: string | null; + }; + totalCount: number; + } | null; + } | null; + } | null; + } | null; +}; diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.test.ts new file mode 100644 index 00000000000..baeb79e6a7f --- /dev/null +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.test.ts @@ -0,0 +1,386 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { mockServices } from '@backstage/backend-test-utils'; +import { ConfigReader } from '@backstage/config'; +import type { Entity } from '@backstage/catalog-model'; +import { + GithubAiAdoptionMetricProvider, + AI_ADOPTION_RATE_THRESHOLD, + AI_ADOPTION_RATE_TIME_RANGES, +} from './GithubAiAdoptionMetricProvider'; +import { GithubClient } from '../github/GithubClient'; + +jest.mock('@backstage/catalog-model', () => ({ + ...jest.requireActual('@backstage/catalog-model'), + getEntitySourceLocation: jest.fn().mockReturnValue({ + type: 'url', + target: 'https://github.com/org/orgRepo/tree/main/', + }), +})); +jest.mock('../github/GithubClient'); + +describe('GithubAiAdoptionMetricProvider', () => { + const mockedLogger = mockServices.logger.mock(); + + describe('getMetrics', () => { + it('should return 3 metrics for each time range', () => { + const provider = GithubAiAdoptionMetricProvider.fromConfig( + new ConfigReader({}), + { logger: mockedLogger }, + ); + const metrics = provider.getMetrics(); + expect(metrics).toHaveLength(3); + expect(metrics.map(m => m.id)).toEqual([ + 'github.aiAdoptionRate[7d]', + 'github.aiAdoptionRate[30d]', + 'github.aiAdoptionRate[90d]', + ]); + }); + + it('should use AI_ADOPTION_RATE_THRESHOLD for all metrics', () => { + const provider = GithubAiAdoptionMetricProvider.fromConfig( + new ConfigReader({}), + { logger: mockedLogger }, + ); + const metrics = provider.getMetrics(); + for (const metric of metrics) { + expect(metric.thresholds).toEqual(AI_ADOPTION_RATE_THRESHOLD); + } + }); + + it('should set type to number for all metrics', () => { + const provider = GithubAiAdoptionMetricProvider.fromConfig( + new ConfigReader({}), + { logger: mockedLogger }, + ); + const metrics = provider.getMetrics(); + for (const metric of metrics) { + expect(metric.type).toBe('number'); + } + }); + }); + + describe('provider identity', () => { + it('should return github as datasource id', () => { + const provider = GithubAiAdoptionMetricProvider.fromConfig( + new ConfigReader({}), + { logger: mockedLogger }, + ); + expect(provider.getProviderDatasourceId()).toBe('github'); + }); + + it('should return github.aiAdoption as provider id', () => { + const provider = GithubAiAdoptionMetricProvider.fromConfig( + new ConfigReader({}), + { logger: mockedLogger }, + ); + expect(provider.getProviderId()).toBe('github.aiAdoption'); + }); + + it('should filter entities with github project-slug annotation', () => { + const provider = GithubAiAdoptionMetricProvider.fromConfig( + new ConfigReader({}), + { logger: mockedLogger }, + ); + const filter = provider.getCatalogFilter(); + expect('metadata.annotations.github.com/project-slug' in filter).toBe( + true, + ); + }); + }); + + describe('constants', () => { + it('should define 3 time ranges', () => { + expect(AI_ADOPTION_RATE_TIME_RANGES).toEqual(['7d', '30d', '90d']); + }); + + it('should define threshold with success >= 0.2, warning >= 0.1, error >= 0', () => { + expect(AI_ADOPTION_RATE_THRESHOLD).toEqual({ + rules: [ + { key: 'success', expression: '>=0.2' }, + { key: 'warning', expression: '>=0.1' }, + { key: 'error', expression: '>=0' }, + ], + }); + }); + }); + + describe('calculateMetrics', () => { + let provider: GithubAiAdoptionMetricProvider; + const mockedGithubClient = GithubClient as jest.MockedClass< + typeof GithubClient + >; + const mockedGithubClientInstance = { + getCommitHistory: jest.fn(), + } as any; + mockedGithubClient.mockImplementation(() => mockedGithubClientInstance); + + const mockEntity: Entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'test-component', + annotations: { + 'github.com/project-slug': 'org/orgRepo', + }, + }, + }; + + beforeEach(() => { + jest.clearAllMocks(); + provider = GithubAiAdoptionMetricProvider.fromConfig( + new ConfigReader({}), + { logger: mockedLogger }, + ); + }); + + it('should return 0 for all ranges when no commits', async () => { + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([]); + + const results = await provider.calculateMetrics(mockEntity); + + expect(results.get('github.aiAdoptionRate[7d]')).toBe(0); + expect(results.get('github.aiAdoptionRate[30d]')).toBe(0); + expect(results.get('github.aiAdoptionRate[90d]')).toBe(0); + }); + + it('should detect Co-Authored-By with Claude as AI-assisted', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: + 'feat: add feature\n\nCo-Authored-By: Claude ', + committedDate: now.toISOString(), + }, + { + message: 'fix: regular commit', + committedDate: now.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + expect(results.get('github.aiAdoptionRate[7d]')).toBe(0.5); + expect(results.get('github.aiAdoptionRate[30d]')).toBe(0.5); + expect(results.get('github.aiAdoptionRate[90d]')).toBe(0.5); + }); + + it('should detect Co-authored-by (lowercase) as AI-assisted', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: + 'feat: add feature\n\nCo-authored-by: Copilot ', + committedDate: now.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + expect(results.get('github.aiAdoptionRate[7d]')).toBe(1); + }); + + it('should detect Assisted-by trailer as AI-assisted', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: 'feat: refactor\n\nAssisted-by: Cursor', + committedDate: now.toISOString(), + }, + { + message: 'fix: manual fix', + committedDate: now.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + expect(results.get('github.aiAdoptionRate[7d]')).toBe(0.5); + }); + + it('should ignore merge commits', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: 'Merge pull request #42 from org/feature', + committedDate: now.toISOString(), + }, + { + message: "Merge branch 'main' into feature", + committedDate: now.toISOString(), + }, + { + message: + 'feat: add feature\n\nCo-Authored-By: Claude ', + committedDate: now.toISOString(), + }, + { + message: 'fix: something', + committedDate: now.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + // 2 merge commits ignored, 1 AI-assisted, 1 not = 0.5 + expect(results.get('github.aiAdoptionRate[7d]')).toBe(0.5); + }); + + it('should return 1.0 when all non-merge commits are AI-assisted', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: + 'feat: new thing\n\nCo-Authored-By: Claude ', + committedDate: now.toISOString(), + }, + { + message: 'fix: another\n\nAssisted-by: Cursor', + committedDate: now.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + expect(results.get('github.aiAdoptionRate[7d]')).toBe(1); + }); + + it('should calculate different ratios for different time ranges', async () => { + const now = new Date(); + const threeDaysAgo = new Date(now); + threeDaysAgo.setDate(threeDaysAgo.getDate() - 3); + const fifteenDaysAgo = new Date(now); + fifteenDaysAgo.setDate(fifteenDaysAgo.getDate() - 15); + const sixtyDaysAgo = new Date(now); + sixtyDaysAgo.setDate(sixtyDaysAgo.getDate() - 60); + + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + // Within 7d: 1 AI, 1 not = 0.5 + { + message: + 'feat: recent\n\nCo-Authored-By: Claude ', + committedDate: threeDaysAgo.toISOString(), + }, + { + message: 'fix: recent manual', + committedDate: threeDaysAgo.toISOString(), + }, + // Within 30d (but not 7d): 0 AI, 1 not + { + message: 'chore: older manual commit', + committedDate: fifteenDaysAgo.toISOString(), + }, + // Within 90d (but not 30d): 1 AI, 0 not + { + message: 'feat: old ai\n\nAssisted-by: Cursor', + committedDate: sixtyDaysAgo.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + // 7d: 1 AI / 2 total = 0.5 + expect(results.get('github.aiAdoptionRate[7d]')).toBe(0.5); + // 30d: 1 AI / 3 total = 0.333... + expect(results.get('github.aiAdoptionRate[30d]')).toBeCloseTo(1 / 3, 10); + // 90d: 2 AI / 4 total = 0.5 + expect(results.get('github.aiAdoptionRate[90d]')).toBe(0.5); + }); + + it('should not count Co-authored-by with non-AI authors', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: + 'feat: pair programming\n\nCo-authored-by: John Doe ', + committedDate: now.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + expect(results.get('github.aiAdoptionRate[7d]')).toBe(0); + }); + + it('should detect various AI tools case-insensitively', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: + 'feat: one\n\nCo-authored-by: CLAUDE ', + committedDate: now.toISOString(), + }, + { + message: 'feat: two\n\nAssisted-by: GitHub Copilot', + committedDate: now.toISOString(), + }, + { + message: 'feat: three\n\nAssisted-by: codeium', + committedDate: now.toISOString(), + }, + { + message: 'feat: four\n\nAssisted-by: Tabnine', + committedDate: now.toISOString(), + }, + { + message: 'feat: five\n\nAssisted-by: Amazon Q Developer', + committedDate: now.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + expect(results.get('github.aiAdoptionRate[7d]')).toBe(1); + }); + + it('should log analysis summary', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: 'feat: ai\n\nCo-Authored-By: Claude ', + committedDate: now.toISOString(), + }, + { + message: 'fix: manual', + committedDate: now.toISOString(), + }, + { + message: 'Merge pull request #1 from org/feat', + committedDate: now.toISOString(), + }, + ]); + + await provider.calculateMetrics(mockEntity); + + expect(mockedLogger.info).toHaveBeenCalledWith( + expect.stringContaining('AI adoption [7d]'), + ); + expect(mockedLogger.info).toHaveBeenCalledWith( + expect.stringContaining('3 commits analyzed'), + ); + expect(mockedLogger.info).toHaveBeenCalledWith( + expect.stringContaining('1 merge commits ignored'), + ); + expect(mockedLogger.info).toHaveBeenCalledWith( + expect.stringContaining('1 AI-assisted'), + ); + expect(mockedLogger.info).toHaveBeenCalledWith( + expect.stringContaining('1 not AI-assisted'), + ); + }); + }); +}); diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.ts new file mode 100644 index 00000000000..7e940cd776e --- /dev/null +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.ts @@ -0,0 +1,219 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { LoggerService } from '@backstage/backend-plugin-api'; +import type { Config } from '@backstage/config'; +import { getEntitySourceLocation, type Entity } from '@backstage/catalog-model'; +import { CATALOG_FILTER_EXISTS } from '@backstage/catalog-client'; +import { + Metric, + ThresholdConfig, +} from '@red-hat-developer-hub/backstage-plugin-scorecard-common'; +import { MetricProvider } from '@red-hat-developer-hub/backstage-plugin-scorecard-node'; +import { GithubClient } from '../github/GithubClient'; +import { GithubCommit } from '../github/types'; +import { getRepositoryInformationFromEntity } from '../github/utils'; + +export const AI_ADOPTION_RATE_TIME_RANGES = ['7d', '30d', '90d']; + +export const AI_ADOPTION_RATE_THRESHOLD: ThresholdConfig = { + rules: [ + { key: 'success', expression: '>=0.2' }, + { key: 'warning', expression: '>=0.1' }, + { key: 'error', expression: '>=0' }, + ], +}; + +/** + * Known AI tool identifiers used in commit trailers. + * Matched case-insensitively against the value after + * `Assisted-by: ` or `Co-Authored-By: ` / `Co-authored-by: `. + */ +const AI_TOOL_PATTERNS: string[] = [ + 'claude', + 'cursor', + 'copilot', + 'github copilot', + 'codeium', + 'cody', + 'tabnine', + 'gemini', + 'amazon q', + 'windsurf', + 'devin', + 'aider', +]; + +function parseDays(range: string): number { + return parseInt(range.replace('d', ''), 10); +} + +function isMergeCommit(message: string): boolean { + return ( + message.startsWith('Merge pull request #') || + message.startsWith('Merge branch ') + ); +} + +function isAiAssistedCommit(message: string): boolean { + const lines = message.split('\n'); + for (const line of lines) { + const trimmed = line.trim(); + let value: string | undefined; + + if (trimmed.startsWith('Assisted-by: ')) { + value = trimmed.slice('Assisted-by: '.length).trim(); + } else if (trimmed.startsWith('Co-Authored-By: ')) { + value = trimmed.slice('Co-Authored-By: '.length).trim(); + } else if (trimmed.startsWith('Co-authored-by: ')) { + value = trimmed.slice('Co-authored-by: '.length).trim(); + } + + if (value) { + const lowerValue = value.toLowerCase(); + if (AI_TOOL_PATTERNS.some(tool => lowerValue.startsWith(tool))) { + return true; + } + } + } + return false; +} + +export class GithubAiAdoptionMetricProvider + implements MetricProvider<'number'> +{ + private readonly githubClient: GithubClient; + private readonly logger: LoggerService; + + private constructor(githubClient: GithubClient, logger: LoggerService) { + this.githubClient = githubClient; + this.logger = logger; + } + + static fromConfig( + config: Config, + options: { logger: LoggerService }, + ): GithubAiAdoptionMetricProvider { + return new GithubAiAdoptionMetricProvider( + new GithubClient(config, options.logger), + options.logger, + ); + } + + getProviderDatasourceId(): string { + return 'github'; + } + + getProviderId(): string { + return 'github.aiAdoption'; + } + + getMetrics(): Metric<'number'>[] { + return AI_ADOPTION_RATE_TIME_RANGES.map(range => ({ + id: `${this.getProviderId()}Rate[${range}]`, + title: `GitHub AI adoption rate (${range})`, + description: `Ratio of AI-assisted commits over the last ${range}.`, + type: 'number' as const, + thresholds: AI_ADOPTION_RATE_THRESHOLD, + })); + } + + getCatalogFilter(): Record { + return { + 'metadata.annotations.github.com/project-slug': CATALOG_FILTER_EXISTS, + }; + } + + async calculateMetrics(entity: Entity): Promise> { + const repository = getRepositoryInformationFromEntity(entity); + const { target } = getEntitySourceLocation(entity); + + const maxDays = Math.max(...AI_ADOPTION_RATE_TIME_RANGES.map(parseDays)); + + const since = new Date(); + since.setDate(since.getDate() - maxDays); + + const commits = await this.githubClient.getCommitHistory( + target, + repository, + since, + ); + + const now = new Date(); + const results = new Map(); + + for (const range of AI_ADOPTION_RATE_TIME_RANGES) { + const days = parseDays(range); + const cutoff = new Date(now); + cutoff.setDate(cutoff.getDate() - days); + + const rangeCommits = commits.filter( + c => new Date(c.committedDate) >= cutoff, + ); + + const { ratio, total, ignored, aiAssisted, notAiAssisted } = + this.analyzeCommits(rangeCommits); + + this.logger.info( + `AI adoption [${range}] for ${repository.owner}/${repository.repo}: ` + + `${total} commits analyzed, ${ignored} merge commits ignored, ` + + `${aiAssisted} AI-assisted, ${notAiAssisted} not AI-assisted, ` + + `ratio=${ratio}`, + ); + + const metricId = `${this.getProviderId()}Rate[${range}]`; + results.set(metricId, ratio); + } + + return results; + } + + private analyzeCommits(commits: GithubCommit[]): { + ratio: number; + total: number; + ignored: number; + aiAssisted: number; + notAiAssisted: number; + } { + let ignored = 0; + let aiAssisted = 0; + let notAiAssisted = 0; + + for (const commit of commits) { + if (isMergeCommit(commit.message)) { + ignored++; + continue; + } + + if (isAiAssistedCommit(commit.message)) { + aiAssisted++; + } else { + notAiAssisted++; + } + } + + const analyzed = aiAssisted + notAiAssisted; + const ratio = analyzed > 0 ? aiAssisted / analyzed : 0; + + return { + ratio, + total: commits.length, + ignored, + aiAssisted, + notAiAssisted, + }; + } +} diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/module.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/module.ts index a7dc873eb46..5770db1c03e 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/module.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/module.ts @@ -24,6 +24,7 @@ import { import { GithubDeploymentPullRequestsCollector } from './collectors/GithubDeploymentPullRequestsCollector'; import { GithubDeploymentWorkflowRunsCollector } from './collectors/GithubDeploymentWorkflowRunsCollector'; import { GithubDeploymentsCollector } from './collectors/GithubDeploymentsCollector'; +import { GithubAiAdoptionMetricProvider } from './metricProviders/GithubAiAdoptionMetricProvider'; import { GithubOpenPRsProvider } from './metricProviders/GithubOpenPRsProvider'; export const scorecardModuleGithub = createBackendModule({ @@ -45,6 +46,7 @@ export const scorecardModuleGithub = createBackendModule({ ); metrics.addMetricProvider( GithubOpenPRsProvider.fromConfig(config, { logger }), + GithubAiAdoptionMetricProvider.fromConfig(config, { logger }), ); }, }); From 9b1ba869ff1a9ea7e6a50bb7fc680112ae5ef696 Mon Sep 17 00:00:00 2001 From: fullsend-fix <278716306+fullsend-ai-coder[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 15:52:50 +0000 Subject: [PATCH 2/3] fix: address review feedback on PR #4580 - Add history: true to AI adoption metric definitions to match all other scorecard metric providers - Make trailer key matching fully case-insensitive by normalizing to lowercase before prefix comparison - Rename DEFAULT_DEPLOYMENT_FETCH_ITEMS_LIMIT to DEFAULT_FETCH_ITEMS_LIMIT to reflect its broader usage across deployments and commit history Addresses review feedback on #4580 --- .../src/github/GithubClient.ts | 13 +++---- .../src/github/constants.ts | 4 +-- .../GithubAiAdoptionMetricProvider.test.ts | 35 +++++++++++++++++++ .../GithubAiAdoptionMetricProvider.ts | 15 ++++---- 4 files changed, 50 insertions(+), 17 deletions(-) diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GithubClient.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GithubClient.ts index 3d1c4d6b6a7..8c09662ff18 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GithubClient.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GithubClient.ts @@ -32,10 +32,7 @@ import { GithubDeploymentsQueryResponse, GithubCommitsPullRequestsQueryResponse, } from './types'; -import { - DEFAULT_DEPLOYMENT_FETCH_ITEMS_LIMIT, - GITHUB_BATCH_SIZE, -} from './constants'; +import { DEFAULT_FETCH_ITEMS_LIMIT, GITHUB_BATCH_SIZE } from './constants'; import { buildCommitsPullRequestsQuery } from './queries/buildCommitsPullRequestsQuery'; import { mapCommitsPullRequests } from './mappers'; @@ -127,7 +124,7 @@ export class GithubClient { options?: { fetchItemsLimit?: number }, ): Promise { const fetchItemsLimit = - options?.fetchItemsLimit ?? DEFAULT_DEPLOYMENT_FETCH_ITEMS_LIMIT; + options?.fetchItemsLimit ?? DEFAULT_FETCH_ITEMS_LIMIT; const octokit = await this.getOctokitClient(url); const deployments: GithubDeployment[] = []; const query = ` @@ -243,7 +240,7 @@ export class GithubClient { options?: { fetchItemsLimit?: number }, ): Promise { const fetchItemsLimit = - options?.fetchItemsLimit ?? DEFAULT_DEPLOYMENT_FETCH_ITEMS_LIMIT; + options?.fetchItemsLimit ?? DEFAULT_FETCH_ITEMS_LIMIT; const octokit = await this.getOctokitRestClient(url); const basehead = `${baseSha}...${headSha}`; @@ -349,7 +346,7 @@ export class GithubClient { options?: { fetchItemsLimit?: number }, ): Promise { const fetchItemsLimit = - options?.fetchItemsLimit ?? DEFAULT_DEPLOYMENT_FETCH_ITEMS_LIMIT; + options?.fetchItemsLimit ?? DEFAULT_FETCH_ITEMS_LIMIT; const octokit = await this.getOctokitRestClient(url); const workflows = await octokit.paginate( @@ -418,7 +415,7 @@ export class GithubClient { options?: { fetchItemsLimit?: number }, ): Promise { const fetchItemsLimit = - options?.fetchItemsLimit ?? DEFAULT_DEPLOYMENT_FETCH_ITEMS_LIMIT; + options?.fetchItemsLimit ?? DEFAULT_FETCH_ITEMS_LIMIT; const octokit = await this.getOctokitClient(url); const commits: GithubCommit[] = []; const query = ` diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/constants.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/constants.ts index be370d21a43..12f6e8f5ae3 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/constants.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/constants.ts @@ -19,6 +19,6 @@ export const GITHUB_BATCH_SIZE = 100; /** * Default client-side cap for GitHub list/compare fetches (deployments, - * deployment workflow runs, and commits between SHAs). + * deployment workflow runs, commits between SHAs, and commit history). */ -export const DEFAULT_DEPLOYMENT_FETCH_ITEMS_LIMIT = 1000; +export const DEFAULT_FETCH_ITEMS_LIMIT = 1000; diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.test.ts index baeb79e6a7f..64c50fe1833 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.test.ts @@ -62,6 +62,17 @@ describe('GithubAiAdoptionMetricProvider', () => { } }); + it('should set history to true for all metrics', () => { + const provider = GithubAiAdoptionMetricProvider.fromConfig( + new ConfigReader({}), + { logger: mockedLogger }, + ); + const metrics = provider.getMetrics(); + for (const metric of metrics) { + expect(metric.history).toBe(true); + } + }); + it('should set type to number for all metrics', () => { const provider = GithubAiAdoptionMetricProvider.fromConfig( new ConfigReader({}), @@ -316,6 +327,30 @@ describe('GithubAiAdoptionMetricProvider', () => { expect(results.get('github.aiAdoptionRate[7d]')).toBe(0); }); + it('should detect trailer keys with non-standard casing', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: + 'feat: all caps\n\nCO-AUTHORED-BY: Claude ', + committedDate: now.toISOString(), + }, + { + message: 'feat: lowercase\n\nassisted-by: Cursor', + committedDate: now.toISOString(), + }, + { + message: + 'feat: mixed\n\nco-Authored-By: Copilot ', + committedDate: now.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + expect(results.get('github.aiAdoptionRate[7d]')).toBe(1); + }); + it('should detect various AI tools case-insensitively', async () => { const now = new Date(); mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.ts index 7e940cd776e..b098571c02a 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.ts @@ -40,7 +40,8 @@ export const AI_ADOPTION_RATE_THRESHOLD: ThresholdConfig = { /** * Known AI tool identifiers used in commit trailers. * Matched case-insensitively against the value after - * `Assisted-by: ` or `Co-Authored-By: ` / `Co-authored-by: `. + * `Assisted-by:` or `Co-authored-by:` (trailer keys are + * matched case-insensitively per the git trailer spec). */ const AI_TOOL_PATTERNS: string[] = [ 'claude', @@ -72,14 +73,13 @@ function isAiAssistedCommit(message: string): boolean { const lines = message.split('\n'); for (const line of lines) { const trimmed = line.trim(); + const lower = trimmed.toLowerCase(); let value: string | undefined; - if (trimmed.startsWith('Assisted-by: ')) { - value = trimmed.slice('Assisted-by: '.length).trim(); - } else if (trimmed.startsWith('Co-Authored-By: ')) { - value = trimmed.slice('Co-Authored-By: '.length).trim(); - } else if (trimmed.startsWith('Co-authored-by: ')) { - value = trimmed.slice('Co-authored-by: '.length).trim(); + if (lower.startsWith('assisted-by: ')) { + value = trimmed.slice('assisted-by: '.length).trim(); + } else if (lower.startsWith('co-authored-by: ')) { + value = trimmed.slice('co-authored-by: '.length).trim(); } if (value) { @@ -128,6 +128,7 @@ export class GithubAiAdoptionMetricProvider description: `Ratio of AI-assisted commits over the last ${range}.`, type: 'number' as const, thresholds: AI_ADOPTION_RATE_THRESHOLD, + history: true, })); } From d9430941b5ddbd06b05d05a1350f4a4399094c50 Mon Sep 17 00:00:00 2001 From: fullsend-fix <278716306+fullsend-ai-coder[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:52:11 +0000 Subject: [PATCH 3/3] fix: address review feedback on PR #4580 - Rename GithubAiAdoptionMetricProvider to GithubAiAdoptionProvider to match the naming convention of GithubOpenPRsProvider - Fix false-positive AI detection by using exact name-part matching instead of startsWith, preventing human names like "Claude Smith" or "Devin Johnson" from being classified as AI-assisted - Add logger.warn when fetchItemsLimit is reached in getCommitHistory, matching the pattern used by getDeployments and getCommitShasBetween - Add logger.warn when defaultBranchRef is null (empty/misconfigured repo) - Remove explicit `: string` return type from getProviderId() to match sibling GithubOpenPRsProvider convention - Add comprehensive GithubClient.getCommitHistory tests covering single-page, multi-page pagination, fetchItemsLimit truncation, repository-not-found error, null node handling, and null defaultBranchRef - Add test for human-name false-positive prevention - Document AI adoption rate metrics in module README (IDs, thresholds, schedule configuration) - Update scorecard-backend README Available Metric Providers table - Add GithubAiAdoptionProvider to providers.md example list Addresses review feedback on #4580 --- .../scorecard-backend-module-github/README.md | 43 +++ .../src/github/GitHubClient.test.ts | 248 ++++++++++++++++++ .../src/github/GithubClient.ts | 20 +- ...st.ts => GithubAiAdoptionProvider.test.ts} | 54 ++-- ...rovider.ts => GithubAiAdoptionProvider.ts} | 36 ++- .../src/module.ts | 4 +- .../plugins/scorecard-backend/README.md | 16 +- .../scorecard-backend/docs/providers.md | 2 +- 8 files changed, 382 insertions(+), 41 deletions(-) rename workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/{GithubAiAdoptionMetricProvider.test.ts => GithubAiAdoptionProvider.test.ts} (89%) rename workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/{GithubAiAdoptionMetricProvider.ts => GithubAiAdoptionProvider.ts} (85%) diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/README.md b/workspaces/scorecard/plugins/scorecard-backend-module-github/README.md index c5b38b3c25b..fa5fbab87ed 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/README.md +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/README.md @@ -70,6 +70,15 @@ This metric counts all pull requests that are currently in an "open" state for t - **Datasource**: `github` - **Unit**: open pull requests (count) +### GitHub AI adoption rate (`github.aiAdoptionRate[7d]`, `[30d]`, `[90d]`) + +These metrics calculate the ratio of AI-assisted commits on the repository's default branch over 7-day, 30-day, and 90-day time windows. A commit is considered AI-assisted when it contains a `Co-authored-by` or `Assisted-by` trailer referencing a known AI tool (Claude, Copilot, Cursor, Codeium, Cody, Tabnine, Gemini, Amazon Q, Windsurf, Devin, Aider). Merge commits are excluded from the calculation. + +- **Metric IDs**: `github.aiAdoptionRate[7d]`, `github.aiAdoptionRate[30d]`, `github.aiAdoptionRate[90d]` +- **Metric Provider ID**: `github.aiAdoption` +- **Type**: Number (ratio from 0 to 1) +- **Datasource**: `github` + ## Collectors This module registers collectors to collect data from GitHub to be used by composite metric providers: @@ -156,6 +165,24 @@ scorecard: expression: '>50' ``` +Default thresholds for `github.aiAdoption`: + +```yaml +# app-config.yaml +scorecard: + metricProviders: + github: + aiAdoption: + thresholds: + rules: + - key: success + expression: '>=0.2' + - key: warning + expression: '>=0.1' + - key: error + expression: '>=0' +``` + See [threshold configuration](../scorecard-backend/docs/thresholds.md) for custom thresholds configuration. ## Configuration @@ -178,4 +205,20 @@ scorecard: seconds: 5 ``` +The `aiAdoption` provider also supports an independent schedule: + +```yaml +scorecard: + metricProviders: + github: + aiAdoption: + schedule: + frequency: + cron: '0 6 * * *' + timeout: + minutes: 5 + initialDelay: + seconds: 5 +``` + The schedule configuration follows Backstage's `SchedulerServiceTaskScheduleDefinitionConfig` [schema](https://github.com/backstage/backstage/blob/master/packages/backend-plugin-api/src/services/definitions/SchedulerService.ts#L157). See [Metric Collection Scheduling](../scorecard-backend/docs/providers.md#metric-collection-scheduling) for custom schedule configuration. diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GitHubClient.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GitHubClient.test.ts index 5bd1ed9cf2e..933b17a7c99 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GitHubClient.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GitHubClient.test.ts @@ -682,6 +682,254 @@ describe('GithubClient', () => { }); }); + describe('getCommitHistory', () => { + it('should return commits from the default branch', async () => { + const url = `https://github.com/owner/repo`; + const since = new Date('2026-05-01T00:00:00.000Z'); + mockedGraphqlClient.mockResolvedValue({ + repository: { + defaultBranchRef: { + target: { + history: { + nodes: [ + { + message: 'feat: add feature', + committedDate: '2026-05-15T10:00:00.000Z', + }, + { + message: 'fix: bug fix', + committedDate: '2026-05-10T10:00:00.000Z', + }, + ], + pageInfo: { + hasNextPage: false, + endCursor: null, + }, + totalCount: 2, + }, + }, + }, + }, + }); + + const commits = await githubClient.getCommitHistory( + url, + repository, + since, + ); + + expect(commits).toEqual([ + { + message: 'feat: add feature', + committedDate: '2026-05-15T10:00:00.000Z', + }, + { + message: 'fix: bug fix', + committedDate: '2026-05-10T10:00:00.000Z', + }, + ]); + expect(mockedGraphqlClient).toHaveBeenCalledTimes(1); + expect(mockedGraphqlClient).toHaveBeenCalledWith( + expect.stringContaining('query getCommitHistory'), + expect.objectContaining({ + owner: repository.owner, + repo: repository.repo, + since: since.toISOString(), + after: null, + }), + ); + expect(getCredentialsSpy).toHaveBeenCalledWith({ url }); + }); + + it('should paginate across multiple pages', async () => { + const url = `https://github.com/owner/repo`; + const since = new Date('2026-05-01T00:00:00.000Z'); + mockedGraphqlClient + .mockResolvedValueOnce({ + repository: { + defaultBranchRef: { + target: { + history: { + nodes: [ + { + message: 'feat: page one', + committedDate: '2026-05-20T10:00:00.000Z', + }, + ], + pageInfo: { + hasNextPage: true, + endCursor: 'cursor-1', + }, + totalCount: 2, + }, + }, + }, + }, + }) + .mockResolvedValueOnce({ + repository: { + defaultBranchRef: { + target: { + history: { + nodes: [ + { + message: 'fix: page two', + committedDate: '2026-05-10T10:00:00.000Z', + }, + ], + pageInfo: { + hasNextPage: false, + endCursor: null, + }, + totalCount: 2, + }, + }, + }, + }, + }); + + const commits = await githubClient.getCommitHistory( + url, + repository, + since, + ); + + expect(commits).toHaveLength(2); + expect(commits[0].message).toBe('feat: page one'); + expect(commits[1].message).toBe('fix: page two'); + expect(mockedGraphqlClient).toHaveBeenCalledTimes(2); + expect(mockedGraphqlClient).toHaveBeenNthCalledWith( + 2, + expect.stringContaining('query getCommitHistory'), + expect.objectContaining({ after: 'cursor-1' }), + ); + }); + + it('should stop paging once fetchItemsLimit is reached', async () => { + const url = `https://github.com/owner/repo`; + const since = new Date('2026-05-01T00:00:00.000Z'); + mockedGraphqlClient.mockResolvedValueOnce({ + repository: { + defaultBranchRef: { + target: { + history: { + nodes: [ + { + message: 'feat: one', + committedDate: '2026-05-20T10:00:00.000Z', + }, + { + message: 'feat: two', + committedDate: '2026-05-15T10:00:00.000Z', + }, + { + message: 'feat: three', + committedDate: '2026-05-10T10:00:00.000Z', + }, + ], + pageInfo: { + hasNextPage: true, + endCursor: 'cursor-1', + }, + totalCount: 5, + }, + }, + }, + }, + }); + + const commits = await githubClient.getCommitHistory( + url, + repository, + since, + { fetchItemsLimit: 2 }, + ); + + expect(commits).toHaveLength(2); + expect(commits[0].message).toBe('feat: one'); + expect(commits[1].message).toBe('feat: two'); + expect(mockedGraphqlClient).toHaveBeenCalledTimes(1); + expect(mockedLogger.warn).toHaveBeenCalledWith( + 'Reached fetchItemsLimit of 2 for commit history in owner/repo; stopping fetch', + ); + }); + + it('should throw when repository is not found or inaccessible', async () => { + const url = `https://github.com/owner/repo`; + mockedGraphqlClient.mockResolvedValue({ repository: null }); + + await expect( + githubClient.getCommitHistory( + url, + repository, + new Date('2026-05-01T00:00:00.000Z'), + ), + ).rejects.toThrow( + `GitHub repository '${repository.owner}/${repository.repo}' was not found or is inaccessible`, + ); + }); + + it('should skip null nodes in commit history', async () => { + const url = `https://github.com/owner/repo`; + const since = new Date('2026-05-01T00:00:00.000Z'); + mockedGraphqlClient.mockResolvedValue({ + repository: { + defaultBranchRef: { + target: { + history: { + nodes: [ + { + message: 'feat: valid commit', + committedDate: '2026-05-15T10:00:00.000Z', + }, + null, + { + message: 'fix: another valid', + committedDate: '2026-05-10T10:00:00.000Z', + }, + ], + pageInfo: { + hasNextPage: false, + endCursor: null, + }, + totalCount: 3, + }, + }, + }, + }, + }); + + const commits = await githubClient.getCommitHistory( + url, + repository, + since, + ); + + expect(commits).toHaveLength(2); + }); + + it('should warn and return empty when defaultBranchRef is null', async () => { + const url = `https://github.com/owner/repo`; + const since = new Date('2026-05-01T00:00:00.000Z'); + mockedGraphqlClient.mockResolvedValue({ + repository: { + defaultBranchRef: null, + }, + }); + + const commits = await githubClient.getCommitHistory( + url, + repository, + since, + ); + + expect(commits).toEqual([]); + expect(mockedLogger.warn).toHaveBeenCalledWith( + 'No default branch found for owner/repo; returning empty commit history', + ); + }); + }); + describe('getWorkflowRuns', () => { it('should return workflow runs filtered by workflow name and date window in ascending order', async () => { const url = `https://github.com/owner/repo`; diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GithubClient.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GithubClient.ts index 8c09662ff18..b268fff2571 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GithubClient.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GithubClient.ts @@ -459,7 +459,14 @@ export class GithubClient { ); } - const history = response.repository.defaultBranchRef?.target?.history; + if (!response.repository.defaultBranchRef) { + this.logger.warn( + `No default branch found for ${repository.owner}/${repository.repo}; returning empty commit history`, + ); + break; + } + + const history = response.repository.defaultBranchRef.target?.history; const pageCommits = history?.nodes ?? []; if (pageCommits.length === 0) { @@ -478,9 +485,14 @@ export class GithubClient { } } - hasMorePages = - commits.length < fetchItemsLimit && - Boolean(history?.pageInfo.hasNextPage); + const githubHasNextPage = Boolean(history?.pageInfo.hasNextPage); + if (commits.length >= fetchItemsLimit && githubHasNextPage) { + this.logger.warn( + `Reached fetchItemsLimit of ${fetchItemsLimit} for commit history in ${repository.owner}/${repository.repo}; stopping fetch`, + ); + } + + hasMorePages = commits.length < fetchItemsLimit && githubHasNextPage; after = history?.pageInfo.endCursor ?? null; } diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionProvider.test.ts similarity index 89% rename from workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.test.ts rename to workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionProvider.test.ts index 64c50fe1833..fe7b2d31923 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionProvider.test.ts @@ -18,10 +18,10 @@ import { mockServices } from '@backstage/backend-test-utils'; import { ConfigReader } from '@backstage/config'; import type { Entity } from '@backstage/catalog-model'; import { - GithubAiAdoptionMetricProvider, + GithubAiAdoptionProvider, AI_ADOPTION_RATE_THRESHOLD, AI_ADOPTION_RATE_TIME_RANGES, -} from './GithubAiAdoptionMetricProvider'; +} from './GithubAiAdoptionProvider'; import { GithubClient } from '../github/GithubClient'; jest.mock('@backstage/catalog-model', () => ({ @@ -33,12 +33,12 @@ jest.mock('@backstage/catalog-model', () => ({ })); jest.mock('../github/GithubClient'); -describe('GithubAiAdoptionMetricProvider', () => { +describe('GithubAiAdoptionProvider', () => { const mockedLogger = mockServices.logger.mock(); describe('getMetrics', () => { it('should return 3 metrics for each time range', () => { - const provider = GithubAiAdoptionMetricProvider.fromConfig( + const provider = GithubAiAdoptionProvider.fromConfig( new ConfigReader({}), { logger: mockedLogger }, ); @@ -52,7 +52,7 @@ describe('GithubAiAdoptionMetricProvider', () => { }); it('should use AI_ADOPTION_RATE_THRESHOLD for all metrics', () => { - const provider = GithubAiAdoptionMetricProvider.fromConfig( + const provider = GithubAiAdoptionProvider.fromConfig( new ConfigReader({}), { logger: mockedLogger }, ); @@ -63,7 +63,7 @@ describe('GithubAiAdoptionMetricProvider', () => { }); it('should set history to true for all metrics', () => { - const provider = GithubAiAdoptionMetricProvider.fromConfig( + const provider = GithubAiAdoptionProvider.fromConfig( new ConfigReader({}), { logger: mockedLogger }, ); @@ -74,7 +74,7 @@ describe('GithubAiAdoptionMetricProvider', () => { }); it('should set type to number for all metrics', () => { - const provider = GithubAiAdoptionMetricProvider.fromConfig( + const provider = GithubAiAdoptionProvider.fromConfig( new ConfigReader({}), { logger: mockedLogger }, ); @@ -87,7 +87,7 @@ describe('GithubAiAdoptionMetricProvider', () => { describe('provider identity', () => { it('should return github as datasource id', () => { - const provider = GithubAiAdoptionMetricProvider.fromConfig( + const provider = GithubAiAdoptionProvider.fromConfig( new ConfigReader({}), { logger: mockedLogger }, ); @@ -95,7 +95,7 @@ describe('GithubAiAdoptionMetricProvider', () => { }); it('should return github.aiAdoption as provider id', () => { - const provider = GithubAiAdoptionMetricProvider.fromConfig( + const provider = GithubAiAdoptionProvider.fromConfig( new ConfigReader({}), { logger: mockedLogger }, ); @@ -103,7 +103,7 @@ describe('GithubAiAdoptionMetricProvider', () => { }); it('should filter entities with github project-slug annotation', () => { - const provider = GithubAiAdoptionMetricProvider.fromConfig( + const provider = GithubAiAdoptionProvider.fromConfig( new ConfigReader({}), { logger: mockedLogger }, ); @@ -131,7 +131,7 @@ describe('GithubAiAdoptionMetricProvider', () => { }); describe('calculateMetrics', () => { - let provider: GithubAiAdoptionMetricProvider; + let provider: GithubAiAdoptionProvider; const mockedGithubClient = GithubClient as jest.MockedClass< typeof GithubClient >; @@ -153,10 +153,9 @@ describe('GithubAiAdoptionMetricProvider', () => { beforeEach(() => { jest.clearAllMocks(); - provider = GithubAiAdoptionMetricProvider.fromConfig( - new ConfigReader({}), - { logger: mockedLogger }, - ); + provider = GithubAiAdoptionProvider.fromConfig(new ConfigReader({}), { + logger: mockedLogger, + }); }); it('should return 0 for all ranges when no commits', async () => { @@ -327,6 +326,31 @@ describe('GithubAiAdoptionMetricProvider', () => { expect(results.get('github.aiAdoptionRate[7d]')).toBe(0); }); + it('should not false-positive on human names that start with AI tool names', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: + 'feat: human named Claude\n\nCo-authored-by: Claude Smith ', + committedDate: now.toISOString(), + }, + { + message: + 'feat: human named Devin\n\nCo-authored-by: Devin Johnson ', + committedDate: now.toISOString(), + }, + { + message: + 'feat: human named Cody\n\nCo-authored-by: Cody Williams ', + committedDate: now.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + expect(results.get('github.aiAdoptionRate[7d]')).toBe(0); + }); + it('should detect trailer keys with non-standard casing', async () => { const now = new Date(); mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionProvider.ts similarity index 85% rename from workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.ts rename to workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionProvider.ts index b098571c02a..686cab71ae2 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionMetricProvider.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionProvider.ts @@ -39,12 +39,15 @@ export const AI_ADOPTION_RATE_THRESHOLD: ThresholdConfig = { /** * Known AI tool identifiers used in commit trailers. - * Matched case-insensitively against the value after - * `Assisted-by:` or `Co-authored-by:` (trailer keys are - * matched case-insensitively per the git trailer spec). + * Matched case-insensitively against the "name" portion of the trailer + * value (everything before the first `<` in `Name `). + * + * Trailer keys (`Assisted-by:`, `Co-authored-by:`) are matched + * case-insensitively per the git trailer spec. */ const AI_TOOL_PATTERNS: string[] = [ 'claude', + 'claude code', 'cursor', 'copilot', 'github copilot', @@ -53,6 +56,7 @@ const AI_TOOL_PATTERNS: string[] = [ 'tabnine', 'gemini', 'amazon q', + 'amazon q developer', 'windsurf', 'devin', 'aider', @@ -69,6 +73,18 @@ function isMergeCommit(message: string): boolean { ); } +/** + * Extracts the "name" portion from a trailer value. + * Git trailers follow the format `Name ` or just `Name`. + * Returns the trimmed, lowercased name part before the first `<`. + */ +function extractTrailerName(value: string): string { + const angleBracketIndex = value.indexOf('<'); + const namePart = + angleBracketIndex >= 0 ? value.slice(0, angleBracketIndex) : value; + return namePart.trim().toLowerCase(); +} + function isAiAssistedCommit(message: string): boolean { const lines = message.split('\n'); for (const line of lines) { @@ -83,8 +99,8 @@ function isAiAssistedCommit(message: string): boolean { } if (value) { - const lowerValue = value.toLowerCase(); - if (AI_TOOL_PATTERNS.some(tool => lowerValue.startsWith(tool))) { + const name = extractTrailerName(value); + if (AI_TOOL_PATTERNS.includes(name)) { return true; } } @@ -92,9 +108,7 @@ function isAiAssistedCommit(message: string): boolean { return false; } -export class GithubAiAdoptionMetricProvider - implements MetricProvider<'number'> -{ +export class GithubAiAdoptionProvider implements MetricProvider<'number'> { private readonly githubClient: GithubClient; private readonly logger: LoggerService; @@ -106,8 +120,8 @@ export class GithubAiAdoptionMetricProvider static fromConfig( config: Config, options: { logger: LoggerService }, - ): GithubAiAdoptionMetricProvider { - return new GithubAiAdoptionMetricProvider( + ): GithubAiAdoptionProvider { + return new GithubAiAdoptionProvider( new GithubClient(config, options.logger), options.logger, ); @@ -117,7 +131,7 @@ export class GithubAiAdoptionMetricProvider return 'github'; } - getProviderId(): string { + getProviderId() { return 'github.aiAdoption'; } diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/module.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/module.ts index 5770db1c03e..4f7f24e2701 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/module.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/module.ts @@ -24,7 +24,7 @@ import { import { GithubDeploymentPullRequestsCollector } from './collectors/GithubDeploymentPullRequestsCollector'; import { GithubDeploymentWorkflowRunsCollector } from './collectors/GithubDeploymentWorkflowRunsCollector'; import { GithubDeploymentsCollector } from './collectors/GithubDeploymentsCollector'; -import { GithubAiAdoptionMetricProvider } from './metricProviders/GithubAiAdoptionMetricProvider'; +import { GithubAiAdoptionProvider } from './metricProviders/GithubAiAdoptionProvider'; import { GithubOpenPRsProvider } from './metricProviders/GithubOpenPRsProvider'; export const scorecardModuleGithub = createBackendModule({ @@ -46,7 +46,7 @@ export const scorecardModuleGithub = createBackendModule({ ); metrics.addMetricProvider( GithubOpenPRsProvider.fromConfig(config, { logger }), - GithubAiAdoptionMetricProvider.fromConfig(config, { logger }), + GithubAiAdoptionProvider.fromConfig(config, { logger }), ); }, }); diff --git a/workspaces/scorecard/plugins/scorecard-backend/README.md b/workspaces/scorecard/plugins/scorecard-backend/README.md index 10d55ae4bc9..fb89710d9e4 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/README.md +++ b/workspaces/scorecard/plugins/scorecard-backend/README.md @@ -94,14 +94,14 @@ For more information about schedule configuration options, see the [Metric Colle The following metric providers are available: -| Provider | Metric ID | Title | Description | Type | -| -------------- | --------------------------------------------------------------------------------------------------------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | ------- | -| **GitHub** | `github.openPRs` | GitHub open PRs | Count of open Pull Requests in GitHub | number | -| **Filecheck** | `filecheck.*` | File Checks | Checks whether specific files (e.g., `README.md`, `LICENSE`, `CODEOWNERS`) exist in a repository. | boolean | -| **Jira** | `jira.openIssues` | Jira open issues | The number of opened issues in Jira | number | -| **OpenSSF** | `openssf.*` | OpenSSF Security Scorecards | 18 security metrics from OpenSSF Scorecards (e.g., `openssf.codeReview`, `openssf.maintained`). Each returns a score from 0-10. | number | -| **Dependabot** | `dependabot.*` | Dependabot Alerts | Critical, High, Medium and Low CVE Alerts | number | -| **DORA** | `dora.deploymentFrequency`, `dora.medianLeadTimeForChanges`, `dora.meanTimeToRestore`, `dora.changeFailureRate` | DORA Metrics | Software delivery performance metrics based on DORA (DevOps Research and Assessment) | number | +| Provider | Metric ID | Title | Description | Type | +| -------------- | --------------------------------------------------------------------------------------------------------------- | --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | ------- | +| **GitHub** | `github.openPRs`, `github.aiAdoptionRate[7d]`, `github.aiAdoptionRate[30d]`, `github.aiAdoptionRate[90d]` | GitHub open PRs, AI adoption rate | Count of open Pull Requests; ratio of AI-assisted commits over 7d, 30d, and 90d windows | number | +| **Filecheck** | `filecheck.*` | File Checks | Checks whether specific files (e.g., `README.md`, `LICENSE`, `CODEOWNERS`) exist in a repository. | boolean | +| **Jira** | `jira.openIssues` | Jira open issues | The number of opened issues in Jira | number | +| **OpenSSF** | `openssf.*` | OpenSSF Security Scorecards | 18 security metrics from OpenSSF Scorecards (e.g., `openssf.codeReview`, `openssf.maintained`). Each returns a score from 0-10. | number | +| **Dependabot** | `dependabot.*` | Dependabot Alerts | Critical, High, Medium and Low CVE Alerts | number | +| **DORA** | `dora.deploymentFrequency`, `dora.medianLeadTimeForChanges`, `dora.meanTimeToRestore`, `dora.changeFailureRate` | DORA Metrics | Software delivery performance metrics based on DORA (DevOps Research and Assessment) | number | To use these providers, install the corresponding backend modules: diff --git a/workspaces/scorecard/plugins/scorecard-backend/docs/providers.md b/workspaces/scorecard/plugins/scorecard-backend/docs/providers.md index 719f7a920bb..89b234415ea 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/docs/providers.md +++ b/workspaces/scorecard/plugins/scorecard-backend/docs/providers.md @@ -190,7 +190,7 @@ schedule: The following are examples of existing metric providers that you can reference: -- **GitHub Datasource**: [GithubOpenPRsProvider](../../scorecard-backend-module-github/src/metricProviders/GithubOpenPRsProvider.ts) +- **GitHub Datasource**: [GithubOpenPRsProvider](../../scorecard-backend-module-github/src/metricProviders/GithubOpenPRsProvider.ts), [GithubAiAdoptionProvider](../../scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionProvider.ts) - **Jira Datasource**: [JiraOpenIssuesProvider](../../scorecard-backend-module-jira/src/metricProviders/JiraOpenIssuesProvider.ts) - **OpenSSF Datasource**: [OpenSSFMetricProvider](../../scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.ts) - **Filecheck Datasource** (batch / multi-metric): [FilecheckMetricProvider](../../scorecard-backend-module-filecheck/src/metricProviders/FilecheckMetricProvider.ts)