diff --git a/workspaces/scorecard/.changeset/add-github-ai-adoption-metric.md b/workspaces/scorecard/.changeset/add-github-ai-adoption-metric.md new file mode 100644 index 00000000000..f79edf8c999 --- /dev/null +++ b/workspaces/scorecard/.changeset/add-github-ai-adoption-metric.md @@ -0,0 +1,5 @@ +--- +'@red-hat-developer-hub/backstage-plugin-scorecard-backend-module-github': minor +--- + +Add AI adoption rate metric provider to the GitHub scorecard module. diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/README.md b/workspaces/scorecard/plugins/scorecard-backend-module-github/README.md index c5b38b3c25b..fa5fbab87ed 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/README.md +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/README.md @@ -70,6 +70,15 @@ This metric counts all pull requests that are currently in an "open" state for t - **Datasource**: `github` - **Unit**: open pull requests (count) +### GitHub AI adoption rate (`github.aiAdoptionRate[7d]`, `[30d]`, `[90d]`) + +These metrics calculate the ratio of AI-assisted commits on the repository's default branch over 7-day, 30-day, and 90-day time windows. A commit is considered AI-assisted when it contains a `Co-authored-by` or `Assisted-by` trailer referencing a known AI tool (Claude, Copilot, Cursor, Codeium, Cody, Tabnine, Gemini, Amazon Q, Windsurf, Devin, Aider). Merge commits are excluded from the calculation. + +- **Metric IDs**: `github.aiAdoptionRate[7d]`, `github.aiAdoptionRate[30d]`, `github.aiAdoptionRate[90d]` +- **Metric Provider ID**: `github.aiAdoption` +- **Type**: Number (ratio from 0 to 1) +- **Datasource**: `github` + ## Collectors This module registers collectors to collect data from GitHub to be used by composite metric providers: @@ -156,6 +165,24 @@ scorecard: expression: '>50' ``` +Default thresholds for `github.aiAdoption`: + +```yaml +# app-config.yaml +scorecard: + metricProviders: + github: + aiAdoption: + thresholds: + rules: + - key: success + expression: '>=0.2' + - key: warning + expression: '>=0.1' + - key: error + expression: '>=0' +``` + See [threshold configuration](../scorecard-backend/docs/thresholds.md) for custom thresholds configuration. ## Configuration @@ -178,4 +205,20 @@ scorecard: seconds: 5 ``` +The `aiAdoption` provider also supports an independent schedule: + +```yaml +scorecard: + metricProviders: + github: + aiAdoption: + schedule: + frequency: + cron: '0 6 * * *' + timeout: + minutes: 5 + initialDelay: + seconds: 5 +``` + The schedule configuration follows Backstage's `SchedulerServiceTaskScheduleDefinitionConfig` [schema](https://github.com/backstage/backstage/blob/master/packages/backend-plugin-api/src/services/definitions/SchedulerService.ts#L157). See [Metric Collection Scheduling](../scorecard-backend/docs/providers.md#metric-collection-scheduling) for custom schedule configuration. diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/config.d.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/config.d.ts index ae0c7e1b17f..ca3c306773f 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/config.d.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/config.d.ts @@ -30,6 +30,12 @@ export interface Config { /** How github.openPRs metric values are categorized */ thresholds?: ThresholdConfig; }; + aiAdoption?: { + /** How often github.aiAdoption metrics will be calculated */ + schedule?: SchedulerServiceTaskScheduleDefinitionConfig; + /** How github.aiAdoption metric values are categorized */ + thresholds?: ThresholdConfig; + }; }; }; }; diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GitHubClient.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GitHubClient.test.ts index 5bd1ed9cf2e..933b17a7c99 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GitHubClient.test.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GitHubClient.test.ts @@ -682,6 +682,254 @@ describe('GithubClient', () => { }); }); + describe('getCommitHistory', () => { + it('should return commits from the default branch', async () => { + const url = `https://github.com/owner/repo`; + const since = new Date('2026-05-01T00:00:00.000Z'); + mockedGraphqlClient.mockResolvedValue({ + repository: { + defaultBranchRef: { + target: { + history: { + nodes: [ + { + message: 'feat: add feature', + committedDate: '2026-05-15T10:00:00.000Z', + }, + { + message: 'fix: bug fix', + committedDate: '2026-05-10T10:00:00.000Z', + }, + ], + pageInfo: { + hasNextPage: false, + endCursor: null, + }, + totalCount: 2, + }, + }, + }, + }, + }); + + const commits = await githubClient.getCommitHistory( + url, + repository, + since, + ); + + expect(commits).toEqual([ + { + message: 'feat: add feature', + committedDate: '2026-05-15T10:00:00.000Z', + }, + { + message: 'fix: bug fix', + committedDate: '2026-05-10T10:00:00.000Z', + }, + ]); + expect(mockedGraphqlClient).toHaveBeenCalledTimes(1); + expect(mockedGraphqlClient).toHaveBeenCalledWith( + expect.stringContaining('query getCommitHistory'), + expect.objectContaining({ + owner: repository.owner, + repo: repository.repo, + since: since.toISOString(), + after: null, + }), + ); + expect(getCredentialsSpy).toHaveBeenCalledWith({ url }); + }); + + it('should paginate across multiple pages', async () => { + const url = `https://github.com/owner/repo`; + const since = new Date('2026-05-01T00:00:00.000Z'); + mockedGraphqlClient + .mockResolvedValueOnce({ + repository: { + defaultBranchRef: { + target: { + history: { + nodes: [ + { + message: 'feat: page one', + committedDate: '2026-05-20T10:00:00.000Z', + }, + ], + pageInfo: { + hasNextPage: true, + endCursor: 'cursor-1', + }, + totalCount: 2, + }, + }, + }, + }, + }) + .mockResolvedValueOnce({ + repository: { + defaultBranchRef: { + target: { + history: { + nodes: [ + { + message: 'fix: page two', + committedDate: '2026-05-10T10:00:00.000Z', + }, + ], + pageInfo: { + hasNextPage: false, + endCursor: null, + }, + totalCount: 2, + }, + }, + }, + }, + }); + + const commits = await githubClient.getCommitHistory( + url, + repository, + since, + ); + + expect(commits).toHaveLength(2); + expect(commits[0].message).toBe('feat: page one'); + expect(commits[1].message).toBe('fix: page two'); + expect(mockedGraphqlClient).toHaveBeenCalledTimes(2); + expect(mockedGraphqlClient).toHaveBeenNthCalledWith( + 2, + expect.stringContaining('query getCommitHistory'), + expect.objectContaining({ after: 'cursor-1' }), + ); + }); + + it('should stop paging once fetchItemsLimit is reached', async () => { + const url = `https://github.com/owner/repo`; + const since = new Date('2026-05-01T00:00:00.000Z'); + mockedGraphqlClient.mockResolvedValueOnce({ + repository: { + defaultBranchRef: { + target: { + history: { + nodes: [ + { + message: 'feat: one', + committedDate: '2026-05-20T10:00:00.000Z', + }, + { + message: 'feat: two', + committedDate: '2026-05-15T10:00:00.000Z', + }, + { + message: 'feat: three', + committedDate: '2026-05-10T10:00:00.000Z', + }, + ], + pageInfo: { + hasNextPage: true, + endCursor: 'cursor-1', + }, + totalCount: 5, + }, + }, + }, + }, + }); + + const commits = await githubClient.getCommitHistory( + url, + repository, + since, + { fetchItemsLimit: 2 }, + ); + + expect(commits).toHaveLength(2); + expect(commits[0].message).toBe('feat: one'); + expect(commits[1].message).toBe('feat: two'); + expect(mockedGraphqlClient).toHaveBeenCalledTimes(1); + expect(mockedLogger.warn).toHaveBeenCalledWith( + 'Reached fetchItemsLimit of 2 for commit history in owner/repo; stopping fetch', + ); + }); + + it('should throw when repository is not found or inaccessible', async () => { + const url = `https://github.com/owner/repo`; + mockedGraphqlClient.mockResolvedValue({ repository: null }); + + await expect( + githubClient.getCommitHistory( + url, + repository, + new Date('2026-05-01T00:00:00.000Z'), + ), + ).rejects.toThrow( + `GitHub repository '${repository.owner}/${repository.repo}' was not found or is inaccessible`, + ); + }); + + it('should skip null nodes in commit history', async () => { + const url = `https://github.com/owner/repo`; + const since = new Date('2026-05-01T00:00:00.000Z'); + mockedGraphqlClient.mockResolvedValue({ + repository: { + defaultBranchRef: { + target: { + history: { + nodes: [ + { + message: 'feat: valid commit', + committedDate: '2026-05-15T10:00:00.000Z', + }, + null, + { + message: 'fix: another valid', + committedDate: '2026-05-10T10:00:00.000Z', + }, + ], + pageInfo: { + hasNextPage: false, + endCursor: null, + }, + totalCount: 3, + }, + }, + }, + }, + }); + + const commits = await githubClient.getCommitHistory( + url, + repository, + since, + ); + + expect(commits).toHaveLength(2); + }); + + it('should warn and return empty when defaultBranchRef is null', async () => { + const url = `https://github.com/owner/repo`; + const since = new Date('2026-05-01T00:00:00.000Z'); + mockedGraphqlClient.mockResolvedValue({ + repository: { + defaultBranchRef: null, + }, + }); + + const commits = await githubClient.getCommitHistory( + url, + repository, + since, + ); + + expect(commits).toEqual([]); + expect(mockedLogger.warn).toHaveBeenCalledWith( + 'No default branch found for owner/repo; returning empty commit history', + ); + }); + }); + describe('getWorkflowRuns', () => { it('should return workflow runs filtered by workflow name and date window in ascending order', async () => { const url = `https://github.com/owner/repo`; diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GithubClient.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GithubClient.ts index 849e0034051..b268fff2571 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GithubClient.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/GithubClient.ts @@ -23,6 +23,8 @@ import { import { graphql } from '@octokit/graphql'; import { Octokit } from '@octokit/rest'; import { + GithubCommit, + GithubCommitHistoryQueryResponse, GithubDeployment, GithubWorkflowRun, GithubPullRequest, @@ -30,10 +32,7 @@ import { GithubDeploymentsQueryResponse, GithubCommitsPullRequestsQueryResponse, } from './types'; -import { - DEFAULT_DEPLOYMENT_FETCH_ITEMS_LIMIT, - GITHUB_BATCH_SIZE, -} from './constants'; +import { DEFAULT_FETCH_ITEMS_LIMIT, GITHUB_BATCH_SIZE } from './constants'; import { buildCommitsPullRequestsQuery } from './queries/buildCommitsPullRequestsQuery'; import { mapCommitsPullRequests } from './mappers'; @@ -125,7 +124,7 @@ export class GithubClient { options?: { fetchItemsLimit?: number }, ): Promise { const fetchItemsLimit = - options?.fetchItemsLimit ?? DEFAULT_DEPLOYMENT_FETCH_ITEMS_LIMIT; + options?.fetchItemsLimit ?? DEFAULT_FETCH_ITEMS_LIMIT; const octokit = await this.getOctokitClient(url); const deployments: GithubDeployment[] = []; const query = ` @@ -241,7 +240,7 @@ export class GithubClient { options?: { fetchItemsLimit?: number }, ): Promise { const fetchItemsLimit = - options?.fetchItemsLimit ?? DEFAULT_DEPLOYMENT_FETCH_ITEMS_LIMIT; + options?.fetchItemsLimit ?? DEFAULT_FETCH_ITEMS_LIMIT; const octokit = await this.getOctokitRestClient(url); const basehead = `${baseSha}...${headSha}`; @@ -347,7 +346,7 @@ export class GithubClient { options?: { fetchItemsLimit?: number }, ): Promise { const fetchItemsLimit = - options?.fetchItemsLimit ?? DEFAULT_DEPLOYMENT_FETCH_ITEMS_LIMIT; + options?.fetchItemsLimit ?? DEFAULT_FETCH_ITEMS_LIMIT; const octokit = await this.getOctokitRestClient(url); const workflows = await octokit.paginate( @@ -408,4 +407,95 @@ export class GithubClient { // normalize to ASC for chronological processing (oldest -> newest). return workflowRuns.reverse(); } + + async getCommitHistory( + url: string, + repository: GithubRepository, + since: Date, + options?: { fetchItemsLimit?: number }, + ): Promise { + const fetchItemsLimit = + options?.fetchItemsLimit ?? DEFAULT_FETCH_ITEMS_LIMIT; + const octokit = await this.getOctokitClient(url); + const commits: GithubCommit[] = []; + const query = ` + query getCommitHistory($owner: String!, $repo: String!, $since: GitTimestamp!, $after: String) { + repository(owner: $owner, name: $repo) { + defaultBranchRef { + target { + ... on Commit { + history(since: $since, first: ${GITHUB_BATCH_SIZE}, after: $after) { + nodes { + message + committedDate + } + pageInfo { + hasNextPage + endCursor + } + totalCount + } + } + } + } + } + } + `; + + let after: string | null = null; + let hasMorePages = true; + + while (hasMorePages && commits.length < fetchItemsLimit) { + const response: GithubCommitHistoryQueryResponse = await octokit(query, { + owner: repository.owner, + repo: repository.repo, + since: since.toISOString(), + after, + }); + + if (!response.repository) { + throw new Error( + `GitHub repository '${repository.owner}/${repository.repo}' was not found or is inaccessible`, + ); + } + + if (!response.repository.defaultBranchRef) { + this.logger.warn( + `No default branch found for ${repository.owner}/${repository.repo}; returning empty commit history`, + ); + break; + } + + const history = response.repository.defaultBranchRef.target?.history; + const pageCommits = history?.nodes ?? []; + + if (pageCommits.length === 0) { + break; + } + + for (const commit of pageCommits) { + if (commits.length >= fetchItemsLimit) { + break; + } + if (commit) { + commits.push({ + message: commit.message, + committedDate: commit.committedDate, + }); + } + } + + const githubHasNextPage = Boolean(history?.pageInfo.hasNextPage); + if (commits.length >= fetchItemsLimit && githubHasNextPage) { + this.logger.warn( + `Reached fetchItemsLimit of ${fetchItemsLimit} for commit history in ${repository.owner}/${repository.repo}; stopping fetch`, + ); + } + + hasMorePages = commits.length < fetchItemsLimit && githubHasNextPage; + after = history?.pageInfo.endCursor ?? null; + } + + return commits; + } } diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/constants.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/constants.ts index be370d21a43..12f6e8f5ae3 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/constants.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/constants.ts @@ -19,6 +19,6 @@ export const GITHUB_BATCH_SIZE = 100; /** * Default client-side cap for GitHub list/compare fetches (deployments, - * deployment workflow runs, and commits between SHAs). + * deployment workflow runs, commits between SHAs, and commit history). */ -export const DEFAULT_DEPLOYMENT_FETCH_ITEMS_LIMIT = 1000; +export const DEFAULT_FETCH_ITEMS_LIMIT = 1000; diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/types.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/types.ts index 3e7f045f05a..a4971c5074b 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/types.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/github/types.ts @@ -82,3 +82,28 @@ export type GithubCommitsPullRequestsQueryResponse = } | null > | null; }; + +export type GithubCommit = { + message: string; + committedDate: string; +}; + +export type GithubCommitHistoryQueryResponse = GraphQlQueryResponseData & { + repository: { + defaultBranchRef?: { + target?: { + history?: { + nodes: Array<{ + message: string; + committedDate: string; + } | null>; + pageInfo: { + hasNextPage: boolean; + endCursor: string | null; + }; + totalCount: number; + } | null; + } | null; + } | null; + } | null; +}; diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionProvider.test.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionProvider.test.ts new file mode 100644 index 00000000000..fe7b2d31923 --- /dev/null +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionProvider.test.ts @@ -0,0 +1,445 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { mockServices } from '@backstage/backend-test-utils'; +import { ConfigReader } from '@backstage/config'; +import type { Entity } from '@backstage/catalog-model'; +import { + GithubAiAdoptionProvider, + AI_ADOPTION_RATE_THRESHOLD, + AI_ADOPTION_RATE_TIME_RANGES, +} from './GithubAiAdoptionProvider'; +import { GithubClient } from '../github/GithubClient'; + +jest.mock('@backstage/catalog-model', () => ({ + ...jest.requireActual('@backstage/catalog-model'), + getEntitySourceLocation: jest.fn().mockReturnValue({ + type: 'url', + target: 'https://github.com/org/orgRepo/tree/main/', + }), +})); +jest.mock('../github/GithubClient'); + +describe('GithubAiAdoptionProvider', () => { + const mockedLogger = mockServices.logger.mock(); + + describe('getMetrics', () => { + it('should return 3 metrics for each time range', () => { + const provider = GithubAiAdoptionProvider.fromConfig( + new ConfigReader({}), + { logger: mockedLogger }, + ); + const metrics = provider.getMetrics(); + expect(metrics).toHaveLength(3); + expect(metrics.map(m => m.id)).toEqual([ + 'github.aiAdoptionRate[7d]', + 'github.aiAdoptionRate[30d]', + 'github.aiAdoptionRate[90d]', + ]); + }); + + it('should use AI_ADOPTION_RATE_THRESHOLD for all metrics', () => { + const provider = GithubAiAdoptionProvider.fromConfig( + new ConfigReader({}), + { logger: mockedLogger }, + ); + const metrics = provider.getMetrics(); + for (const metric of metrics) { + expect(metric.thresholds).toEqual(AI_ADOPTION_RATE_THRESHOLD); + } + }); + + it('should set history to true for all metrics', () => { + const provider = GithubAiAdoptionProvider.fromConfig( + new ConfigReader({}), + { logger: mockedLogger }, + ); + const metrics = provider.getMetrics(); + for (const metric of metrics) { + expect(metric.history).toBe(true); + } + }); + + it('should set type to number for all metrics', () => { + const provider = GithubAiAdoptionProvider.fromConfig( + new ConfigReader({}), + { logger: mockedLogger }, + ); + const metrics = provider.getMetrics(); + for (const metric of metrics) { + expect(metric.type).toBe('number'); + } + }); + }); + + describe('provider identity', () => { + it('should return github as datasource id', () => { + const provider = GithubAiAdoptionProvider.fromConfig( + new ConfigReader({}), + { logger: mockedLogger }, + ); + expect(provider.getProviderDatasourceId()).toBe('github'); + }); + + it('should return github.aiAdoption as provider id', () => { + const provider = GithubAiAdoptionProvider.fromConfig( + new ConfigReader({}), + { logger: mockedLogger }, + ); + expect(provider.getProviderId()).toBe('github.aiAdoption'); + }); + + it('should filter entities with github project-slug annotation', () => { + const provider = GithubAiAdoptionProvider.fromConfig( + new ConfigReader({}), + { logger: mockedLogger }, + ); + const filter = provider.getCatalogFilter(); + expect('metadata.annotations.github.com/project-slug' in filter).toBe( + true, + ); + }); + }); + + describe('constants', () => { + it('should define 3 time ranges', () => { + expect(AI_ADOPTION_RATE_TIME_RANGES).toEqual(['7d', '30d', '90d']); + }); + + it('should define threshold with success >= 0.2, warning >= 0.1, error >= 0', () => { + expect(AI_ADOPTION_RATE_THRESHOLD).toEqual({ + rules: [ + { key: 'success', expression: '>=0.2' }, + { key: 'warning', expression: '>=0.1' }, + { key: 'error', expression: '>=0' }, + ], + }); + }); + }); + + describe('calculateMetrics', () => { + let provider: GithubAiAdoptionProvider; + const mockedGithubClient = GithubClient as jest.MockedClass< + typeof GithubClient + >; + const mockedGithubClientInstance = { + getCommitHistory: jest.fn(), + } as any; + mockedGithubClient.mockImplementation(() => mockedGithubClientInstance); + + const mockEntity: Entity = { + apiVersion: 'backstage.io/v1alpha1', + kind: 'Component', + metadata: { + name: 'test-component', + annotations: { + 'github.com/project-slug': 'org/orgRepo', + }, + }, + }; + + beforeEach(() => { + jest.clearAllMocks(); + provider = GithubAiAdoptionProvider.fromConfig(new ConfigReader({}), { + logger: mockedLogger, + }); + }); + + it('should return 0 for all ranges when no commits', async () => { + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([]); + + const results = await provider.calculateMetrics(mockEntity); + + expect(results.get('github.aiAdoptionRate[7d]')).toBe(0); + expect(results.get('github.aiAdoptionRate[30d]')).toBe(0); + expect(results.get('github.aiAdoptionRate[90d]')).toBe(0); + }); + + it('should detect Co-Authored-By with Claude as AI-assisted', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: + 'feat: add feature\n\nCo-Authored-By: Claude ', + committedDate: now.toISOString(), + }, + { + message: 'fix: regular commit', + committedDate: now.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + expect(results.get('github.aiAdoptionRate[7d]')).toBe(0.5); + expect(results.get('github.aiAdoptionRate[30d]')).toBe(0.5); + expect(results.get('github.aiAdoptionRate[90d]')).toBe(0.5); + }); + + it('should detect Co-authored-by (lowercase) as AI-assisted', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: + 'feat: add feature\n\nCo-authored-by: Copilot ', + committedDate: now.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + expect(results.get('github.aiAdoptionRate[7d]')).toBe(1); + }); + + it('should detect Assisted-by trailer as AI-assisted', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: 'feat: refactor\n\nAssisted-by: Cursor', + committedDate: now.toISOString(), + }, + { + message: 'fix: manual fix', + committedDate: now.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + expect(results.get('github.aiAdoptionRate[7d]')).toBe(0.5); + }); + + it('should ignore merge commits', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: 'Merge pull request #42 from org/feature', + committedDate: now.toISOString(), + }, + { + message: "Merge branch 'main' into feature", + committedDate: now.toISOString(), + }, + { + message: + 'feat: add feature\n\nCo-Authored-By: Claude ', + committedDate: now.toISOString(), + }, + { + message: 'fix: something', + committedDate: now.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + // 2 merge commits ignored, 1 AI-assisted, 1 not = 0.5 + expect(results.get('github.aiAdoptionRate[7d]')).toBe(0.5); + }); + + it('should return 1.0 when all non-merge commits are AI-assisted', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: + 'feat: new thing\n\nCo-Authored-By: Claude ', + committedDate: now.toISOString(), + }, + { + message: 'fix: another\n\nAssisted-by: Cursor', + committedDate: now.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + expect(results.get('github.aiAdoptionRate[7d]')).toBe(1); + }); + + it('should calculate different ratios for different time ranges', async () => { + const now = new Date(); + const threeDaysAgo = new Date(now); + threeDaysAgo.setDate(threeDaysAgo.getDate() - 3); + const fifteenDaysAgo = new Date(now); + fifteenDaysAgo.setDate(fifteenDaysAgo.getDate() - 15); + const sixtyDaysAgo = new Date(now); + sixtyDaysAgo.setDate(sixtyDaysAgo.getDate() - 60); + + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + // Within 7d: 1 AI, 1 not = 0.5 + { + message: + 'feat: recent\n\nCo-Authored-By: Claude ', + committedDate: threeDaysAgo.toISOString(), + }, + { + message: 'fix: recent manual', + committedDate: threeDaysAgo.toISOString(), + }, + // Within 30d (but not 7d): 0 AI, 1 not + { + message: 'chore: older manual commit', + committedDate: fifteenDaysAgo.toISOString(), + }, + // Within 90d (but not 30d): 1 AI, 0 not + { + message: 'feat: old ai\n\nAssisted-by: Cursor', + committedDate: sixtyDaysAgo.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + // 7d: 1 AI / 2 total = 0.5 + expect(results.get('github.aiAdoptionRate[7d]')).toBe(0.5); + // 30d: 1 AI / 3 total = 0.333... + expect(results.get('github.aiAdoptionRate[30d]')).toBeCloseTo(1 / 3, 10); + // 90d: 2 AI / 4 total = 0.5 + expect(results.get('github.aiAdoptionRate[90d]')).toBe(0.5); + }); + + it('should not count Co-authored-by with non-AI authors', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: + 'feat: pair programming\n\nCo-authored-by: John Doe ', + committedDate: now.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + expect(results.get('github.aiAdoptionRate[7d]')).toBe(0); + }); + + it('should not false-positive on human names that start with AI tool names', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: + 'feat: human named Claude\n\nCo-authored-by: Claude Smith ', + committedDate: now.toISOString(), + }, + { + message: + 'feat: human named Devin\n\nCo-authored-by: Devin Johnson ', + committedDate: now.toISOString(), + }, + { + message: + 'feat: human named Cody\n\nCo-authored-by: Cody Williams ', + committedDate: now.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + expect(results.get('github.aiAdoptionRate[7d]')).toBe(0); + }); + + it('should detect trailer keys with non-standard casing', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: + 'feat: all caps\n\nCO-AUTHORED-BY: Claude ', + committedDate: now.toISOString(), + }, + { + message: 'feat: lowercase\n\nassisted-by: Cursor', + committedDate: now.toISOString(), + }, + { + message: + 'feat: mixed\n\nco-Authored-By: Copilot ', + committedDate: now.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + expect(results.get('github.aiAdoptionRate[7d]')).toBe(1); + }); + + it('should detect various AI tools case-insensitively', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: + 'feat: one\n\nCo-authored-by: CLAUDE ', + committedDate: now.toISOString(), + }, + { + message: 'feat: two\n\nAssisted-by: GitHub Copilot', + committedDate: now.toISOString(), + }, + { + message: 'feat: three\n\nAssisted-by: codeium', + committedDate: now.toISOString(), + }, + { + message: 'feat: four\n\nAssisted-by: Tabnine', + committedDate: now.toISOString(), + }, + { + message: 'feat: five\n\nAssisted-by: Amazon Q Developer', + committedDate: now.toISOString(), + }, + ]); + + const results = await provider.calculateMetrics(mockEntity); + + expect(results.get('github.aiAdoptionRate[7d]')).toBe(1); + }); + + it('should log analysis summary', async () => { + const now = new Date(); + mockedGithubClientInstance.getCommitHistory.mockResolvedValue([ + { + message: 'feat: ai\n\nCo-Authored-By: Claude ', + committedDate: now.toISOString(), + }, + { + message: 'fix: manual', + committedDate: now.toISOString(), + }, + { + message: 'Merge pull request #1 from org/feat', + committedDate: now.toISOString(), + }, + ]); + + await provider.calculateMetrics(mockEntity); + + expect(mockedLogger.info).toHaveBeenCalledWith( + expect.stringContaining('AI adoption [7d]'), + ); + expect(mockedLogger.info).toHaveBeenCalledWith( + expect.stringContaining('3 commits analyzed'), + ); + expect(mockedLogger.info).toHaveBeenCalledWith( + expect.stringContaining('1 merge commits ignored'), + ); + expect(mockedLogger.info).toHaveBeenCalledWith( + expect.stringContaining('1 AI-assisted'), + ); + expect(mockedLogger.info).toHaveBeenCalledWith( + expect.stringContaining('1 not AI-assisted'), + ); + }); + }); +}); diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionProvider.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionProvider.ts new file mode 100644 index 00000000000..686cab71ae2 --- /dev/null +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionProvider.ts @@ -0,0 +1,234 @@ +/* + * Copyright Red Hat, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { LoggerService } from '@backstage/backend-plugin-api'; +import type { Config } from '@backstage/config'; +import { getEntitySourceLocation, type Entity } from '@backstage/catalog-model'; +import { CATALOG_FILTER_EXISTS } from '@backstage/catalog-client'; +import { + Metric, + ThresholdConfig, +} from '@red-hat-developer-hub/backstage-plugin-scorecard-common'; +import { MetricProvider } from '@red-hat-developer-hub/backstage-plugin-scorecard-node'; +import { GithubClient } from '../github/GithubClient'; +import { GithubCommit } from '../github/types'; +import { getRepositoryInformationFromEntity } from '../github/utils'; + +export const AI_ADOPTION_RATE_TIME_RANGES = ['7d', '30d', '90d']; + +export const AI_ADOPTION_RATE_THRESHOLD: ThresholdConfig = { + rules: [ + { key: 'success', expression: '>=0.2' }, + { key: 'warning', expression: '>=0.1' }, + { key: 'error', expression: '>=0' }, + ], +}; + +/** + * Known AI tool identifiers used in commit trailers. + * Matched case-insensitively against the "name" portion of the trailer + * value (everything before the first `<` in `Name `). + * + * Trailer keys (`Assisted-by:`, `Co-authored-by:`) are matched + * case-insensitively per the git trailer spec. + */ +const AI_TOOL_PATTERNS: string[] = [ + 'claude', + 'claude code', + 'cursor', + 'copilot', + 'github copilot', + 'codeium', + 'cody', + 'tabnine', + 'gemini', + 'amazon q', + 'amazon q developer', + 'windsurf', + 'devin', + 'aider', +]; + +function parseDays(range: string): number { + return parseInt(range.replace('d', ''), 10); +} + +function isMergeCommit(message: string): boolean { + return ( + message.startsWith('Merge pull request #') || + message.startsWith('Merge branch ') + ); +} + +/** + * Extracts the "name" portion from a trailer value. + * Git trailers follow the format `Name ` or just `Name`. + * Returns the trimmed, lowercased name part before the first `<`. + */ +function extractTrailerName(value: string): string { + const angleBracketIndex = value.indexOf('<'); + const namePart = + angleBracketIndex >= 0 ? value.slice(0, angleBracketIndex) : value; + return namePart.trim().toLowerCase(); +} + +function isAiAssistedCommit(message: string): boolean { + const lines = message.split('\n'); + for (const line of lines) { + const trimmed = line.trim(); + const lower = trimmed.toLowerCase(); + let value: string | undefined; + + if (lower.startsWith('assisted-by: ')) { + value = trimmed.slice('assisted-by: '.length).trim(); + } else if (lower.startsWith('co-authored-by: ')) { + value = trimmed.slice('co-authored-by: '.length).trim(); + } + + if (value) { + const name = extractTrailerName(value); + if (AI_TOOL_PATTERNS.includes(name)) { + return true; + } + } + } + return false; +} + +export class GithubAiAdoptionProvider implements MetricProvider<'number'> { + private readonly githubClient: GithubClient; + private readonly logger: LoggerService; + + private constructor(githubClient: GithubClient, logger: LoggerService) { + this.githubClient = githubClient; + this.logger = logger; + } + + static fromConfig( + config: Config, + options: { logger: LoggerService }, + ): GithubAiAdoptionProvider { + return new GithubAiAdoptionProvider( + new GithubClient(config, options.logger), + options.logger, + ); + } + + getProviderDatasourceId(): string { + return 'github'; + } + + getProviderId() { + return 'github.aiAdoption'; + } + + getMetrics(): Metric<'number'>[] { + return AI_ADOPTION_RATE_TIME_RANGES.map(range => ({ + id: `${this.getProviderId()}Rate[${range}]`, + title: `GitHub AI adoption rate (${range})`, + description: `Ratio of AI-assisted commits over the last ${range}.`, + type: 'number' as const, + thresholds: AI_ADOPTION_RATE_THRESHOLD, + history: true, + })); + } + + getCatalogFilter(): Record { + return { + 'metadata.annotations.github.com/project-slug': CATALOG_FILTER_EXISTS, + }; + } + + async calculateMetrics(entity: Entity): Promise> { + const repository = getRepositoryInformationFromEntity(entity); + const { target } = getEntitySourceLocation(entity); + + const maxDays = Math.max(...AI_ADOPTION_RATE_TIME_RANGES.map(parseDays)); + + const since = new Date(); + since.setDate(since.getDate() - maxDays); + + const commits = await this.githubClient.getCommitHistory( + target, + repository, + since, + ); + + const now = new Date(); + const results = new Map(); + + for (const range of AI_ADOPTION_RATE_TIME_RANGES) { + const days = parseDays(range); + const cutoff = new Date(now); + cutoff.setDate(cutoff.getDate() - days); + + const rangeCommits = commits.filter( + c => new Date(c.committedDate) >= cutoff, + ); + + const { ratio, total, ignored, aiAssisted, notAiAssisted } = + this.analyzeCommits(rangeCommits); + + this.logger.info( + `AI adoption [${range}] for ${repository.owner}/${repository.repo}: ` + + `${total} commits analyzed, ${ignored} merge commits ignored, ` + + `${aiAssisted} AI-assisted, ${notAiAssisted} not AI-assisted, ` + + `ratio=${ratio}`, + ); + + const metricId = `${this.getProviderId()}Rate[${range}]`; + results.set(metricId, ratio); + } + + return results; + } + + private analyzeCommits(commits: GithubCommit[]): { + ratio: number; + total: number; + ignored: number; + aiAssisted: number; + notAiAssisted: number; + } { + let ignored = 0; + let aiAssisted = 0; + let notAiAssisted = 0; + + for (const commit of commits) { + if (isMergeCommit(commit.message)) { + ignored++; + continue; + } + + if (isAiAssistedCommit(commit.message)) { + aiAssisted++; + } else { + notAiAssisted++; + } + } + + const analyzed = aiAssisted + notAiAssisted; + const ratio = analyzed > 0 ? aiAssisted / analyzed : 0; + + return { + ratio, + total: commits.length, + ignored, + aiAssisted, + notAiAssisted, + }; + } +} diff --git a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/module.ts b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/module.ts index a7dc873eb46..4f7f24e2701 100644 --- a/workspaces/scorecard/plugins/scorecard-backend-module-github/src/module.ts +++ b/workspaces/scorecard/plugins/scorecard-backend-module-github/src/module.ts @@ -24,6 +24,7 @@ import { import { GithubDeploymentPullRequestsCollector } from './collectors/GithubDeploymentPullRequestsCollector'; import { GithubDeploymentWorkflowRunsCollector } from './collectors/GithubDeploymentWorkflowRunsCollector'; import { GithubDeploymentsCollector } from './collectors/GithubDeploymentsCollector'; +import { GithubAiAdoptionProvider } from './metricProviders/GithubAiAdoptionProvider'; import { GithubOpenPRsProvider } from './metricProviders/GithubOpenPRsProvider'; export const scorecardModuleGithub = createBackendModule({ @@ -45,6 +46,7 @@ export const scorecardModuleGithub = createBackendModule({ ); metrics.addMetricProvider( GithubOpenPRsProvider.fromConfig(config, { logger }), + GithubAiAdoptionProvider.fromConfig(config, { logger }), ); }, }); diff --git a/workspaces/scorecard/plugins/scorecard-backend/README.md b/workspaces/scorecard/plugins/scorecard-backend/README.md index 10d55ae4bc9..fb89710d9e4 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/README.md +++ b/workspaces/scorecard/plugins/scorecard-backend/README.md @@ -94,14 +94,14 @@ For more information about schedule configuration options, see the [Metric Colle The following metric providers are available: -| Provider | Metric ID | Title | Description | Type | -| -------------- | --------------------------------------------------------------------------------------------------------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | ------- | -| **GitHub** | `github.openPRs` | GitHub open PRs | Count of open Pull Requests in GitHub | number | -| **Filecheck** | `filecheck.*` | File Checks | Checks whether specific files (e.g., `README.md`, `LICENSE`, `CODEOWNERS`) exist in a repository. | boolean | -| **Jira** | `jira.openIssues` | Jira open issues | The number of opened issues in Jira | number | -| **OpenSSF** | `openssf.*` | OpenSSF Security Scorecards | 18 security metrics from OpenSSF Scorecards (e.g., `openssf.codeReview`, `openssf.maintained`). Each returns a score from 0-10. | number | -| **Dependabot** | `dependabot.*` | Dependabot Alerts | Critical, High, Medium and Low CVE Alerts | number | -| **DORA** | `dora.deploymentFrequency`, `dora.medianLeadTimeForChanges`, `dora.meanTimeToRestore`, `dora.changeFailureRate` | DORA Metrics | Software delivery performance metrics based on DORA (DevOps Research and Assessment) | number | +| Provider | Metric ID | Title | Description | Type | +| -------------- | --------------------------------------------------------------------------------------------------------------- | --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | ------- | +| **GitHub** | `github.openPRs`, `github.aiAdoptionRate[7d]`, `github.aiAdoptionRate[30d]`, `github.aiAdoptionRate[90d]` | GitHub open PRs, AI adoption rate | Count of open Pull Requests; ratio of AI-assisted commits over 7d, 30d, and 90d windows | number | +| **Filecheck** | `filecheck.*` | File Checks | Checks whether specific files (e.g., `README.md`, `LICENSE`, `CODEOWNERS`) exist in a repository. | boolean | +| **Jira** | `jira.openIssues` | Jira open issues | The number of opened issues in Jira | number | +| **OpenSSF** | `openssf.*` | OpenSSF Security Scorecards | 18 security metrics from OpenSSF Scorecards (e.g., `openssf.codeReview`, `openssf.maintained`). Each returns a score from 0-10. | number | +| **Dependabot** | `dependabot.*` | Dependabot Alerts | Critical, High, Medium and Low CVE Alerts | number | +| **DORA** | `dora.deploymentFrequency`, `dora.medianLeadTimeForChanges`, `dora.meanTimeToRestore`, `dora.changeFailureRate` | DORA Metrics | Software delivery performance metrics based on DORA (DevOps Research and Assessment) | number | To use these providers, install the corresponding backend modules: diff --git a/workspaces/scorecard/plugins/scorecard-backend/docs/providers.md b/workspaces/scorecard/plugins/scorecard-backend/docs/providers.md index 719f7a920bb..89b234415ea 100644 --- a/workspaces/scorecard/plugins/scorecard-backend/docs/providers.md +++ b/workspaces/scorecard/plugins/scorecard-backend/docs/providers.md @@ -190,7 +190,7 @@ schedule: The following are examples of existing metric providers that you can reference: -- **GitHub Datasource**: [GithubOpenPRsProvider](../../scorecard-backend-module-github/src/metricProviders/GithubOpenPRsProvider.ts) +- **GitHub Datasource**: [GithubOpenPRsProvider](../../scorecard-backend-module-github/src/metricProviders/GithubOpenPRsProvider.ts), [GithubAiAdoptionProvider](../../scorecard-backend-module-github/src/metricProviders/GithubAiAdoptionProvider.ts) - **Jira Datasource**: [JiraOpenIssuesProvider](../../scorecard-backend-module-jira/src/metricProviders/JiraOpenIssuesProvider.ts) - **OpenSSF Datasource**: [OpenSSFMetricProvider](../../scorecard-backend-module-openssf/src/metricProviders/OpenSSFMetricProvider.ts) - **Filecheck Datasource** (batch / multi-metric): [FilecheckMetricProvider](../../scorecard-backend-module-filecheck/src/metricProviders/FilecheckMetricProvider.ts)