From f31239c0d25f14675c632cc1e95ed4b53dc90a3a Mon Sep 17 00:00:00 2001 From: Drew Davis Date: Tue, 15 Sep 2026 10:01:22 -0400 Subject: [PATCH] ci: Detect duplicate code with jscpd --- .github/workflows/jscpd.yml | 73 +++++++++++++++++++ .jscpd.json | 19 +++++ Makefile | 1 + knip.json | 3 +- package.json | 3 + scripts/ci/dupes.mjs | 137 ++++++++++++++++++++++++++++++++++++ yarn.lock | 92 ++++++++++++++++++++++++ 7 files changed, 327 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/jscpd.yml create mode 100644 .jscpd.json create mode 100644 scripts/ci/dupes.mjs diff --git a/.github/workflows/jscpd.yml b/.github/workflows/jscpd.yml new file mode 100644 index 0000000000..950885e4b8 --- /dev/null +++ b/.github/workflows/jscpd.yml @@ -0,0 +1,73 @@ +name: jscpd - Duplicate Code Analysis +on: + pull_request: + branches: [main] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + jscpd: + timeout-minutes: 10 + runs-on: ubuntu-24.04 + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout PR branch + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + # --baseline-from-ref scans the base branch's tree, so the whole + # history has to be present. Blobless, not treeless: jscpd needs the + # trees to materialise the base ref's files. + fetch-depth: 0 + filter: blob:none + + - name: Setup node + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version-file: '.nvmrc' + cache-dependency-path: 'yarn.lock' + cache: 'yarn' + + - name: Install dependencies + run: yarn install --immutable + + # Advisory check. dupes.mjs always exits 0; continue-on-error covers the + # case where it cannot run at all, so a tool failure can never block a PR. + - name: Build duplication report + continue-on-error: true + env: + BASE_REF: ${{ github.base_ref }} + run: + node scripts/ci/dupes.mjs --base "origin/$BASE_REF" --markdown > + /tmp/jscpd-comment.md + + - name: Read report + id: report + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + result-encoding: string + script: | + const fs = require('fs'); + const path = '/tmp/jscpd-comment.md'; + const body = fs.existsSync(path) + ? fs.readFileSync(path, 'utf8').trim() + : ''; + if (body === '') { + core.error('No duplication report was produced; skipping the PR comment.'); + return ''; + } + // Logged as well as commented so fork PRs, which cannot be + // commented on, still surface the result somewhere. + core.info(body); + return body; + + - name: Comment PR with duplication report + # Skip for fork PRs: GITHUB_TOKEN cannot write to the base repo + if: + steps.report.outputs.result != '' && + github.event.pull_request.head.repo.fork != true + uses: mshick/add-pr-comment@ec328af66588ab8f77cdeb2c264f14aba45bbf59 # v3.12.0 + with: + message: ${{ steps.report.outputs.result }} + message-id: jscpd-duplication diff --git a/.jscpd.json b/.jscpd.json new file mode 100644 index 0000000000..3906f9b8c0 --- /dev/null +++ b/.jscpd.json @@ -0,0 +1,19 @@ +{ + "minLines": 5, + "minTokens": 70, + "format": ["typescript", "tsx"], + "ignore": [ + "**/__tests__/**", + "**/__mocks__/**", + "**/__snapshots__/**", + "**/__fixtures__/**", + "**/*.test.ts", + "**/*.test.tsx", + "**/*.spec.ts", + "**/*.spec.tsx", + "**/*.stories.tsx", + "**/*.d.ts", + "packages/app/tests/e2e/**", + "packages/api/migrations/**" + ] +} diff --git a/Makefile b/Makefile index 2f94a89b63..3df1797226 100644 --- a/Makefile +++ b/Makefile @@ -114,6 +114,7 @@ ci-build: ci-lint: npx nx run-many -t ci:lint node scripts/ci/ratchet.mjs + -yarn dupes scripts/ci/check-openapi-sync.sh .PHONY: ci-openapi diff --git a/knip.json b/knip.json index 16cd951d26..09cb2158e4 100644 --- a/knip.json +++ b/knip.json @@ -49,7 +49,8 @@ "concurrently", "dotenv", "babel-plugin-react-compiler", - "@stryker-mutator/core" + "@stryker-mutator/core", + "jscpd" ], "include": ["nsExports"], "exclude": ["enumMembers", "duplicates"] diff --git a/package.json b/package.json index fb6916e7d4..6659ab07d9 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "eslint-plugin-security": "^3.0.1", "eslint-plugin-simple-import-sort": "^12.1.1", "husky": "^8.0.3", + "jscpd": "5.1.2", "knip": "^6.19.0", "lint-staged": "^13.1.2", "nx": "23.1.1", @@ -49,6 +50,8 @@ "knip:ci": "knip --reporter json", "ratchet": "node scripts/ci/ratchet.mjs", "ratchet:update": "node scripts/ci/ratchet.mjs --update", + "dupes": "node scripts/ci/dupes.mjs", + "dupes:all": "node scripts/ci/dupes.mjs --all", "lint": "npx nx run-many -t ci:lint", "lint:fix": "npx nx run-many -t lint:fix", "version": "make version", diff --git a/scripts/ci/dupes.mjs b/scripts/ci/dupes.mjs new file mode 100644 index 0000000000..e35889a470 --- /dev/null +++ b/scripts/ci/dupes.mjs @@ -0,0 +1,137 @@ +#!/usr/bin/env node +// Duplicate-code report. Advisory: always exits 0. +// +// jscpd's console reporter ignores --baseline-from-ref and prints every clone, +// so "what did this branch add" is only available as an `isNew` flag in the +// JSON report. This wraps the JSON run and renders the new clones. + +import { execFileSync } from 'node:child_process'; +import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const PACKAGES = ['packages/app', 'packages/api', 'packages/common-utils']; +const CONFIG = join(ROOT, '.jscpd.json'); + +// Cap the number of duplicates listed in the PR comment +const MAX_LISTED_IN_COMMENT = 20; + +const argv = process.argv.slice(2); +const all = argv.includes('--all'); +const markdown = argv.includes('--markdown'); +const baseIdx = argv.indexOf('--base'); +const baseArg = argv[baseIdx + 1]; +if (baseIdx !== -1 && (!baseArg || baseArg.startsWith('--'))) { + console.error('--base requires a git ref, e.g. --base origin/main'); + process.exit(0); +} +const base = baseIdx === -1 ? 'origin/main' : baseArg; + +// A malformed .jscpd.json makes jscpd log one line to stderr and carry on with +// its defaults, which scans markdown, tests and CHANGELOG.md. Fail loudly here +// instead of silently reporting noise. +try { + JSON.parse(readFileSync(CONFIG, 'utf8')); +} catch (e) { + console.error(`Cannot parse ${CONFIG}: ${e.message}`); + process.exit(0); +} + +const out = mkdtempSync(join(tmpdir(), 'jscpd-')); +// process.exit skips finally blocks, so every exit path clears the temp dir +// through here instead. +const bail = message => { + console.error(message); + rmSync(out, { recursive: true, force: true }); + process.exit(0); +}; + +const args = [ + ...PACKAGES, + '--absolute', + '--no-tips', + '--reporters', + 'json', + '--output', + out, +]; +if (!all) args.push('--baseline-from-ref', base); + +try { + execFileSync(join(ROOT, 'node_modules', '.bin', 'jscpd'), args, { + cwd: ROOT, + stdio: ['ignore', 'ignore', 'inherit'], + }); +} catch (e) { + bail(`jscpd failed to run: ${e.message}`); +} + +const report = join(out, 'jscpd-report.json'); +if (!existsSync(report)) bail('jscpd produced no report.'); + +const data = JSON.parse(readFileSync(report, 'utf8')); +rmSync(out, { recursive: true, force: true }); +const total = data.statistics?.total ?? {}; +const rel = p => (p.startsWith(`${ROOT}/`) ? p.slice(ROOT.length + 1) : p); +const at = f => `${rel(f.name)}:${f.start}-${f.end}`; +const duplicates = all + ? (data.duplicates ?? []) + : (data.duplicates ?? []).filter(c => c.isNew); +const plural = duplicates.length === 1 ? '' : 's'; + +if (markdown) { + const lines = ['## Duplicate code analysis', '']; + if (duplicates.length === 0) { + lines.push('No new duplicate code introduced by this PR.'); + } else { + lines.push( + `**${duplicates.length} new clone${plural}** introduced by this PR.`, + '', + ); + for (const c of duplicates.slice(0, MAX_LISTED_IN_COMMENT)) { + lines.push( + `- \`${at(c.firstFile)}\` ↔ \`${at(c.secondFile)}\` — ${c.lines} lines`, + ); + } + if (duplicates.length > MAX_LISTED_IN_COMMENT) { + lines.push(`- …and ${duplicates.length - MAX_LISTED_IN_COMMENT} more`); + } + lines.push( + '', + 'If one of these duplicates a helper that already exists, import it instead', + 'of redefining it. If the duplication is deliberate, ignore this comment —', + 'the check is advisory and never fails the build.', + ); + } + lines.push( + '', + '---', + `[jscpd](https://github.com/kucherenko/jscpd) compared this branch against \`${base}\``, + 'across app, api and common-utils. Only new clones are listed; the codebase', + `currently holds ${total.clones ?? 0} in total (${(total.percentage ?? 0).toFixed(2)}%).`, + 'Run `yarn dupes` locally to reproduce, or `yarn dupes:all` for the full picture.', + ); + console.log(lines.join('\n')); + process.exit(0); +} + +if (duplicates.length === 0) { + console.log( + all ? 'No duplicate code found.' : `No new duplicate code vs ${base}.`, + ); +} else { + const scope = all ? 'in app, api and common-utils' : `vs ${base}`; + console.log(`${duplicates.length} clone${plural} ${scope}:\n`); + for (const c of duplicates) { + console.log(` ${at(c.firstFile)}`); + console.log(` ${at(c.secondFile)} (${c.lines} lines)\n`); + } +} +if (!all) { + console.log( + `Total duplication: ${total.clones ?? 0} clones (${(total.percentage ?? 0).toFixed(2)}%). ` + + 'Run `yarn dupes:all` to list them.', + ); +} diff --git a/yarn.lock b/yarn.lock index 9946b4048c..7694bf8173 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18638,6 +18638,7 @@ __metadata: eslint-plugin-security: "npm:^3.0.1" eslint-plugin-simple-import-sort: "npm:^12.1.1" husky: "npm:^8.0.3" + jscpd: "npm:5.1.2" knip: "npm:^6.19.0" lint-staged: "npm:^13.1.2" nx: "npm:23.1.1" @@ -20516,6 +20517,97 @@ __metadata: languageName: node linkType: hard +"jscpd-darwin-arm64@npm:5.1.2": + version: 5.1.2 + resolution: "jscpd-darwin-arm64@npm:5.1.2" + conditions: os=darwin & cpu=arm64 + languageName: node + linkType: hard + +"jscpd-darwin-x64@npm:5.1.2": + version: 5.1.2 + resolution: "jscpd-darwin-x64@npm:5.1.2" + conditions: os=darwin & cpu=x64 + languageName: node + linkType: hard + +"jscpd-linux-arm64-gnu@npm:5.1.2": + version: 5.1.2 + resolution: "jscpd-linux-arm64-gnu@npm:5.1.2" + conditions: os=linux & cpu=arm64 & libc=glibc + languageName: node + linkType: hard + +"jscpd-linux-arm64-musl@npm:5.1.2": + version: 5.1.2 + resolution: "jscpd-linux-arm64-musl@npm:5.1.2" + conditions: os=linux & cpu=arm64 & libc=musl + languageName: node + linkType: hard + +"jscpd-linux-x64-gnu@npm:5.1.2": + version: 5.1.2 + resolution: "jscpd-linux-x64-gnu@npm:5.1.2" + conditions: os=linux & cpu=x64 & libc=glibc + languageName: node + linkType: hard + +"jscpd-linux-x64-musl@npm:5.1.2": + version: 5.1.2 + resolution: "jscpd-linux-x64-musl@npm:5.1.2" + conditions: os=linux & cpu=x64 & libc=musl + languageName: node + linkType: hard + +"jscpd-windows-arm64-msvc@npm:5.1.2": + version: 5.1.2 + resolution: "jscpd-windows-arm64-msvc@npm:5.1.2" + conditions: os=win32 & cpu=arm64 + languageName: node + linkType: hard + +"jscpd-windows-x64-msvc@npm:5.1.2": + version: 5.1.2 + resolution: "jscpd-windows-x64-msvc@npm:5.1.2" + conditions: os=win32 & cpu=x64 + languageName: node + linkType: hard + +"jscpd@npm:5.1.2": + version: 5.1.2 + resolution: "jscpd@npm:5.1.2" + dependencies: + jscpd-darwin-arm64: "npm:5.1.2" + jscpd-darwin-x64: "npm:5.1.2" + jscpd-linux-arm64-gnu: "npm:5.1.2" + jscpd-linux-arm64-musl: "npm:5.1.2" + jscpd-linux-x64-gnu: "npm:5.1.2" + jscpd-linux-x64-musl: "npm:5.1.2" + jscpd-windows-arm64-msvc: "npm:5.1.2" + jscpd-windows-x64-msvc: "npm:5.1.2" + dependenciesMeta: + jscpd-darwin-arm64: + optional: true + jscpd-darwin-x64: + optional: true + jscpd-linux-arm64-gnu: + optional: true + jscpd-linux-arm64-musl: + optional: true + jscpd-linux-x64-gnu: + optional: true + jscpd-linux-x64-musl: + optional: true + jscpd-windows-arm64-msvc: + optional: true + jscpd-windows-x64-msvc: + optional: true + bin: + jscpd: run-jscpd.js + checksum: 10c0/ed84dda137489c4c9e068b1d2da8ea046fd68726e5240c3b0bf7e1394e82bd17c198e5f3ebe37e2846f2d42f7605cf2aba7dde0fc7f54c098a42a2980f89b40e + languageName: node + linkType: hard + "jsdom@npm:^26.1.0": version: 26.1.0 resolution: "jsdom@npm:26.1.0"