update-benchmarks.yml

name: Update Benchmarks

on:
  issues:
    types: [opened, labeled]

jobs:
  update:
    if: contains(github.event.issue.labels.*.name, 'benchmark-result')
    runs-on: ubuntu-latest
    permissions:
      contents: write
      issues: write

    steps:
      - uses: actions/checkout@v4

      - name: Parse benchmark data and append row
        id: parse
        uses: actions/github-script@v7
        with:
          script: |
            const body = context.payload.issue.body || '';

            // Extract JSON from <!-- benchmark-data ... --> comment
            const match = body.match(/<!--\s*benchmark-data\s*\n([\s\S]*?)\n-->/);
            if (!match) {
              core.setFailed('No benchmark-data block found in issue body.');
              return;
            }

            let data;
            try {
              data = JSON.parse(match[1].trim());
            } catch (e) {
              core.setFailed(`Failed to parse benchmark JSON: ${e.message}`);
              return;
            }

            const s = data.system || {};
            const cpu = (s.cpu || 'unknown').replace(/\|/g, '/').substring(0, 40);
            const gpu = s.cuda ? (s.gpu || 'GPU').replace(/\|/g, '/').substring(0, 30) : 'CPU only';
            const os = s.os || '—';
            const date = data.date || '—';

            const getField = (results, model, field, fallback = '—') => {
              const r = (results || []).find(x => x.model === model);
              return r && !r.error ? String(r[field]) + (field === 'rtf' ? 'x' : 's') : fallback;
            };

            const tinyRtf   = getField(data.results, 'tiny',   'rtf');
            const smallRtf  = getField(data.results, 'small',  'rtf');
            const mediumRtf = getField(data.results, 'medium', 'rtf');

            const row = `| ${date} | ${cpu} | ${gpu} | ${os} | ${tinyRtf} | ${smallRtf} | ${mediumRtf} |`;
            core.setOutput('row', row);
            core.setOutput('issue_number', String(context.payload.issue.number));

      - name: Write row to BENCHMARKS.md
        run: |
          ROW="${{ steps.parse.outputs.row }}"
          # Replace the placeholder "no results" lines and insert the new row before the footer
          python3 - <<'PYEOF'
          import re, os

          row = os.environ['ROW']
          path = 'BENCHMARKS.md'
          content = open(path).read()

          # Remove placeholder row if present
          content = re.sub(r'\| — \| — \| — \| — \| — \| — \| — \|\n\n\*No results yet.*\*\n', '', content)

          # Insert new row after the table header separator
          header_sep = '|------|-----|-----|----|----------|-----------|------------|'
          if header_sep in content:
              content = content.replace(header_sep, header_sep + '\n' + row, 1)

          open(path, 'w').write(content)
          print('Row inserted.')
          PYEOF
        env:
          ROW: ${{ steps.parse.outputs.row }}

      - name: Commit updated BENCHMARKS.md
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add BENCHMARKS.md
          git diff --cached --quiet || git commit -m "benchmarks: add result from issue #${{ steps.parse.outputs.issue_number }}"
          git push

      - name: Close issue with comment
        uses: actions/github-script@v7
        with:
          script: |
            await github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: parseInt('${{ steps.parse.outputs.issue_number }}'),
              body: '✅ Your benchmark result has been added to [BENCHMARKS.md](../blob/main/BENCHMARKS.md). Thank you!'
            });
            await github.rest.issues.update({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: parseInt('${{ steps.parse.outputs.issue_number }}'),
              state: 'closed'
            });