From 1aaa3babe1742290509c79af7b5b4a810f478d79 Mon Sep 17 00:00:00 2001 From: Soxoj Date: Sun, 17 May 2026 20:08:09 +0200 Subject: [PATCH 1/2] Web version + some improvements --- .gitignore | 23 + docs.md | 301 +++++++++++ gitcolombo.html | 1080 ++++++++++++++++++++++++++++++++++++++++ gitcolombo.py | 218 ++++++++ test_gitcolombo_e2e.py | 229 +++++++++ 5 files changed, 1851 insertions(+) create mode 100644 docs.md create mode 100644 gitcolombo.html create mode 100644 test_gitcolombo_e2e.py diff --git a/.gitignore b/.gitignore index e69de29..3988b1e 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1,23 @@ +# system / editor +.DS_Store + +# python +__pycache__/ +*.pyc + +# claude code state +.claude/ + +# secrets — NEVER commit +.token +**/.token + +# session artefacts (cloned target repos, tool clones, reference downloads) +GONZOsint/ +repos/ +email-tools/ +gitrecon/ +ghintel.html + +# local service script +move_repos.py diff --git a/docs.md b/docs.md new file mode 100644 index 0000000..de25c0b --- /dev/null +++ b/docs.md @@ -0,0 +1,301 @@ +# Extraction methods + +This document describes how the standalone web tool +(`gitcolombo.html` — a single file, open it in a browser) and the +Python CLI (`gitcolombo.py`) discover identities (name + email pairs) +from GitHub. + +All methods rely on the public GitHub REST API (no auth required, but a +personal access token raises the rate limit from 60 to 5000 requests +per hour). + +--- + +## Mode A — Find real email by username + +Goal: given a GitHub username, return a ranked list of `(email, name)` +identities that most likely belong to the real person behind the +account. + +This is the same general approach taken by +[gitrecon](https://github.com/GONZOsint/gitrecon). + +### Step 1 — Profile lookup + +``` +GET https://api.github.com/users/{username} +``` + +We read `name`, `login`, `public_repos`, `created_at`. The `name` +field is later used to rank identities — an email paired with a +commit-author name that matches the profile name is treated as +high-confidence. + +### Step 2 — PGP keys (strongest signal) + +``` +GET https://api.github.com/users/{username}/gpg_keys +``` + +Returns the PGP public keys the user has uploaded to GitHub. Each +key (and each subkey) exposes its UIDs as a list of objects: + +```json +[ + { + "key_id": "D6C7A19E9CFF4BB8", + "revoked": false, + "emails": [ + {"email": "real@example.com", "verified": true} + ], + "subkeys": [ + { + "key_id": "1053D098A0070FEA", + "revoked": false, + "emails": [...] + } + ] + } +] +``` + +Why this is the strongest signal in this tool: + +- The user uploaded the key themselves — this is direct identity disclosure, not + inferred from commit metadata. +- `verified: true` means GitHub sent a confirmation email to that address and the + user clicked the link. This is **cryptographically and procedurally verified**, + unlike `commit.author.email`, which is freely settable in `git config user.email`. +- The endpoint is public, no auth required, and unaffected by "Keep my email + address private". + +We walk both top-level keys and their `subkeys[]`, skipping any with +`revoked: true`. Emails are deduplicated against later sources, so a +match between a PGP UID and a commit email lights up both signals and +boosts ranking. + +Limitations: many users never upload a PGP key. None of the surveyed +peer tools (gitrecon, EmailFinder, GitFive, Gitmails, etc.) use this +endpoint, so it is a unique source. + +### Step 3 — Public events (primary commit source) + +``` +GET https://api.github.com/users/{username}/events/public?per_page=100 +``` + +The `/events/public` endpoint returns up to ~90 days of the user's +public activity. Each entry has a `type` and a `payload`. We are only +interested in `PushEvent`: + +```json +{ + "type": "PushEvent", + "repo": { "name": "owner/repo" }, + "payload": { + "commits": [ + { + "sha": "...", + "author": { "email": "real@example.com", "name": "Real Name" }, + "message": "..." + } + ] + } +} +``` + +For every `PushEvent.payload.commits[]` we collect +`author.email` + `author.name`. + +Why this works: when the user pushes from their local machine, git +stamps the commit with whatever email is configured locally +(`user.email`). GitHub keeps that email in the public event feed even +if the user later enables "Keep my email address private" in the web +UI — events that already happened are not rewritten. + +### Step 4 — Commits fallback + +If `/events/public` returned nothing useful (e.g. the user has been +inactive for more than ~90 days, or every event email is +`@users.noreply.github.com`), we fall back to scanning commits in +the user's own repositories: + +``` +GET https://api.github.com/users/{username}/repos?per_page=100&sort=updated +``` + +We keep non-fork repositories and, for each one, query: + +``` +GET https://api.github.com/repos/{owner}/{repo}/commits?author={username}&per_page=100 +``` + +For every returned commit we read both author and committer: + +- `commit.author.email` / `commit.author.name` +- `commit.committer.email` / `commit.committer.name` + +The `?author=` filter on the commits endpoint accepts either a +GitHub login or an email — using the login restricts results to +commits GitHub has actually linked to this account. + +### Step 5 — Ranking + +Each unique `(email, name)` pair is annotated with: + +- `count` — how often it was seen +- `sources` — which events/repos it came from +- `matchesProfile` — whether `name` (case-insensitive, trimmed) equals + the profile `name` from Step 1 + +Identities where `matchesProfile === true` are shown first, then +remaining ones sorted by `count` descending. + +--- + +## Mode B — Extract emails from commits + +Goal: dump every `(email, login)` pair that appears as a commit author +across a user's or org's repositories — useful for mapping a team or +auditing a single repo. + +### Step 1 — Resolve target + +The input is either `username` or `owner/repo`. + +For an `owner/repo`: + +``` +GET https://api.github.com/repos/{owner}/{repo} +``` + +For a `username`, we paginate through: + +``` +GET https://api.github.com/users/{username}/repos?per_page=100&page={n} +``` + +Forks are skipped when the **Skip forks** checkbox is on (default). + +### Step 2 — Walk commits + +For every repository: + +``` +GET https://api.github.com/repos/{owner}/{repo}/commits?per_page=100&page={n} +``` + +For every commit object: + +```json +{ + "sha": "...", + "commit": { + "author": { "email": "alice@example.com", "name": "Alice" }, + "committer": { "email": "bob@example.com", "name": "Bob" } + }, + "author": { "login": "alice42" }, + "committer": { "login": "bob42" } +} +``` + +We collect: + +- `commit.author.email` + top-level `author.login` +- `commit.committer.email` + top-level `committer.login` + +(Top-level `author.login` is GitHub's resolved account for the commit; +it may be `null` for commits whose email isn't tied to any GitHub +account.) + +Pagination stops when the API returns an empty array. When +**Only first 100 commits/repo** is enabled, only page 1 is requested. + +### Step 3 — Dedupe + +Identities are deduplicated by lowercased email. Duplicates increment +a `count` field instead of producing new rows. + +--- + +## Commit-message trailer extraction + +For every commit body the tool sees (whether from `/events`, +`/search/commits`, or `/repos/{owner}/{repo}/commits`), it scans the +message for well-known git trailers and treats each match as an +additional identity. The regex is anchored per line and matches: + +``` +^(Signed-off-by|Co-authored-by|Reviewed-by|Tested-by|Reported-by|Acked-by|Suggested-by|Cc):\s+(name)\s+<(email)>$ +``` + +| Trailer | Where it usually comes from | +|---|---| +| `Signed-off-by:` | `git commit -s` — DCO projects (Linux kernel, Docker, kubernetes, many enterprise projects with CLAs) | +| `Co-authored-by:` | GitHub's UI when squash-merging a PR with multiple contributors | +| `Reviewed-by:` | Patch-review workflows (kernel, libvirt, qemu, …) | +| `Tested-by:` / `Acked-by:` / `Reported-by:` / `Suggested-by:` | Kernel-style review chains | +| `Cc:` | Notification chain in mailing-list patches | + +Why this is a strong signal: trailer lines are added **intentionally** +by tooling or by humans during patch review, and they typically point +to real mailboxes (no point in CC-ing or signing-off as a fake +address — it defeats the workflow's purpose). Even when the commit +author's email is a noreply, the trailers often carry plain emails. + +Tool comparison: among the 14 surveyed peer tools, only `fue` (Ruby, +GraphQL-based) and `GitFive` parse these trailers. Gitcolombo's +implementation mirrors fue's regex while staying fully in-browser +(Mode A + Mode B) and in the Python CLI's `--search` flow. + +Each trailer email is tagged with a `trailer` badge in the UI (hover +to see the trailer types it appeared in). The Python CLI prints the +roles for each identity in square brackets, e.g. +`[signed-off-by, reviewed-by, author]`. + +## Filters + +### Ignore `@users.noreply.github.com` + +When the checkbox is enabled, any email matching the regular +expression `/@users\.noreply\.github\.com$/i` is dropped before being +recorded. This is the address GitHub generates for users who turn on +"Keep my email address private" — it is *not* a real mailbox and +matching it back to a person requires extracting the leading numeric +prefix (the user's internal numeric id) which doesn't directly reveal +their identity. + +These emails are still useful as a signal that the user is privacy-conscious, +which is why the option is a toggle, not a hard filter. + +### Skip forks (Mode B only) + +Fork repositories are noisy: their commit history is dominated by the +upstream project, not the user being investigated. The checkbox is on +by default. + +### Limit to first 100 commits/repo (Mode B only) + +Each repo's commit list is paginated. Stopping after page 1 keeps the +total request count predictable; turn this off if you need full +coverage on small projects. + +--- + +## Rate limits + +GitHub allows 60 unauthenticated requests per hour per IP and 5000 +requests per hour per authenticated token. The tool surfaces a clear +message on `403`/`429` responses and shows whatever partial results +were already collected. + +A token only needs the default `public_repo` read scope — paste a +fine-grained PAT with no extra permissions and you're set. + +--- + +## References + +- gitrecon — +- ghintel — (UI inspiration) +- GitHub REST API — diff --git a/gitcolombo.html b/gitcolombo.html new file mode 100644 index 0000000..367df54 --- /dev/null +++ b/gitcolombo.html @@ -0,0 +1,1080 @@ + + + + +gitcolombo // github osint + + + + + +
+ + +
+
// extract identities from public github activity
+ +
+ +
+ + +
+ + +
+ + + +
+
+ + +
+
+
+ + + + + + + + + + +
+ + + +
+ +
+
+ +
+ +
+ +
+ +
github.com/Soxoj/gitcolombo // standalone
+
+ +
+ + + + + + + + + + diff --git a/gitcolombo.py b/gitcolombo.py index 44143a2..807b9cc 100755 --- a/gitcolombo.py +++ b/gitcolombo.py @@ -20,12 +20,35 @@ GITHUB_USER_STATS = 'https://api.github.com/users/{}' GITHUB_USER_REPOS = 'https://api.github.com/users/{}/repos?per_page=100&page={}' +GITHUB_USER_GPG_KEYS = 'https://api.github.com/users/{}/gpg_keys' +GITHUB_SEARCH_COMMITS = 'https://api.github.com/search/commits?q=author:{}&per_page=100&page={}' GITHUB_PER_PAGE_LIMIT = 100 +GITHUB_SEARCH_MAX_PAGES = 10 # /search/* caps results at 1000 + +# Well-known git trailer keys (DCO sign-off, GitHub co-authorship, kernel reviews). +# A real email in any of these is a strong identity signal: trailers are +# typically added intentionally by tooling (`git commit -s`, GitHub UI's +# "Co-authored-by", patch-review workflows) rather than being auto-generated. +TRAILER_RE = re.compile( + r'^(?PSigned-off-by|Co-authored-by|Reviewed-by|Tested-by|Reported-by|Acked-by|Suggested-by|Cc):\s+(?P[^<]+?)\s+<(?P[^>]+)>\s*$', + re.MULTILINE | re.IGNORECASE, +) SYSTEM_EMAILS = [ 'noreply@github.com', ] +# Service noreply addresses from any vendor (github, anthropic, gitlab, ...) +# plus GitHub's user-private `{id}+{login}@users.noreply.github.com` pattern. +SYSTEM_EMAIL_RE = re.compile( + r'(^(?:noreply|no-reply|donotreply|do-not-reply)@|@users\.noreply\.github\.com$)', + re.IGNORECASE, +) + + +def is_system_email(email): + return bool(email and SYSTEM_EMAIL_RE.search(email)) + def get_public_repos_count(nickname): url = GITHUB_USER_STATS @@ -63,6 +86,186 @@ def get_github_repos(nickname, only_forks=True, repos_count=GITHUB_PER_PAGE_LIMI return repos_links +def get_gpg_keys_emails(nickname, token=None): + """Fetch user-uploaded PGP keys via /users/{u}/gpg_keys and yield emails. + + These emails come from the key's UIDs — the user uploaded them themselves, + so this is a direct identity disclosure. `verified=True` means GitHub has + confirmed the user controls that mailbox. + + Yields dicts: {email, verified, key_id, created_at, source ('primary'|'subkey')}. + """ + headers = { + 'Accept': 'application/vnd.github+json', + 'User-Agent': 'gitcolombo', + } + if token: + headers['Authorization'] = 'Bearer ' + token + req_url = GITHUB_USER_GPG_KEYS.format(nickname) + req = urllib.request.Request(req_url, headers=headers) + try: + response = urllib.request.urlopen(req, timeout=20) + except Exception as e: + logging.debug('gpg_keys %s: %s', req_url, e) + return + try: + keys = json.loads(response.read().decode('utf-8')) + except Exception as e: + logging.debug('gpg_keys parse error: %s', e) + return + seen = set() + + def _walk(key, source): + if not key or key.get('revoked'): + return + key_id = key.get('key_id', '') + created = key.get('created_at', '') + for entry in (key.get('emails') or []): + email = entry.get('email') + if not email: + continue + k = email.lower() + if k in seen: + continue + seen.add(k) + yield { + 'email': email, + 'verified': bool(entry.get('verified')), + 'key_id': key_id, + 'created_at': created, + 'source': source, + } + + for key in (keys or []): + for row in _walk(key, 'primary'): + yield row + for sub in (key.get('subkeys') or []): + for row in _walk(sub, 'subkey'): + yield row + + +def print_gpg_results(results, ignore_noreply=True): + """Pretty-print get_gpg_keys_emails() output.""" + rows = [ + r for r in results + if not (ignore_noreply and is_system_email(r['email'])) + ] + if not rows: + return False + print('PGP key UIDs (uploaded by the user, public via /users/{u}/gpg_keys):') + print(DELIMITER) + # verified first + rows.sort(key=lambda r: (not r['verified'], r['email'])) + for r in rows: + flag = 'verified' if r['verified'] else 'unverified' + print(' {:40} [{}] key_id={} ({})'.format( + r['email'], flag, r['key_id'] or '?', r['source'] + )) + print() + return True + + +def search_commits_by_author(nickname, token=None): + """Use /search/commits?q=author:{u} to find commits across all of public GitHub. + + Yields dicts: {email, name, role ('author'|'committer'), repo, sha, date}. + """ + headers = { + 'Accept': 'application/vnd.github+json', + 'User-Agent': 'gitcolombo', + } + if token: + headers['Authorization'] = 'Bearer ' + token + seen = set() + for page in range(1, GITHUB_SEARCH_MAX_PAGES + 1): + req_url = GITHUB_SEARCH_COMMITS.format(nickname, page) + req = urllib.request.Request(req_url, headers=headers) + try: + response = urllib.request.urlopen(req, timeout=20) + except Exception as e: + logging.debug('search/commits %s: %s', req_url, e) + return + try: + data = json.loads(response.read().decode('utf-8')) + except Exception as e: + logging.debug('search/commits parse error: %s', e) + return + items = data.get('items') or [] + if not items: + return + for item in items: + commit = item.get('commit') or {} + repo = (item.get('repository') or {}).get('full_name', '') + sha = item.get('sha', '') + date = (commit.get('author') or {}).get('date', '') + message = commit.get('message') or '' + for role in ('author', 'committer'): + who = commit.get(role) or {} + email = who.get('email') + name = who.get('name') or '' + if not email: + continue + key = (email.lower(), name.lower(), role) + if key in seen: + continue + seen.add(key) + yield { + 'email': email, 'name': name, 'role': role, + 'repo': repo, 'sha': sha, 'date': date, + } + # trailers in the commit message body (Signed-off-by, Co-authored-by, ...) + for tm in TRAILER_RE.finditer(message): + t_key = tm.group('key').lower() + t_name = (tm.group('name') or '').strip() + t_email = (tm.group('email') or '').strip() + if not t_email: + continue + # reject malformed names: ':' implies another trailer label was + # crammed onto the same line; '@' implies a @-mention or stray + # handle. Real personal names don't contain either. + if ':' in t_name or '@' in t_name: + continue + key = (t_email.lower(), t_name.lower(), t_key) + if key in seen: + continue + seen.add(key) + yield { + 'email': t_email, 'name': t_name, 'role': t_key, + 'repo': repo, 'sha': sha, 'date': date, + } + if len(items) < GITHUB_PER_PAGE_LIMIT: + return + + +def print_search_results(results, ignore_noreply=True): + """Pretty-print search_commits_by_author() output grouped by (email, name).""" + groups = {} + for r in results: + if ignore_noreply and is_system_email(r['email']): + continue + key = (r['email'], r['name']) + groups.setdefault(key, []).append(r) + + if not groups: + print('No public commits found via /search/commits.') + return + + print('Found {} unique (email, name) identities:'.format(len(groups))) + print(DELIMITER) + ordered = sorted(groups.items(), key=lambda kv: -len(kv[1])) + for (email, name), rows in ordered: + repos = sorted({r['repo'] for r in rows if r['repo']}) + roles = sorted({r['role'] for r in rows}) + roles_str = ', '.join(roles) + print('{} <{}> x{} [{}]'.format( + name or '', email, len(rows), roles_str + )) + for repo in repos[:5]: + print(' repo: {}'.format(repo)) + if len(repos) > 5: + print(' ... +{} more repos'.format(len(repos) - 5)) + + def find_all_repos_recursively(path): git_dirs = [] for current_dir, dirs, _ in os.walk(path): @@ -311,6 +514,10 @@ def main(): parser.add_argument('-u', '--url', help='url of git repo') parser.add_argument('--github', action='store_true', help='try to extract extended info from GitHub') parser.add_argument('--nickname', type=str, help='try to download repos from all platforms by nickname') + parser.add_argument('--search', type=str, metavar='USERNAME', + help='use /search/commits API to find emails by author (no cloning, ~1000 results max)') + parser.add_argument('--no-ignore-noreply', action='store_true', + help='do not filter @users.noreply.github.com / noreply@github.com from --search results') parser.add_argument('-r', '--recursive', action='store_true', help='recursive directory processing') parser.add_argument('--debug', action='store_true', help='print debug information') # TODO: clone repos as bare @@ -320,6 +527,17 @@ def main(): log_level = logging.INFO if not args.debug else logging.DEBUG logging.basicConfig(level=log_level, format='-'*40 + '\n%(levelname)s: %(message)s') + if args.search: + token = os.environ.get('GITHUB_TOKEN') + ignore = not args.no_ignore_noreply + gpg = list(get_gpg_keys_emails(args.search, token=token)) + had_gpg = print_gpg_results(gpg, ignore_noreply=ignore) + results = list(search_commits_by_author(args.search, token=token)) + print_search_results(results, ignore_noreply=ignore) + if not had_gpg and not results: + print('No emails found via /gpg_keys or /search/commits.') + return + analyst = None analyst = GitAnalyst() diff --git a/test_gitcolombo_e2e.py b/test_gitcolombo_e2e.py new file mode 100644 index 0000000..4594167 --- /dev/null +++ b/test_gitcolombo_e2e.py @@ -0,0 +1,229 @@ +"""Live end-to-end tests for gitcolombo — hit the real GitHub API. + +These are intentionally NOT unit tests. They make real HTTP requests to +api.github.com and are skipped automatically in CI (when $CI is set, +which both GitHub Actions and most other runners do by default). + +Run locally: + python3 -m unittest test_gitcolombo_e2e -v + # optional, much higher rate limits: + GITHUB_TOKEN=ghp_… python3 -m unittest test_gitcolombo_e2e -v +""" +import os +import unittest + +import gitcolombo + + +IS_CI = bool(os.environ.get("CI")) + + +@unittest.skipIf(IS_CI, "live network test — skipped in CI ($CI is set)") +class TestGpgKeysLive(unittest.TestCase): + """/users/{u}/gpg_keys against a real user known to have a verified PGP UID. + + Target: `sindresorhus`. His primary key (key_id D6C7A19E9CFF4BB8, uploaded + 2016-04) lists `sindresorhus@gmail.com` with `verified: true`. The key has + been stable for years; if GitHub ever returns a different shape or the user + rotates keys, swap the target rather than weakening the assertion. + """ + + TARGET = "sindresorhus" + EXPECTED_EMAIL = "sindresorhus@gmail.com" + + def setUp(self): + self.token = os.environ.get("GITHUB_TOKEN") + + def test_returns_at_least_one_entry(self): + results = list(gitcolombo.get_gpg_keys_emails(self.TARGET, token=self.token)) + self.assertGreater( + len(results), 0, + f"expected at least one GPG email for @{self.TARGET}", + ) + + def test_verified_email_present(self): + results = list(gitcolombo.get_gpg_keys_emails(self.TARGET, token=self.token)) + verified = {r["email"].lower() for r in results if r["verified"]} + self.assertIn( + self.EXPECTED_EMAIL.lower(), verified, + f"expected verified UID {self.EXPECTED_EMAIL} in {sorted(verified)}", + ) + + def test_entry_shape(self): + results = list(gitcolombo.get_gpg_keys_emails(self.TARGET, token=self.token)) + match = next( + (r for r in results if r["email"].lower() == self.EXPECTED_EMAIL.lower()), + None, + ) + self.assertIsNotNone(match, "expected entry not found") + self.assertTrue(match["verified"]) + self.assertIn(match["source"], ("primary", "subkey")) + self.assertTrue(match["key_id"], "key_id should not be empty") + + +@unittest.skipIf(IS_CI, "live network test — skipped in CI ($CI is set)") +class TestTrailersLive(unittest.TestCase): + """Verify trailer parsing against /search/commits results for a kernel maintainer. + + Linux kernel commits ALWAYS have `Signed-off-by:` (DCO is enforced), + and most also carry one or more of `Acked-by:`, `Reviewed-by:`, + `Tested-by:`, `Reported-by:`, `Cc:` from the patch-review chain. + These tests assert that the extractor surfaces ≥ 2 distinct trailer + types from a single `--search gregkh` run, plus a specific check for + `signed-off-by` which is guaranteed by DCO. + + Target: gregkh (Greg Kroah-Hartman). His commit pool on GitHub + spans hundreds of repos (including kernel mirrors), so /search/commits + will return diverse trailer types regardless of GitHub indexing + fluctuations. + """ + + TARGET = "gregkh" + + TRAILER_KEYS = frozenset({ + "signed-off-by", "co-authored-by", "reviewed-by", "tested-by", + "reported-by", "acked-by", "suggested-by", "cc", + }) + + @classmethod + def setUpClass(cls): + token = os.environ.get("GITHUB_TOKEN") + cls.results = list(gitcolombo.search_commits_by_author( + cls.TARGET, token=token, + )) + cls.roles_seen = {r["role"] for r in cls.results} + cls.trailer_roles = cls.roles_seen & cls.TRAILER_KEYS + + def test_collected_some_identities(self): + self.assertGreater( + len(self.results), 50, + "expected dozens of identities from kernel maintainer's commits", + ) + + def test_signed_off_by_present(self): + # DCO-enforced project — Signed-off-by must appear. + self.assertIn("signed-off-by", self.trailer_roles) + + def test_at_least_two_distinct_trailer_types(self): + self.assertGreaterEqual( + len(self.trailer_roles), 2, + f"expected ≥2 trailer kinds, saw {sorted(self.trailer_roles)}", + ) + + def test_trailer_entries_have_email_and_role(self): + trailer_entries = [r for r in self.results if r["role"] in self.TRAILER_KEYS] + self.assertGreater(len(trailer_entries), 0) + for entry in trailer_entries[:5]: + self.assertTrue(entry["email"]) + self.assertIn("@", entry["email"]) + self.assertTrue(entry["repo"]) + + +class TestSystemEmailFilter(unittest.TestCase): + """is_system_email() classifies vendor noreply addresses across domains.""" + + SYSTEM = [ + "noreply@github.com", + "noreply@anthropic.com", # Claude Code Co-Authored-By trailer + "no-reply@gitlab.com", + "donotreply@example.org", + "DoNotReply@example.org", # case-insensitive + "12345+ghost@users.noreply.github.com", + ] + + REAL = [ + "alice@example.com", + "torvalds@linux-foundation.org", + "noreply-thing@example.com", # not a noreply, just starts with noreply- + "support@noreply.example.com", # noreply is in the domain but mailbox is real + ] + + def test_system_emails_classified(self): + for e in self.SYSTEM: + self.assertTrue(gitcolombo.is_system_email(e), f"expected system: {e}") + + def test_real_emails_not_classified(self): + for e in self.REAL: + self.assertFalse(gitcolombo.is_system_email(e), f"expected real: {e}") + + +class TestTrailerRegex(unittest.TestCase): + """Offline sanity check on the trailer regex. + + Not strictly e2e — but a fast deterministic guard that the regex + can extract every supported trailer type from a canonical message. + Runs in CI too (no network). + """ + + SAMPLE_MESSAGE = ( + "usb: serial: cp210x: add support for a new device\n" + "\n" + "This patch adds support for the newly released widget.\n" + "\n" + "Reported-by: Foo Bar \n" + "Tested-by: Tester One \n" + "Reviewed-by: Bob Reviewer \n" + "Acked-by: Carol Maintainer \n" + "Cc: stable@vger.kernel.org \n" + "Co-authored-by: Alice Coder \n" + "Suggested-by: Dave Idea \n" + "Signed-off-by: Greg Kroah-Hartman \n" + ) + + def test_all_trailer_types_match(self): + keys = {m.group("key").lower() + for m in gitcolombo.TRAILER_RE.finditer(self.SAMPLE_MESSAGE)} + expected = { + "signed-off-by", "co-authored-by", "reviewed-by", "tested-by", + "reported-by", "acked-by", "suggested-by", "cc", + } + self.assertEqual(keys, expected) + + def test_email_extraction(self): + emails = {m.group("email").lower() + for m in gitcolombo.TRAILER_RE.finditer(self.SAMPLE_MESSAGE)} + self.assertIn("gregkh@linuxfoundation.org", emails) + self.assertIn("alice@nowhere.dev", emails) + self.assertIn("foo@bar.com", emails) + + def test_no_match_on_plain_prose(self): + # "Signed-off-by:" mentioned in body text without proper format must not match + prose = "We use Signed-off-by lines in this project but here it's just text." + self.assertEqual(list(gitcolombo.TRAILER_RE.finditer(prose)), []) + + def test_rejects_crammed_line_with_mention_and_label(self): + # Regression: a Linux kernel commit body line of the form + # Reported-by: Foo Bar (@kaitas) Signed-off-by: Linus + # used to be captured with the entire mush as the "name". The lazy + # regex now still matches, but the post-filter on ':' / '@' must drop + # the entry instead of producing the malformed name. + from gitcolombo import search_commits_by_author # noqa: F401 + msg = ( + "Reported-by: Foo Bar (@kaitas) Signed-off-by: Linus Torvalds " + "\n" + ) + # simulate the post-filter step used inside search_commits_by_author + captured = [] + for tm in gitcolombo.TRAILER_RE.finditer(msg): + name = (tm.group("name") or "").strip() + if ":" in name or "@" in name: + continue + captured.append(name) + self.assertEqual(captured, [], f"should have rejected, got: {captured}") + + +@unittest.skipIf(IS_CI, "live network test — skipped in CI ($CI is set)") +class TestGpgKeysNoResult(unittest.TestCase): + """A user without uploaded PGP keys must yield zero results, not crash. + + Target: `GONZOsint` — verified during the methodology audit to have 0 keys. + """ + + def test_empty_result_for_user_without_keys(self): + token = os.environ.get("GITHUB_TOKEN") + results = list(gitcolombo.get_gpg_keys_emails("GONZOsint", token=token)) + self.assertEqual(results, []) + + +if __name__ == "__main__": + unittest.main() From 0e03f2b54882bcefe714102bddffdd5c6e264145 Mon Sep 17 00:00:00 2001 From: Soxoj Date: Mon, 18 May 2026 15:04:07 +0200 Subject: [PATCH 2/2] Additional search across repos --- docs.md | 32 ++++++++++++++++++++++++++++---- gitcolombo.html | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/docs.md b/docs.md index de25c0b..c185799 100644 --- a/docs.md +++ b/docs.md @@ -78,7 +78,7 @@ Limitations: many users never upload a PGP key. None of the surveyed peer tools (gitrecon, EmailFinder, GitFive, Gitmails, etc.) use this endpoint, so it is a unique source. -### Step 3 — Public events (primary commit source) +### Step 3 — Public events (recent commits) ``` GET https://api.github.com/users/{username}/events/public?per_page=100 @@ -113,9 +113,33 @@ stamps the commit with whatever email is configured locally if the user later enables "Keep my email address private" in the web UI — events that already happened are not rewritten. -### Step 4 — Commits fallback +### Step 4 — Commit search across all public repos -If `/events/public` returned nothing useful (e.g. the user has been +``` +GET https://api.github.com/search/commits?q=author:{username}&per_page=100&page={n} +``` + +Unlike `/events/public`, the commit-search index spans **every public +commit on GitHub** that has been associated (by email or login) with the +target account, including commits in repositories the user does not +own. It also surfaces old commits that fell out of the 90-day events +window. + +The endpoint caps results at 1000 commits = 10 pages of 100. Rate limit +is stricter than core: 10 req/min unauthenticated, 30 req/min with a +token (vs 5000/hr for core endpoints). + +For each `item` we read both the author/committer pair from +`item.commit.{author, committer}` and parse trailers from +`item.commit.message` (see the "Commit-message trailer extraction" +section). + +This is the same approach GitFive uses for its "lite" search mode and +the reason it surfaces emails that older tools miss. + +### Step 5 — Commits fallback + +If the steps above returned nothing useful (e.g. the user has been inactive for more than ~90 days, or every event email is `@users.noreply.github.com`), we fall back to scanning commits in the user's own repositories: @@ -139,7 +163,7 @@ The `?author=` filter on the commits endpoint accepts either a GitHub login or an email — using the login restricts results to commits GitHub has actually linked to this account. -### Step 5 — Ranking +### Step 6 — Ranking Each unique `(email, name)` pair is annotated with: diff --git a/gitcolombo.html b/gitcolombo.html index 367df54..82a15c5 100644 --- a/gitcolombo.html +++ b/gitcolombo.html @@ -455,7 +455,11 @@

[ A ] Find real email by username

  • GET /users/{u}/events/public?per_page=100
    → iterate PushEvent.payload.commits[].author.{email, name}
    → covers roughly the last 90 days of public activity
    • -
  • If /events returned nothing useful, fall back to: +
  • GET /search/commits?q=author:{u}&per_page=100 (paginated, cap ~1000)
    + → catches commits in any public repo, not just the user's own; + surfaces old emails that fell out of the 90-day events window. + This is what GitFive's "lite" mode does too.
    • +
  • If the steps above returned only noreply addresses, fall back to:
    • GET /users/{u}/repos?sort=updated — list non-fork repos
    • GET /repos/{owner}/{repo}/commits?author={u} — commits attributed to this account
      • @@ -792,6 +796,34 @@

      ! Warning — commit emails can be forged

      setStatus("events lookup failed: " + e.message, { error: true }); } + // /search/commits — catches commits in ANY public repo, not just the + // user's own; surfaces old emails that fell out of the 90-day events window. + // /search/* has a stricter rate limit (10/min unauth, 30/min auth) and + // caps at 1000 results = 10 pages of 100. + setStatus("scanning /search/commits…", { spin: true }); + try { + for (let page = 1; page <= 10; page++) { + if (stopFlag) break; + const data = await ghFetch( + `/search/commits?q=author:${encodeURIComponent(username)}&per_page=100&page=${page}` + ); + const items = (data && data.items) || []; + if (items.length === 0) break; + for (const item of items) { + const c = item.commit || {}; + const repoFull = item.repository && item.repository.full_name; + const a = c.author, co = c.committer; + if (a) record(a.email, a.name, repoFull); + if (co) record(co.email, co.name, repoFull); + recordTrailers(c.message, repoFull); + } + if (items.length < 100) break; + } + } catch (e) { + if (e.message === "STOPPED") throw e; + // soft-fail: /search rate-limit is harsh; whatever was found so far stays + } + const havePublicReal = [...identities.values()].some((i) => !i.noreply); // fallback: scan commits in user's own non-fork repositories