From 98a895f6f8b4f3685ead4ad4574e81bca56acb80 Mon Sep 17 00:00:00 2001 From: rushitgit <95172033+rushitgit@users.noreply.github.com> Date: Wed, 29 Jul 2026 05:54:49 +0400 Subject: [PATCH] fix(harness): preserve batch repo identity Key batch targets by GitHub owner and repository so same-named repos do not share a clone, log, or resume state. Collapse equivalent URL forms before scheduling duplicate scans. --- harness/local_harness/batch/README.md | 4 ++ harness/local_harness/batch/run.py | 81 +++++++++++++++++++++++---- harness/tests/test_batch_run.py | 58 +++++++++++++++++++ 3 files changed, 132 insertions(+), 11 deletions(-) diff --git a/harness/local_harness/batch/README.md b/harness/local_harness/batch/README.md index e1550a5..4b5c69a 100644 --- a/harness/local_harness/batch/README.md +++ b/harness/local_harness/batch/README.md @@ -9,6 +9,10 @@ Scan arbitrary GitHub repos using the shared VulnHunter scan engine. 3. `python -m local_harness.batch.run status` (check progress) 4. `python -m local_harness.batch.run collect` (gather results into `to_upload/`) +Clone directories use `__` names so same-named repositories under +different owners remain independent. Equivalent duplicate entries are scanned +once. + ## Options - `--re-clone` — remove and re-clone repos that already exist locally diff --git a/harness/local_harness/batch/run.py b/harness/local_harness/batch/run.py index 931587d..d702b55 100644 --- a/harness/local_harness/batch/run.py +++ b/harness/local_harness/batch/run.py @@ -9,8 +9,10 @@ import argparse import os +import re import sys import time +from urllib.parse import urlsplit sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) @@ -24,12 +26,54 @@ from local_harness.scan import clean_incomplete_results, has_valid_results, scan_targets, ts +_GITHUB_COMPONENT_RE = re.compile(r"[A-Za-z0-9._-]+") + + +def _repo_parts_from_url(url): + """Return the owner and repository from a supported GitHub clone URL. + + Both URL-style remotes and the common git@github.com:owner/repo.git form + are accepted. + """ + value = url.strip() + if "://" not in value: + host, separator, path = value.partition(":") + if not separator or host.rsplit("@", 1)[-1].casefold() != "github.com": + raise ValueError("expected a github.com repository URL") + else: + parsed = urlsplit(value) + if parsed.scheme not in {"http", "https", "ssh", "git"}: + raise ValueError("expected an HTTP(S), SSH, or git GitHub URL") + if (parsed.hostname or "").casefold() not in {"github.com", "www.github.com"}: + raise ValueError("expected a github.com repository URL") + path = parsed.path + + parts = [part for part in path.rstrip("/").split("/") if part] + if len(parts) != 2: + raise ValueError("expected a GitHub repository URL ending in OWNER/REPOSITORY") + + owner, repo = parts + if repo.endswith(".git"): + repo = repo[:-4] + + if any( + component in {"", ".", ".."} or not _GITHUB_COMPONENT_RE.fullmatch(component) + for component in (owner, repo) + ): + raise ValueError("repository path contains unsupported characters") + return owner, repo + + def repo_name_from_url(url): """Extract repo name from a GitHub URL.""" - name = url.rstrip("/").split("/")[-1] - if name.endswith(".git"): - name = name[:-4] - return name + _, repo = _repo_parts_from_url(url) + return repo + + +def repo_target_name_from_url(url): + """Return a stable, filesystem-safe batch target name for a GitHub URL.""" + owner, repo = _repo_parts_from_url(url) + return f"{owner.casefold()}__{repo.casefold()}" def cmd_scan(args): @@ -39,19 +83,34 @@ def cmd_scan(args): print("No URLs found in REPO_LIST.txt") sys.exit(1) - print(f"[{ts()}] Found {len(urls)} repos to process") - for i, url in enumerate(urls): - print(f" [{i + 1}] {url}") + repo_targets = {} + clone_failures = [] + duplicate_count = 0 + for url in urls: + try: + target_name = repo_target_name_from_url(url) + except ValueError as exc: + clone_failures.append((url, f"invalid repository URL: {exc}")) + continue + if target_name in repo_targets: + duplicate_count += 1 + continue + repo_targets[target_name] = url + + print(f"[{ts()}] Found {len(urls)} repo list entries") + for i, (target_name, url) in enumerate(repo_targets.items()): + print(f" [{i + 1}] {url} -> {target_name}") + if duplicate_count: + noun = "entry" if duplicate_count == 1 else "entries" + print(f" Ignoring {duplicate_count} duplicate repo {noun}") print(flush=True) # Phase 1: Clone print(f"[{ts()}] Cloning repos (depth=1) ...") folders = [] - clone_failures = [] - for url in urls: - name = repo_name_from_url(url) - target_dir = os.path.join(BATCH_CLONE_BASE_DIR, name) + for target_name, url in repo_targets.items(): + target_dir = os.path.join(BATCH_CLONE_BASE_DIR, target_name) target_dir, error = shallow_clone(url, target_dir, re_clone=args.re_clone) if error: print(f" [{ts()}] CLONE FAILED: {url}\n {error}", flush=True) diff --git a/harness/tests/test_batch_run.py b/harness/tests/test_batch_run.py index 8a59669..1fe687e 100644 --- a/harness/tests/test_batch_run.py +++ b/harness/tests/test_batch_run.py @@ -12,6 +12,30 @@ def test_repo_name_from_url(): assert brun.repo_name_from_url("https://github.com/a/widget/") == "widget" +@pytest.mark.parametrize( + ("url", "expected"), + [ + ("https://github.com/Acme/Widget", "acme__widget"), + ("ssh://git@github.com/Acme/Widget.git", "acme__widget"), + ("git@github.com:Acme/Widget.git", "acme__widget"), + ], +) +def test_repo_target_name_from_url(url, expected): + assert brun.repo_target_name_from_url(url) == expected + + +@pytest.mark.parametrize( + "url", + [ + "https://example.com/acme/widget", + "https://github.com/acme/widget/subdir", + ], +) +def test_repo_target_name_rejects_unsupported_urls(url): + with pytest.raises(ValueError): + brun.repo_target_name_from_url(url) + + class _Args: def __init__(self, **kw): self.__dict__.update(kw) @@ -49,6 +73,40 @@ def fake_scan_targets(targets, max_workers=None, log_filename=None, readonly=Fal brun.cmd_scan(_Args(re_clone=False, resume=False, max_workers=2, readonly=False)) +def test_cmd_scan_uses_unique_repository_targets(monkeypatch, tmp_path): + monkeypatch.setattr( + brun, + "parse_repo_list", + lambda: [ + "https://github.com/team-one/service", + "https://github.com/team-two/service", + "git@github.com:TEAM-ONE/service.git", + ], + ) + monkeypatch.setattr(brun, "BATCH_CLONE_BASE_DIR", str(tmp_path)) + clone_dirs = [] + monkeypatch.setattr( + brun, + "shallow_clone", + lambda url, target_dir, re_clone=False: ( + clone_dirs.append(target_dir) or target_dir, + None, + ), + ) + scanned = [] + monkeypatch.setattr( + brun, + "scan_targets", + lambda targets, **kwargs: scanned.extend(targets) or [], + ) + + brun.cmd_scan(_Args(re_clone=False, resume=False, max_workers=2, readonly=False)) + + expected = ["team-one__service", "team-two__service"] + assert [brun.os.path.basename(path) for path in clone_dirs] == expected + assert [target["key"] for target in scanned] == expected + + def test_cmd_scan_resume_all_have_results(monkeypatch, tmp_path): monkeypatch.setattr(brun, "parse_repo_list", lambda: ["https://github.com/a/b"]) monkeypatch.setattr(brun, "BATCH_CLONE_BASE_DIR", str(tmp_path))