diff --git a/scripts/classifier-board.py b/scripts/classifier-board.py index 634ea4a..d9441de 100644 --- a/scripts/classifier-board.py +++ b/scripts/classifier-board.py @@ -45,7 +45,7 @@ RCAGG_DB = "/var/lib/ework-web/share/rc-agg.db" ABSORBED_LABEL = "已被收" ABSORBED_COMMENT = ( - "(已收)本单已归入根因聚合 issue(token 见该单 body),机器人已停止在本单上的工作," + "(已收)本单已归入根因聚合 issue(token 见聚合单 body),机器人已停止在本单上的工作," "修复走聚合 issue 统一出 PR;本单保留不关,修复验证后统一关闭。" ) @@ -82,6 +82,17 @@ def rc_lookup(cat): number = row[0] try: it = gh_rest(f"issues/{number}") + except urllib.error.HTTPError as e: + if e.code == 404: + # issue deleted upstream -> dead record; drop it so the next + # round creates a fresh ticket (same treatment as closed) + c2 = rcdb() + c2.execute("DELETE FROM rc_agg WHERE category=?", (cat,)) + c2.commit(); c2.close() + print(f"classifier-board: agg #{number} for {cat!r} deleted upstream (404) -> record dropped") + return None + print(f"classifier-board: agg verify failed for {cat!r} (#{number}): HTTP {e.code} -> trusting DB record (no duplicate create)") + return {"number": number, "title": "", "body": "", "state": "open", "_unverified": True} except Exception as e: print(f"classifier-board: agg verify failed for {cat!r} (#{number}): {e} -> trusting DB record (no duplicate create)") return {"number": number, "title": "", "body": "", "state": "open", "_unverified": True} @@ -173,7 +184,8 @@ def gh_gql(query, variables): def load_state(): try: - return json.load(open(STATE)) + with open(STATE, encoding="utf-8") as f: + return json.load(f) except Exception: return {} @@ -214,13 +226,11 @@ def parse_classifier_json(text): out.extend(o for o in arr if isinstance(o, dict) and isinstance(o.get("number"), int)) return out -def fetch_batch(merged, anchor, st): - """Pull anchor-thread comments until every open issue number already has an - entry in `merged` (or the thread is exhausted). Pages of 50, newest first — - so the first page alone already refreshes the latest batch.""" - cursor_seen = st.get("anchor_comment_id") +def fetch_batch(merged, anchor): + """Pull anchor-thread comments (newest first) until every open issue number + already has an entry in `merged` (or the thread is exhausted). Pages of 50 + — the first page alone already refreshes the latest batch.""" page = 1 - refreshed = False while page <= 10: try: comments = gh_rest(f"issues/{anchor}/comments?per_page=50&page={page}") @@ -230,19 +240,36 @@ def fetch_batch(merged, anchor, st): if not isinstance(comments, list) or not comments: break for cm in comments: - if cursor_seen and cm.get("id") == cursor_seen and refreshed: - return for o in parse_classifier_json(cm.get("body", "")): n = o["number"] if n not in merged or (cm.get("created_at") or "") >= (merged[n].get("_at") or ""): o["_at"] = cm.get("created_at") or "" merged[n] = o - refreshed = True if all(x in merged for x in open_numbers()): break page += 1 - if cursor_seen and refreshed: - save_state(dict(st, anchor_comment_id=cursor_seen)) + +def known_numbers(): + """Every mirror issue number ever seen (open + closed).""" + c = sqlite3.connect(f"file:{DB}?mode=ro", uri=True) + rows = c.execute("""SELECT number FROM issues + WHERE project_id=(SELECT id FROM projects WHERE name='billion-context')""").fetchall() + c.close() + return {n for (n,) in rows} + +def validate_merged(merged): + """M3: classifier-json numbers are MIRROR issue numbers; an unknown number + means a wrong number space (or a hallucinated id) — silently suppressing / + labelling the wrong issue would be worse than missing one round, so drop + it with a WARN.""" + known = known_numbers() + keep = {} + for n, o in merged.items(): + if n in known: + keep[n] = o + else: + print(f"classifier-board: WARN classifier-json number {n} is not a known mirror issue -> dropped") + return keep def open_numbers(): c = sqlite3.connect(f"file:{DB}?mode=ro", uri=True) @@ -266,8 +293,11 @@ def incorporated_numbers(merged, open_set): for n, o in merged.items(): if n not in open_set: continue - cat = str(o.get("category") or "").strip() - by_cat.setdefault(cat or "?", []).append(n) + # M2: one canonical bucket key for empty categories, identical in all + # three places (incorporated_numbers / update_agg / mark_members) so an + # unclassified member can never fall through the label/comment path. + cat = (o.get("category") or "").strip() or "(未分类)" + by_cat.setdefault(cat, []).append(n) out = {} for cat, nums in by_cat.items(): if cat == "tinyfix" or len(nums) >= AGG_THRESHOLD: @@ -437,8 +467,10 @@ def update_agg(merged, open_set): ) if len(nums) < AGG_THRESHOLD: # below threshold: never create; but keep an existing ticket's body - # fresh so a dispatched worker sees the true (shrunk) member list - if it is not None: + # fresh so a dispatched worker sees the true (shrunk) member list. + # Members of a 1-issue category are deliberately NOT suppressed: + # the aggregation ticket exists for the worker, not yet for a batch. + if it is not None and ((it.get("title") or "") != title or (it.get("body") or "") != body): try: gh_rest(f"issues/{it['number']}", {"title": title, "body": body}, method="PATCH") print(f"classifier-board: refreshed (shrunken) agg #{it['number']} for {cat!r} ({len(nums)} items)") @@ -461,13 +493,18 @@ def update_agg(merged, open_set): continue rc_remember(cat, it.get("number")) else: - if (it.get("title") or "") != title: - it = dict(it, title=title) - try: - gh_rest(f"issues/{it['number']}", {"title": it.get("title"), "body": body}, method="PATCH") - print(f"classifier-board: updated agg issue #{it['number']} for {cat!r} ({len(nums)} issues)") - except Exception as e: - print(f"classifier-board: agg update failed #{it.get('number')}: {e}") + # L2: PATCH only when title/body actually changed — a blind per-round + # rewrite would clobber worker notes in the body and waste REST. + # (The marker is re-emitted in the template, so a changed body still + # self-heals a stripped marker.) _unverified dicts carry body="" and + # therefore always PATCH, which is exactly the repair path. + changed = (it.get("title") or "") != title or (it.get("body") or "") != body + if changed: + try: + gh_rest(f"issues/{it['number']}", {"title": title, "body": body}, method="PATCH") + print(f"classifier-board: updated agg issue #{it.get('number')} for {cat!r} ({len(nums)} issues)") + except Exception as e: + print(f"classifier-board: agg update failed #{it.get('number')}: {e}") rc_remember(cat, it.get("number")) ensured[cat] = {"number": it.get("number"), "title": title} return ensured @@ -478,7 +515,10 @@ def mark_members(agg_by_cat, merged, open_set): daemon drop-gate has to be in place before the comment lands, or the comment would wake a worker on the member issue. agg_by_cat: {cat: {"number": , "title": }} from - update_agg (avoids a second rc_lookup/REST round-trip per category).""" + update_agg (avoids a second rc_lookup/REST round-trip per category). + M1: the state mark is only written AFTER the comment POST succeeds, so a + transient 403/rate-limit retries next round instead of silently skipping + the member forever.""" st = load_state() marked = dict(st.get("absorbed_marks") or {}) newly = 0 @@ -488,7 +528,8 @@ def mark_members(agg_by_cat, merged, open_set): for n, obj in merged.items(): if n not in open_set: continue - if (obj.get("category") or "").strip() != cat: + mcat = (obj.get("category") or "").strip() or "(未分类)" + if mcat != cat: continue key = str(n) if key in marked: @@ -513,11 +554,14 @@ def mark_members(agg_by_cat, merged, open_set): ensure_label(ABSORBED_LABEL, "d9d9d9") gh_rest(f"issues/{gnum}/labels", {"labels": [ABSORBED_LABEL]}, method="POST") except Exception as e: - print(f"classifier-board: absorbed-label failed for GH #{gnum}: {e}") + print(f"classifier-board: absorbed-label failed for GH #{gnum} (non-fatal, comment decides): {e}") try: gh_rest(f"issues/{gnum}/comments", {"body": f"{ABSORBED_COMMENT}(聚合单:{agg_title_txt},GitHub #{num})"}, method="POST") except Exception as e: - print(f"classifier-board: absorbed comment failed for GH #{gnum}: {e}") + # not marked -> next round retries (label is guarded by + # issue_has_label, so no duplicate label on retry) + print(f"classifier-board: absorbed comment failed for GH #{gnum} -> not marked, retry next round: {e}") + continue marked[key] = num newly += 1 time.sleep(0.4) @@ -613,12 +657,12 @@ def main(): # M2 guard: carry last round's classifications forward so a short/partial # anchor-thread scan (page cap, transient 403) can never silently drop an # issue out of the suppression gate (drop = bot wakes on absorbed member). - # The cache is seeded INTO st before fetch_batch, because fetch_batch may - # call save_state(st) for the cursor — a stale `st` without the cache would - # clobber it. - st = dict(st, merged_cache=dict(st.get("merged_cache") or {})) - merged = dict(st["merged_cache"]) - fetch_batch(merged, anchor, st) + # Known trade-off: an issue re-classified into a batch that later DROPS it + # wholesale keeps its last category until a newer comment overwrites it; + # closed members are filtered by open_set every round. + merged = dict(st.get("merged_cache") or {}) + fetch_batch(merged, anchor) + merged = validate_merged(merged) save_state(dict(st, merged_cache=merged)) if not merged: print("classifier-board: no classifier-json found on anchor thread yet — keeping previous suppression gate intact") @@ -628,14 +672,17 @@ def main(): # members FIRST (drop-gate + stop-work), only then (re)create aggregation # issues and post member marks — so no member comment can wake a worker. write_suppression(merged, open_set) - agg_by_cat = {} - if merged: - agg_by_cat = update_agg(merged, open_set) - mark_members(agg_by_cat, merged, open_set) + agg_by_cat = update_agg(merged, open_set) + ensure_label(ABSORBED_LABEL, "d9d9d9") + mark_members(agg_by_cat, merged, open_set) ensure_label(AGG_LABEL) num = board_discussion_number(st) if num is None: num = create_board() + # L4: reload state before writing — `st` predates mark_members and a + # whole-file write would clobber its absorbed_marks, re-commenting + # every member once next round. + st = load_state() save_state(dict(st, discussion=num)) update_board(num, render(merged, open_set)) print(f"classifier-board: board #{num} updated ({len(merged)} classified, {len(open_set)} open)")