Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 85 additions & 38 deletions scripts/classifier-board.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
RCAGG_DB = "/var/lib/ework-web/share/rc-agg.db"
ABSORBED_LABEL = "已被收"
ABSORBED_COMMENT = (
"(已收)本单已归入根因聚合 issue(token 见该单 body),机器人已停止在本单上的工作,"
"(已收)本单已归入根因聚合 issue(token 见聚合单 body),机器人已停止在本单上的工作,"
"修复走聚合 issue 统一出 PR;本单保留不关,修复验证后统一关闭。"
)

Expand Down Expand Up @@ -82,6 +82,17 @@ def rc_lookup(cat):
number = row[0]
try:
it = gh_rest(f"issues/{number}")
except urllib.error.HTTPError as e:
if e.code == 404:
# issue deleted upstream -> dead record; drop it so the next
# round creates a fresh ticket (same treatment as closed)
c2 = rcdb()
c2.execute("DELETE FROM rc_agg WHERE category=?", (cat,))
c2.commit(); c2.close()
print(f"classifier-board: agg #{number} for {cat!r} deleted upstream (404) -> record dropped")
return None
print(f"classifier-board: agg verify failed for {cat!r} (#{number}): HTTP {e.code} -> trusting DB record (no duplicate create)")
return {"number": number, "title": "", "body": "", "state": "open", "_unverified": True}
except Exception as e:
print(f"classifier-board: agg verify failed for {cat!r} (#{number}): {e} -> trusting DB record (no duplicate create)")
return {"number": number, "title": "", "body": "", "state": "open", "_unverified": True}
Expand Down Expand Up @@ -173,7 +184,8 @@ def gh_gql(query, variables):

def load_state():
try:
return json.load(open(STATE))
with open(STATE, encoding="utf-8") as f:
return json.load(f)
except Exception:
return {}

Expand Down Expand Up @@ -214,13 +226,11 @@ def parse_classifier_json(text):
out.extend(o for o in arr if isinstance(o, dict) and isinstance(o.get("number"), int))
return out

def fetch_batch(merged, anchor, st):
"""Pull anchor-thread comments until every open issue number already has an
entry in `merged` (or the thread is exhausted). Pages of 50, newest first —
so the first page alone already refreshes the latest batch."""
cursor_seen = st.get("anchor_comment_id")
def fetch_batch(merged, anchor):
"""Pull anchor-thread comments (newest first) until every open issue number
already has an entry in `merged` (or the thread is exhausted). Pages of 50
— the first page alone already refreshes the latest batch."""
page = 1
refreshed = False
while page <= 10:
try:
comments = gh_rest(f"issues/{anchor}/comments?per_page=50&page={page}")
Expand All @@ -230,19 +240,36 @@ def fetch_batch(merged, anchor, st):
if not isinstance(comments, list) or not comments:
break
for cm in comments:
if cursor_seen and cm.get("id") == cursor_seen and refreshed:
return
for o in parse_classifier_json(cm.get("body", "")):
n = o["number"]
if n not in merged or (cm.get("created_at") or "") >= (merged[n].get("_at") or ""):
o["_at"] = cm.get("created_at") or ""
merged[n] = o
refreshed = True
if all(x in merged for x in open_numbers()):
break
page += 1
if cursor_seen and refreshed:
save_state(dict(st, anchor_comment_id=cursor_seen))

def known_numbers():
"""Every mirror issue number ever seen (open + closed)."""
c = sqlite3.connect(f"file:{DB}?mode=ro", uri=True)
rows = c.execute("""SELECT number FROM issues
WHERE project_id=(SELECT id FROM projects WHERE name='billion-context')""").fetchall()
c.close()
return {n for (n,) in rows}

def validate_merged(merged):
"""M3: classifier-json numbers are MIRROR issue numbers; an unknown number
means a wrong number space (or a hallucinated id) — silently suppressing /
labelling the wrong issue would be worse than missing one round, so drop
it with a WARN."""
known = known_numbers()
keep = {}
for n, o in merged.items():
if n in known:
keep[n] = o
else:
print(f"classifier-board: WARN classifier-json number {n} is not a known mirror issue -> dropped")
return keep

def open_numbers():
c = sqlite3.connect(f"file:{DB}?mode=ro", uri=True)
Expand All @@ -266,8 +293,11 @@ def incorporated_numbers(merged, open_set):
for n, o in merged.items():
if n not in open_set:
continue
cat = str(o.get("category") or "").strip()
by_cat.setdefault(cat or "?", []).append(n)
# M2: one canonical bucket key for empty categories, identical in all
# three places (incorporated_numbers / update_agg / mark_members) so an
# unclassified member can never fall through the label/comment path.
cat = (o.get("category") or "").strip() or "(未分类)"
by_cat.setdefault(cat, []).append(n)
out = {}
for cat, nums in by_cat.items():
if cat == "tinyfix" or len(nums) >= AGG_THRESHOLD:
Expand Down Expand Up @@ -437,8 +467,10 @@ def update_agg(merged, open_set):
)
if len(nums) < AGG_THRESHOLD:
# below threshold: never create; but keep an existing ticket's body
# fresh so a dispatched worker sees the true (shrunk) member list
if it is not None:
# fresh so a dispatched worker sees the true (shrunk) member list.
# Members of a 1-issue category are deliberately NOT suppressed:
# the aggregation ticket exists for the worker, not yet for a batch.
if it is not None and ((it.get("title") or "") != title or (it.get("body") or "") != body):
try:
gh_rest(f"issues/{it['number']}", {"title": title, "body": body}, method="PATCH")
print(f"classifier-board: refreshed (shrunken) agg #{it['number']} for {cat!r} ({len(nums)} items)")
Expand All @@ -461,13 +493,18 @@ def update_agg(merged, open_set):
continue
rc_remember(cat, it.get("number"))
else:
if (it.get("title") or "") != title:
it = dict(it, title=title)
try:
gh_rest(f"issues/{it['number']}", {"title": it.get("title"), "body": body}, method="PATCH")
print(f"classifier-board: updated agg issue #{it['number']} for {cat!r} ({len(nums)} issues)")
except Exception as e:
print(f"classifier-board: agg update failed #{it.get('number')}: {e}")
# L2: PATCH only when title/body actually changed — a blind per-round
# rewrite would clobber worker notes in the body and waste REST.
# (The marker is re-emitted in the template, so a changed body still
# self-heals a stripped marker.) _unverified dicts carry body="" and
# therefore always PATCH, which is exactly the repair path.
changed = (it.get("title") or "") != title or (it.get("body") or "") != body
if changed:
try:
gh_rest(f"issues/{it['number']}", {"title": title, "body": body}, method="PATCH")
print(f"classifier-board: updated agg issue #{it.get('number')} for {cat!r} ({len(nums)} issues)")
except Exception as e:
print(f"classifier-board: agg update failed #{it.get('number')}: {e}")
rc_remember(cat, it.get("number"))
ensured[cat] = {"number": it.get("number"), "title": title}
return ensured
Expand All @@ -478,7 +515,10 @@ def mark_members(agg_by_cat, merged, open_set):
daemon drop-gate has to be in place before the comment lands, or the
comment would wake a worker on the member issue.
agg_by_cat: {cat: {"number": <gh issue number>, "title": <str>}} from
update_agg (avoids a second rc_lookup/REST round-trip per category)."""
update_agg (avoids a second rc_lookup/REST round-trip per category).
M1: the state mark is only written AFTER the comment POST succeeds, so a
transient 403/rate-limit retries next round instead of silently skipping
the member forever."""
st = load_state()
marked = dict(st.get("absorbed_marks") or {})
newly = 0
Expand All @@ -488,7 +528,8 @@ def mark_members(agg_by_cat, merged, open_set):
for n, obj in merged.items():
if n not in open_set:
continue
if (obj.get("category") or "").strip() != cat:
mcat = (obj.get("category") or "").strip() or "(未分类)"
if mcat != cat:
continue
key = str(n)
if key in marked:
Expand All @@ -513,11 +554,14 @@ def mark_members(agg_by_cat, merged, open_set):
ensure_label(ABSORBED_LABEL, "d9d9d9")
gh_rest(f"issues/{gnum}/labels", {"labels": [ABSORBED_LABEL]}, method="POST")
except Exception as e:
print(f"classifier-board: absorbed-label failed for GH #{gnum}: {e}")
print(f"classifier-board: absorbed-label failed for GH #{gnum} (non-fatal, comment decides): {e}")
try:
gh_rest(f"issues/{gnum}/comments", {"body": f"{ABSORBED_COMMENT}(聚合单:{agg_title_txt},GitHub #{num})"}, method="POST")
except Exception as e:
print(f"classifier-board: absorbed comment failed for GH #{gnum}: {e}")
# not marked -> next round retries (label is guarded by
# issue_has_label, so no duplicate label on retry)
print(f"classifier-board: absorbed comment failed for GH #{gnum} -> not marked, retry next round: {e}")
continue
marked[key] = num
newly += 1
time.sleep(0.4)
Expand Down Expand Up @@ -613,12 +657,12 @@ def main():
# M2 guard: carry last round's classifications forward so a short/partial
# anchor-thread scan (page cap, transient 403) can never silently drop an
# issue out of the suppression gate (drop = bot wakes on absorbed member).
# The cache is seeded INTO st before fetch_batch, because fetch_batch may
# call save_state(st) for the cursor — a stale `st` without the cache would
# clobber it.
st = dict(st, merged_cache=dict(st.get("merged_cache") or {}))
merged = dict(st["merged_cache"])
fetch_batch(merged, anchor, st)
# Known trade-off: an issue re-classified into a batch that later DROPS it
# wholesale keeps its last category until a newer comment overwrites it;
# closed members are filtered by open_set every round.
merged = dict(st.get("merged_cache") or {})
fetch_batch(merged, anchor)
merged = validate_merged(merged)
save_state(dict(st, merged_cache=merged))
if not merged:
print("classifier-board: no classifier-json found on anchor thread yet — keeping previous suppression gate intact")
Expand All @@ -628,14 +672,17 @@ def main():
# members FIRST (drop-gate + stop-work), only then (re)create aggregation
# issues and post member marks — so no member comment can wake a worker.
write_suppression(merged, open_set)
agg_by_cat = {}
if merged:
agg_by_cat = update_agg(merged, open_set)
mark_members(agg_by_cat, merged, open_set)
agg_by_cat = update_agg(merged, open_set)
ensure_label(ABSORBED_LABEL, "d9d9d9")
mark_members(agg_by_cat, merged, open_set)
ensure_label(AGG_LABEL)
num = board_discussion_number(st)
if num is None:
num = create_board()
# L4: reload state before writing — `st` predates mark_members and a
# whole-file write would clobber its absorbed_marks, re-commenting
# every member once next round.
st = load_state()
save_state(dict(st, discussion=num))
update_board(num, render(merged, open_set))
print(f"classifier-board: board #{num} updated ({len(merged)} classified, {len(open_set)} open)")
Expand Down
Loading