From d9f8ec098c2c5ae9396e423bed3b9316be79987d Mon Sep 17 00:00:00 2001 From: Execute Plan Date: Tue, 11 Aug 2026 03:12:58 -0400 Subject: [PATCH 1/2] feat: Grok interpret-rank, Ranked Sheets, Teams match alerts, Atlas config Add post-ingest Grok ranking against company objectives, Google Sheets Ranked tab sync, and Teams Adaptive Card pings for scores at or above 40. Tune keywords and objectives for Atlas Flow Group and document plug-and-play keys, function list, and roadmap for operators. --- .env.example | 16 + .gitignore | 2 + AGENTS.md | 44 +- README.md | 41 +- config/keywords.yaml | 168 +++--- config/notify.yaml | 34 ++ config/objectives.yaml | 116 ++++ config/settings.example.env | 16 + docs/AS_BUILT.md | 92 +++- docs/BACKLOG.md | 179 ++++++ docs/CHANGE_PLAYBOOK.md | 15 + docs/DATA_UPDATE_DIRECTIVES.md | 30 +- docs/DECISIONS.md | 19 +- docs/INDEX.md | 7 +- docs/PLUG_AND_PLAY.md | 338 ++++++++++++ docs/STATUS.md | 78 ++- pyproject.toml | 3 + scripts/google_sheets_setup.md | 25 +- scripts/ops_runbook.md | 55 +- src/opportunity_ingest/cli.py | 368 ++++++++++++- src/opportunity_ingest/config.py | 133 ++++- src/opportunity_ingest/interpret_rank.py | 671 +++++++++++++++++++++++ src/opportunity_ingest/notify.py | 342 +++++++++++- src/opportunity_ingest/pipeline.py | 43 +- src/opportunity_ingest/sheets_sync.py | 216 +++++++- tests/test_filter.py | 21 +- tests/test_interpret_rank.py | 252 +++++++++ tests/test_notify.py | 99 ++++ tests/test_sheets_sync.py | 98 ++++ tests/test_smoke.py | 2 + 30 files changed, 3330 insertions(+), 193 deletions(-) create mode 100644 config/notify.yaml create mode 100644 config/objectives.yaml create mode 100644 docs/BACKLOG.md create mode 100644 docs/PLUG_AND_PLAY.md create mode 100644 src/opportunity_ingest/interpret_rank.py create mode 100644 tests/test_interpret_rank.py diff --git a/.env.example b/.env.example index 356b2e4..2a05c1f 100644 --- a/.env.example +++ b/.env.example @@ -28,6 +28,22 @@ LOG_LEVEL=INFO # pip install -e ".[sheets]" # GOOGLE_SHEET_ID= # from docs.google.com/spreadsheets/d//edit # GOOGLE_SHEET_TAB=Ingest +# GOOGLE_SHEET_RANK_TAB=Ranked # Grok interpret-rank full-replace tab (never Ingest) # GOOGLE_SERVICE_ACCOUNT_FILE=secrets/google-service-account.json # # or inline JSON (e.g. CI secret): # GOOGLE_SERVICE_ACCOUNT_JSON= + +# Optional Grok interpret-rank (xAI API; does not mutate ingest rows): +# pip install -e ".[ai]" +# XAI_API_KEY= +# XAI_BASE_URL=https://api.x.ai/v1 +# XAI_MODEL=grok-4.5 +# OBJECTIVES_PATH=config/objectives.yaml + +# Teams Workflows webhooks (ops alerts + high-match capture pings): +# TEAMS_WEBHOOK_URL= # hard fail / streak / partial (ops) +# TEAMS_MATCH_WEBHOOK_URL= # optional dedicated capture channel; else ops URL +# TEAMS_MATCH_NOTIFY_ENABLED=true +# TEAMS_MATCH_SCORE_THRESHOLD=40 # 0–100; RelevanceScore (ingest) or Grok fit +# TEAMS_MATCH_MAX_ITEMS=8 +# NOTIFY_CONFIG_PATH=config/notify.yaml diff --git a/.gitignore b/.gitignore index 4658a48..815932b 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,8 @@ build/ # Local data / state / logs (keep directory placeholders) data/*.db data/*.csv +data/rankings/ +data/atlas_crawl/ !data/.gitkeep logs/* !logs/.gitkeep diff --git a/AGENTS.md b/AGENTS.md index 03528a8..12242f1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ |-------|--------| | **Project** | Contract Bidding System — Phase 1 opportunity ingest | | **Package** | `opportunity_ingest` (`src/opportunity_ingest/`) | -| **As-of** | 2026-08-10 (as-built; Google Sheets sync included) | +| **As-of** | 2026-08-11 (as-built; Sheets sync + optional Grok interpret-rank) | | **Primary language** | Python 3.11+ | | **Default store** | SQLite (`STORAGE_BACKEND=sqlite`) | @@ -24,8 +24,9 @@ Daily (or on-demand) pipeline: 6. Dedupe by `OpportunityID` + normalized `Link`. 7. **Create-only** writes to SQLite (default) or SharePoint (optional). 8. Optional: export CSV, **full-replace sync to Google Sheets `Ingest` tab**, Teams alerts. +9. Optional post-ingest: **Grok interpret-rank** — plain-English rewrite + fit ranking vs `config/objectives.yaml` (report only). -**SQLite is the system of record for day-1.** Google Sheets is a **view** (overwritten on sync). SharePoint is a **pluggable store**, not required for go-live. +**SQLite is the system of record for day-1.** Google Sheets is a **view** (overwritten on sync). SharePoint is a **pluggable store**, not required for go-live. Grok ranking writes reports under `data/rankings/`; it does **not** mutate store Status/Notes/RelevanceScore. --- @@ -35,14 +36,17 @@ Daily (or on-demand) pipeline: |----------|------|---------| | 1 | **This file (`AGENTS.md`)** | Non-negotiable rules + orientation | | 2 | [`docs/STATUS.md`](docs/STATUS.md) | Living readiness snapshot (what works / next) | -| 3 | [`docs/DATA_UPDATE_DIRECTIVES.md`](docs/DATA_UPDATE_DIRECTIVES.md) | **MUST / MUST NOT** for data writes, schema, sync | -| 4 | [`docs/AS_BUILT.md`](docs/AS_BUILT.md) | Current architecture, modules, CLI, env | -| 5 | [`docs/CHANGE_PLAYBOOK.md`](docs/CHANGE_PLAYBOOK.md) | How to implement common change types | -| 6 | [`docs/INDEX.md`](docs/INDEX.md) | Full documentation index | -| 7 | [`docs/phase1-canadabuys-sharepoint-implementation-schema.md`](docs/phase1-canadabuys-sharepoint-implementation-schema.md) | Original design (rev 4); prefer AS_BUILT if conflict | -| 8 | [`scripts/ops_runbook.md`](scripts/ops_runbook.md) | Human operator procedures | -| 9 | [`scripts/google_sheets_setup.md`](scripts/google_sheets_setup.md) | Sheets service-account setup | -| 10 | [`scripts/provision_sharepoint_list.md`](scripts/provision_sharepoint_list.md) | SharePoint activation (deferred) | +| 3 | [`docs/PLUG_AND_PLAY.md`](docs/PLUG_AND_PLAY.md) | Human keys + plug-and-play steps for built features | +| 4 | [`docs/DATA_UPDATE_DIRECTIVES.md`](docs/DATA_UPDATE_DIRECTIVES.md) | **MUST / MUST NOT** for data writes, schema, sync | +| 5 | [`docs/AS_BUILT.md`](docs/AS_BUILT.md) | Current architecture, modules, CLI, env | +| 6 | [`docs/BACKLOG.md`](docs/BACKLOG.md) | Roadmap / discussed but **not built** | +| 7 | [`docs/CHANGE_PLAYBOOK.md`](docs/CHANGE_PLAYBOOK.md) | How to implement common change types | +| 8 | [`docs/INDEX.md`](docs/INDEX.md) | Full documentation index | +| 9 | [`docs/DECISIONS.md`](docs/DECISIONS.md) | ADR-style as-built decisions | +| 10 | [`docs/phase1-canadabuys-sharepoint-implementation-schema.md`](docs/phase1-canadabuys-sharepoint-implementation-schema.md) | Original design (rev 4); prefer AS_BUILT if conflict | +| 11 | [`scripts/ops_runbook.md`](scripts/ops_runbook.md) | Human operator procedures | +| 12 | [`scripts/google_sheets_setup.md`](scripts/google_sheets_setup.md) | Sheets service-account setup | +| 13 | [`scripts/provision_sharepoint_list.md`](scripts/provision_sharepoint_list.md) | SharePoint activation (deferred) | --- @@ -56,8 +60,9 @@ Daily (or on-demand) pipeline: 6. **CanadaBuys download requires browser-like User-Agent** (CDN returns 403 otherwise). See `download.py` `DEFAULT_HEADERS`. 7. **Google Sheets `Ingest` tab is full-replace.** Manual edits on that tab are wiped next `sync-sheets`. Human work → separate tab (e.g. `Review`). 8. **Keywords:** engineering owns `config/keywords.yaml`. Ops request changes via issues. -9. **Secrets:** never commit `.env`, `secrets/`, service-account JSON, Azure secrets. -10. **Phase 1 non-goals:** multi-source (MERX/etc.), AI ranking, auto-bidding, historical backfill, dual-write SQLite+SharePoint, two-way Sheets Status sync. +9. **Secrets:** never commit `.env`, `secrets/`, service-account JSON, Azure secrets, `XAI_API_KEY`. +10. **Phase 1 non-goals:** multi-source (MERX/etc.), auto-bidding, historical backfill, dual-write SQLite+SharePoint, two-way Sheets Status sync. +11. **Grok interpret-rank is post-ingest only.** Never write AI fit scores into `contract_opportunities` Status/Notes/RelevanceScore. Objectives live in `config/objectives.yaml` (eng-owned). --- @@ -75,15 +80,18 @@ src/opportunity_ingest/ models.py # TenderRecord, OpportunityFields, ExistingKeys config.py # pydantic-settings state.py # zero-new streak JSON - notify.py # Teams Workflows Adaptive Card + notify.py # Teams ops alerts + high-match capture cards (CTA links) exit_codes.py # exit + notify matrix sheets_sync.py # SQLite → Google Sheets full tab replace + interpret_rank.py # Grok rephrase + fit rank (report only) storage/ base.py # OpportunityStore Protocol, normalize_link factory.py # sqlite | sharepoint sqlite_store.py # default backend + export_csv sharepoint_store.py # Graph adapter (optional) config/keywords.yaml +config/objectives.yaml # company objectives for interpret-rank +config/notify.yaml # Teams match threshold / card limits .github/workflows/ci.yml .github/workflows/daily-canadabuys-ingest.yml ``` @@ -98,9 +106,11 @@ python -m opportunity_ingest download-sample [--out PATH] python -m opportunity_ingest check-store python -m opportunity_ingest export-csv [--out PATH] python -m opportunity_ingest sync-sheets [--sheet-id ID] [--tab NAME] +python -m opportunity_ingest interpret-rank [--status STATUS] [--limit N] [--sync-sheets|--no-sync-sheets] [--rank-tab NAME] +python -m opportunity_ingest sync-rank-sheets [--from-json PATH] [--rank-tab NAME] ``` -Optional install extras: `pip install -e ".[sheets]"` for Google Sheets; SharePoint uses core deps (`msal`, `httpx`). +Optional install extras: `pip install -e ".[sheets]"` for Google Sheets; `pip install -e ".[ai]"` for Grok interpret-rank; SharePoint uses core deps (`msal`, `httpx`). Grok rankings sync to tab **`Ranked`** (never `Ingest`). --- @@ -115,7 +125,9 @@ Optional install extras: `pip install -e ".[sheets]"` for Google Sheets; SharePo | Store schema | `sqlite_store.py` DDL + `models.OpportunityFields` + design schema | | Pipeline order / metrics | `pipeline.py` | | Exit / notify policy | `exit_codes.py`, `notify.py` | -| Sheets sync | `sheets_sync.py`, `cli.py` `sync-sheets` | +| Sheets sync (opportunities) | `sheets_sync.py`, `cli.py` `sync-sheets` → tab `Ingest` | +| Sheets sync (Grok ranks) | `sheets_sync.py` `sync_rankings_to_sheet`, `interpret-rank --sync-sheets`, `sync-rank-sheets` → tab `Ranked` | +| Grok interpret / rank | `interpret_rank.py`, `config/objectives.yaml`, `cli.py` `interpret-rank` | | SharePoint | `storage/sharepoint_store.py`, `scripts/provision_sharepoint_list.md` | | Daily schedule | `.github/workflows/daily-canadabuys-ingest.yml` | | Env vars | `config.py`, `.env.example`, `config/settings.example.env` | @@ -128,6 +140,8 @@ Optional install extras: `pip install -e ".[sheets]"` for Google Sheets; SharePo |------|------| | `data/contract_opportunities.db` | SQLite system of record | | `data/sample-openTenderNotice.csv` | Downloaded open-tender sample | +| `data/rankings/interpret-*.md` | Grok ranked brief (human) | +| `data/rankings/interpret-*.json` | Grok ranked brief (machine) | | `data/export-opportunities.csv` | Human CSV export | | `state/zero_new_streak.json` | Zero-new calendar streak | | `logs/run-*.json` | Per-run metrics | diff --git a/README.md b/README.md index fde5cea..e482473 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ Phase 1 pipeline: download CanadaBuys **Open Tender Notices** CSV, filter by key | **Optional store** | SharePoint via Microsoft Graph (`STORAGE_BACKEND=sharepoint`) | | **Optional view** | Google Sheets tab full-replace (`sync-sheets`) | | **LLM / agent entry** | **[`AGENTS.md`](AGENTS.md)** ← start here for AI-assisted development | +| **Human plug-in** | **[`docs/PLUG_AND_PLAY.md`](docs/PLUG_AND_PLAY.md)** ← keys, steps, roadmap to go live | --- @@ -15,10 +16,12 @@ Phase 1 pipeline: download CanadaBuys **Open Tender Notices** CSV, filter by key | Doc | Audience | Purpose | |-----|----------|---------| +| [`docs/PLUG_AND_PLAY.md`](docs/PLUG_AND_PLAY.md) | **Operators / humans** | **Keys to plug, function list, go-live sequence, roadmap** | | [`AGENTS.md`](AGENTS.md) | LLMs + devs | Hard rules, module map, task routing | | [`docs/STATUS.md`](docs/STATUS.md) | Everyone | Living readiness / what works now | | [`docs/INDEX.md`](docs/INDEX.md) | Everyone | Full doc index | | [`docs/AS_BUILT.md`](docs/AS_BUILT.md) | Devs / LLMs | Implemented architecture | +| [`docs/BACKLOG.md`](docs/BACKLOG.md) | Everyone | Roadmap / not built yet | | [`docs/DATA_UPDATE_DIRECTIVES.md`](docs/DATA_UPDATE_DIRECTIVES.md) | Devs / LLMs | **MUST / MUST NOT** for data writes & sync | | [`docs/CHANGE_PLAYBOOK.md`](docs/CHANGE_PLAYBOOK.md) | Devs / LLMs | How to make common changes | | [`docs/DECISIONS.md`](docs/DECISIONS.md) | Devs / LLMs | Key ADRs | @@ -42,6 +45,9 @@ If design doc and as-built conflict → **prefer code + AS_BUILT + DATA_UPDATE_D ## Quick start (Windows) +**Full human checklist (keys, Sheets, Grok, Teams, roadmap):** +→ **[`docs/PLUG_AND_PLAY.md`](docs/PLUG_AND_PLAY.md)** + ```powershell cd "C:\Users\Vikrant\Documents\Agentic AI Learning\contractbiddingsystem" python -m venv .venv @@ -71,6 +77,22 @@ pip install -e ".[sheets]" python -m opportunity_ingest sync-sheets ``` +### Grok interpret-rank (optional AI brief) + +Plain-English rewrite + rank against `config/objectives.yaml` (does **not** change SQLite rows). +When `GOOGLE_SHEET_ID` is set (your live sheet), rankings are **full-replaced** into a **`Ranked`** tab (separate from `Ingest`): + +```powershell +pip install -e ".[ai]" +pip install -e ".[sheets]" # same service account as sync-sheets +# Set XAI_API_KEY + existing GOOGLE_* in root .env +python -m opportunity_ingest interpret-rank --status New --limit 20 +# Local: data/rankings/interpret-*.md +# Sheet: tab Ranked (created if missing) +# Skip sheet push: --no-sync-sheets +# Re-push last report without Grok: python -m opportunity_ingest sync-rank-sheets +``` + **Notes learned in setup:** - `pip install -e .` needs the trailing `.` @@ -88,14 +110,18 @@ python -m opportunity_ingest download-sample [--out PATH] python -m opportunity_ingest check-store python -m opportunity_ingest export-csv [--out PATH] python -m opportunity_ingest sync-sheets [--sheet-id ID] [--tab NAME] +python -m opportunity_ingest interpret-rank [--status STATUS] [--limit N] [--sync-sheets|--no-sync-sheets] +python -m opportunity_ingest sync-rank-sheets [--from-json PATH] [--rank-tab NAME] ``` | Command | Persist store? | Notes | |---------|----------------|--------| | `run` | Only with `--write` | Default dry-run | | `export-csv` | No | Snapshot for Excel | -| `sync-sheets` | No SQLite write | **Full-replace** Sheet tab | +| `sync-sheets` | No SQLite write | **Full-replace** Sheet tab `Ingest` | | `check-store` | No | Health + key count | +| `interpret-rank` | No | Grok rewrite + rank → `data/rankings/` + optional Sheets **Ranked** | +| `sync-rank-sheets` | No | Re-push ranking JSON → Sheets **Ranked** (no Grok call) | --- @@ -107,15 +133,22 @@ python -m opportunity_ingest sync-sheets [--sheet-id ID] [--tab NAME] | `DATA_DIR` | `data` | DB + exports | | `MAX_CREATE` | `50` | Create-**attempt** budget; `0`=unlimited | | `GOOGLE_SHEET_ID` | — | Sheets sync | -| `GOOGLE_SHEET_TAB` | `Ingest` | Tab overwritten on sync | +| `GOOGLE_SHEET_TAB` | `Ingest` | Opportunity tab overwritten on sync | +| `GOOGLE_SHEET_RANK_TAB` | `Ranked` | Grok ranking tab overwritten on interpret-rank | | `GOOGLE_SERVICE_ACCOUNT_FILE` | — | Path to JSON key | -| `TEAMS_WEBHOOK_URL` | — | Failure / streak alerts | +| `TEAMS_WEBHOOK_URL` | — | Ops alerts + match fallback | +| `TEAMS_MATCH_WEBHOOK_URL` | — | Optional capture-channel webhook | +| `TEAMS_MATCH_SCORE_THRESHOLD` | `40` | Ping when score ≥ N | +| `XAI_API_KEY` | — | Grok interpret-rank | +| `OBJECTIVES_PATH` | `config/objectives.yaml` | AI ranking frame of reference | -Full list: `.env.example`, `config/settings.example.env`, `docs/AS_BUILT.md`. +Full list: `.env.example`, `config/settings.example.env`, [`docs/PLUG_AND_PLAY.md`](docs/PLUG_AND_PLAY.md), `docs/AS_BUILT.md`. **Data rules (critical):** create-only; never silent-truncate Link; Sheets is a view. → [`docs/DATA_UPDATE_DIRECTIVES.md`](docs/DATA_UPDATE_DIRECTIVES.md) +**What exists vs not built:** [`docs/STATUS.md`](docs/STATUS.md) · [`docs/AS_BUILT.md`](docs/AS_BUILT.md) · [`docs/BACKLOG.md`](docs/BACKLOG.md) + --- ## Development diff --git a/config/keywords.yaml b/config/keywords.yaml index 5104654..2524786 100644 --- a/config/keywords.yaml +++ b/config/keywords.yaml @@ -1,106 +1,138 @@ # Engineering-owned keyword config for CanadaBuys open-tender filtering. -# Ops requests changes via GitHub issues; eng reviews/merges. -# Do not add bare "teams" or "strategy" — use multi-word / scoped terms only. +# Tuned for Atlas Flow Group capabilities (site crawl 2026-08-11): +# https://atlasflowgroupwebsite.vikrant-singh1.workers.dev/ +# Ops request changes via GitHub issues; eng reviews/merges. +# +# Atlas Flow Group is NOT a classic MSP / ServiceNow SI / construction vendor. +# Prefer: operating-model advisory, human-governed AI, Microsoft-native enablement +# (M365, Copilot, Power Platform), process/workflow under authority design. +# Avoid bare high-noise terms (teams, strategy, service desk alone). # # Matching notes: # - Short terms (len <= short_term_max_len) use word-boundary regex. -# - Unique-by-term weights; nested overlaps are span-aware: a shorter term is -# dropped only when every hit is covered by a longer term's span (e.g. -# "managed service" inside "managed services"). Independent shorter hits -# still count (e.g. "workflow automation and general automation"). -# - Terms dual-listed across groups (e.g. "power automate") contribute one -# weight but attribute all declaring groups for diversity scoring. +# - Unique-by-term weights; nested overlaps are span-aware. +# - Terms dual-listed across groups contribute one weight but all groups for scoring. version: 1 match: case_sensitive: false - fields: ["title", "description"] # honored by loader (title | description) + fields: ["title", "description"] search_category: true short_term_max_len: 4 groups: + # Critical — site: "Microsoft-native", M365, Copilot, Power Platform microsoft_cloud: - label: "Microsoft / M365 / Power Platform / Azure / Copilot" + label: "Microsoft-native (M365 / Copilot / Power Platform / Azure)" keywords: - - { term: "microsoft 365", weight: 20 } - - { term: "microsoft365", weight: 20 } - - { term: "m365", weight: 18 } + - { term: "microsoft 365", weight: 22 } + - { term: "microsoft365", weight: 22 } + - { term: "m365", weight: 20 } - { term: "office 365", weight: 18 } - { term: "o365", weight: 15 } + - { term: "microsoft-native", weight: 20 } + - { term: "microsoft native", weight: 20 } - { term: "sharepoint", weight: 16 } - - { term: "power platform", weight: 20 } - - { term: "power apps", weight: 18 } - - { term: "power automate", weight: 18 } - - { term: "power bi", weight: 14 } - - { term: "dynamics 365", weight: 16 } - - { term: "copilot", weight: 20 } - - { term: "azure", weight: 14 } - - { term: "entra id", weight: 12 } + - { term: "power platform", weight: 22 } + - { term: "power apps", weight: 20 } + - { term: "power automate", weight: 20 } + - { term: "power bi", weight: 12 } + - { term: "copilot", weight: 22 } + - { term: "microsoft copilot", weight: 22 } + - { term: "azure", weight: 12 } + - { term: "entra id", weight: 14 } - { term: "active directory", weight: 10 } - { term: "microsoft teams", weight: 12 } + - { term: "dynamics 365", weight: 10 } - managed_services: - label: "Managed services" - keywords: - - { term: "managed service", weight: 16 } - - { term: "managed services", weight: 16 } - - { term: "msp", weight: 10 } - - { term: "managed it", weight: 14 } - - { term: "outsourcing", weight: 10 } - - { term: "service desk", weight: 14 } - - { term: "end user computing", weight: 12 } - - { term: "desktop support", weight: 12 } - - { term: "device management", weight: 12 } - - { term: "intune", weight: 14 } - - itsm_servicenow: - label: "ITSM / ServiceNow" + # Critical — site: Authority, operating model, source of truth, runbooks, registries + operating_authority: + label: "Operating model / authority / governance design" keywords: - - { term: "servicenow", weight: 20 } - - { term: "service now", weight: 18 } - - { term: "itsm", weight: 16 } - - { term: "itil", weight: 12 } - - { term: "incident management", weight: 12 } - - { term: "service management", weight: 14 } - - { term: "cmdb", weight: 12 } + - { term: "operating model", weight: 22 } + - { term: "operating system", weight: 12 } + - { term: "source of truth", weight: 18 } + - { term: "source of authority", weight: 20 } + - { term: "decision rights", weight: 16 } + - { term: "operating contract", weight: 18 } + - { term: "operating contracts", weight: 18 } + - { term: "runbook", weight: 14 } + - { term: "runbooks", weight: 14 } + - { term: "governance model", weight: 16 } + - { term: "governance framework", weight: 16 } + - { term: "corporate governance", weight: 10 } + - { term: "operational governance", weight: 18 } + - { term: "process governance", weight: 16 } + - { term: "business process", weight: 12 } + - { term: "process design", weight: 14 } + - { term: "process discipline", weight: 14 } + - { term: "traceability", weight: 12 } + - { term: "accountability", weight: 10 } + # High — lead offering on /services advisory_consulting: - label: "Advisory / Consulting (ops)" + label: "Advisory / operating-model consulting" keywords: + - { term: "advisory services", weight: 18 } + - { term: "advisory service", weight: 16 } - { term: "management consulting", weight: 12 } - - { term: "advisory services", weight: 12 } - { term: "professional services", weight: 10 } - - { term: "digital transformation", weight: 14 } - - { term: "operating model", weight: 12 } - - { term: "change management", weight: 10 } + - { term: "digital transformation", weight: 12 } + - { term: "change management", weight: 12 } - { term: "business analysis", weight: 10 } + - { term: "operations assessment", weight: 16 } + - { term: "organisational design", weight: 12 } + - { term: "organizational design", weight: 12 } + - { term: "business transformation", weight: 12 } + # High — site: automation & agents only under human authority automation_process: - label: "Automation / Process improvement" + label: "Workflow automation / agents under governance" keywords: - - { term: "automation", weight: 14 } - - { term: "rpa", weight: 14 } - - { term: "robotic process", weight: 16 } + - { term: "workflow automation", weight: 18 } + - { term: "workflow design", weight: 16 } + - { term: "process automation", weight: 16 } - { term: "process improvement", weight: 14 } - - { term: "workflow automation", weight: 12 } + - { term: "business process automation", weight: 16 } + - { term: "robotic process", weight: 12 } + - { term: "rpa", weight: 12 } + - { term: "low code", weight: 14 } + - { term: "no code", weight: 12 } + - { term: "power automate", weight: 20 } - { term: "orchestration", weight: 10 } - - { term: "low code", weight: 12 } - - { term: "no code", weight: 10 } - - { term: "power automate", weight: 18 } + - { term: "intelligent automation", weight: 14 } + # Bare "automation" kept moderate — many false friends in construction notices + - { term: "automation", weight: 10 } + # High — site: human-governed AI, AI assistance without authority (not unowned GenAI product) ai_operations: - label: "AI + operations / AI governance" + label: "Human-governed AI / AI assistance / responsible AI" + keywords: + - { term: "artificial intelligence", weight: 14 } + - { term: "generative ai", weight: 16 } + - { term: "genai", weight: 14 } + - { term: "ai governance", weight: 22 } + - { term: "responsible ai", weight: 20 } + - { term: "ai assistance", weight: 18 } + - { term: "human in the loop", weight: 16 } + - { term: "human-in-the-loop", weight: 16 } + - { term: "ai adoption", weight: 14 } + - { term: "ai enablement", weight: 16 } + - { term: "llm", weight: 12 } + - { term: "machine learning", weight: 10 } + - { term: "data governance", weight: 14 } + - { term: "ai operations", weight: 12 } + + # Low / selective — only when notice is really about knowledge ops / continuous improvement loops + knowledge_flow: + label: "Knowledge / continuous operating flow" keywords: - - { term: "artificial intelligence", weight: 16 } - - { term: "machine learning", weight: 14 } - - { term: "generative ai", weight: 18 } - - { term: "genai", weight: 16 } - - { term: "ai governance", weight: 18 } - - { term: "responsible ai", weight: 16 } - - { term: "llm", weight: 14 } - - { term: "ai operations", weight: 16 } - - { term: "mlops", weight: 14 } - - { term: "data governance", weight: 12 } + - { term: "knowledge management", weight: 12 } + - { term: "knowledge base", weight: 10 } + - { term: "continuous improvement", weight: 12 } + - { term: "operational excellence", weight: 12 } + - { term: "shared services", weight: 10 } +# Prefer services-style procurement categories (not construction). category_boosts: "*SRV": 5 "SRV": 5 diff --git a/config/notify.yaml b/config/notify.yaml new file mode 100644 index 0000000..8755039 --- /dev/null +++ b/config/notify.yaml @@ -0,0 +1,34 @@ +# Teams / notification pipeline config (engineering-owned). +# Loaded by opportunity_ingest when present; env vars override (see Settings). +# +# Ops alerts (hard fail / partial errors / zero-new streak) still use +# TEAMS_WEBHOOK_URL via notify_ingest_alert. +# +# Match alerts: when NEW opportunities clear the score bar, post a channel +# Adaptive Card with summary + Open notice CTA links. +version: 1 + +teams: + # Master switch for match pings (ops alerts still follow TEAMS_WEBHOOK_URL). + match_notify_enabled: true + + # Score threshold 0–100 (40 = "above 40% match" on our 0–100 scales). + # - ingest write path: rule RelevanceScore + # - interpret-rank path: Grok fit_score + match_score_threshold: 40 + + # Prefer TEAMS_MATCH_WEBHOOK_URL env for a dedicated capture channel; + # if unset, falls back to TEAMS_WEBHOOK_URL (same Workflows connector). + # use_ops_webhook_fallback: true # always true in code when match URL empty + + # Cap how many opportunities appear in one Teams card (rest summarized). + max_items_per_message: 8 + + # Truncate free-text fields for Adaptive Card readability. + title_max_chars: 120 + summary_max_chars: 280 + + # Card branding + card_title_ingest: "New opportunity matches (≥ threshold)" + card_title_rank: "Grok fit matches (≥ threshold)" + cta_label: "Open notice" diff --git a/config/objectives.yaml b/config/objectives.yaml new file mode 100644 index 0000000..bf42b67 --- /dev/null +++ b/config/objectives.yaml @@ -0,0 +1,116 @@ +# Engineering-owned company objectives for Grok-assisted interpret-rank. +# Source of truth for WHO we are: Atlas Flow Group public site (crawled 2026-08-11). +# https://atlasflowgroupwebsite.vikrant-singh1.workers.dev/ +# Pages: /, /services, /approach, /about, /contact +# Align keyword groups in config/keywords.yaml with these objectives. +# +# Used by: python -m opportunity_ingest interpret-rank +# This file does NOT change ingest scoring; it frames AI ranking only. +version: 2 + +company: + name: "Atlas Flow Group" + website: "https://atlasflowgroupwebsite.vikrant-singh1.workers.dev/" + email: "hello@atlasflowgroup.com" + summary: > + SMB-focused advisory and Microsoft-native enablement firm. We design + operating systems where people hold authority, AI assists without taking + control, and every material decision is traceable by design. Lead with + advisory (authority, operating model, governance readiness), then + implement Copilot / Power Platform / M365 workflows under that model — + not tool sprawl, not unowned AI, not large multi-year SI theatre. + geography: > + Canada-oriented public and private buyers are in scope when the work is + operating-model, process, Microsoft-native AI enablement, or human-governed + automation. Prefer mid-market / SMB-right-sized scopes; large SI-only + footprints are a poor fit. + posture: > + One Authority · Human Governed · Traceable by Design. Authority before + automation. AI assistance without authority. Implementation over slideware. + Not a pure product vendor, not a classic MSP/service desk outsourcer, + not ServiceNow platform SI, not construction/facilities/commodities. + +# Priority: critical | high | medium | low (soft ranking hints for Grok). +objectives: + - id: operating_authority + label: "Operating model / authority / governance design" + priority: critical + description: > + Help leaders define one source of authority for operations: decision + rights, canonical source of truth, operating contracts, registries, + runbooks, and governance readiness. Discovery and authority assessment + engagements. AtlasFlow OS thinking delivered through advisory. + + - id: microsoft_cloud + label: "Microsoft-native enablement (M365 / Copilot / Power Platform)" + priority: critical + description: > + Practical Microsoft-native AI and workflow enablement on the stack SMBs + already own: Microsoft 365, Copilot, Power Platform (Power Apps / Power + Automate), SharePoint/Teams patterns, with security and governance that + fit real organisations — wired into operating contracts, not pilot demos. + + - id: advisory_consulting + label: "Advisory retainer & operations counsel" + priority: high + description: > + Ongoing specialized advisory for operations/IT leaders and owners + (roughly 50–500 employees) who need a partner without hiring a large SI. + Operations and authority assessment, prioritisation of high-leverage + improvements, decision support without generic strategy decks. + + - id: automation_process + label: "Governed workflow automation & agents" + priority: high + description: > + Workflow design and automation/agents only where authority is already + defined. Continuous flow Capture → Review → Curate → Use → Govern → Scale. + Execution under human authority — never above it. Power Automate and + adjacent automation under an operating contract. + + - id: ai_operations + label: "Human-governed AI / AI governance / responsible enablement" + priority: high + description: > + AI adoption and enablement with human governance: AI assists; people + remain accountable. Strong fit for AI governance, responsible AI, + Copilot adoption with decision rights, and anti-patterns of unowned AI. + Weak fit for pure GenAI product platforms, AI that replaces ownership, + or hype-driven transformation without governance. + + - id: knowledge_flow + label: "Traceable operating flow & continuous improvement" + priority: medium + description: > + Design work so it is reviewable, auditable, and improvable; shared + intelligence, knowledge that is curated and reused, operational + excellence loops — always attached to authority/governance design. + +# Soft filters for the interpreter (not hard ingest rules). +guidance: + prefer: + - "Operating model, authority, governance, runbooks, decision rights" + - "Microsoft 365 / Copilot / Power Platform enablement with process ownership" + - "Human-governed AI, AI governance, responsible AI adoption" + - "Workflow automation under clear operating contracts" + - "Advisory / professional services scopes (not pure product resale)" + - "SMB-right-sized or modular public-sector work (not multi-year mega-SI only)" + - "Services categories (*SRV) with real ops/IT/process scope" + deprioritize: + - "Construction, facilities, elevator, snow/ice, food/beverage design, pure advertising creative" + - "Classic MSP / service desk / desktop outsourcing as the core SOW (not our offer)" + - "ServiceNow / ITSM platform implementation as primary scope (not on our site)" + - "Specialized product platforms (e.g. third-party risk intelligence SaaS) unless we partner" + - "Military payroll / niche HRIS sustainment without ops-model angle" + - "Keyword false positives with no operating-model or Microsoft-native scope" + - "Buyers who want AI to replace human ownership" + - "Hype AI transformation without governance" + output_tone: > + Plain language for an Atlas Flow Group capture lead. Lead with whether the + buyer needs authority/governance design and/or Microsoft-native enablement + under human control. Be candid: high keyword scores do not equal fit when + the SOW is product resale, construction, or classic outsourcing. + ranking_order_hint: > + Score highest when operating-model/advisory + Microsoft-native or + human-governed AI clearly align. Medium when only one pillar fits. + Low/pass for out-of-scope domains even if a single keyword matched. diff --git a/config/settings.example.env b/config/settings.example.env index ec8c998..1100a2e 100644 --- a/config/settings.example.env +++ b/config/settings.example.env @@ -51,6 +51,22 @@ LOG_LEVEL=INFO # .env must be at REPO ROOT (not scripts/) # GOOGLE_SHEET_ID= # GOOGLE_SHEET_TAB=Ingest +# GOOGLE_SHEET_RANK_TAB=Ranked # GOOGLE_SERVICE_ACCOUNT_FILE=secrets/google-service-account.json # # CI alternative: # GOOGLE_SERVICE_ACCOUNT_JSON= + +# --- Grok interpret-rank (optional; does not mutate SQLite rows) --- +# pip install -e ".[ai]" +# XAI_API_KEY= +# XAI_BASE_URL=https://api.x.ai/v1 +# XAI_MODEL=grok-4.5 +# OBJECTIVES_PATH=config/objectives.yaml + +# --- Teams notify --- +# TEAMS_WEBHOOK_URL= +# TEAMS_MATCH_WEBHOOK_URL= +# TEAMS_MATCH_NOTIFY_ENABLED=true +# TEAMS_MATCH_SCORE_THRESHOLD=40 +# TEAMS_MATCH_MAX_ITEMS=8 +# NOTIFY_CONFIG_PATH=config/notify.yaml diff --git a/docs/AS_BUILT.md b/docs/AS_BUILT.md index 1fcb9df..34de10e 100644 --- a/docs/AS_BUILT.md +++ b/docs/AS_BUILT.md @@ -1,10 +1,11 @@ -# As-built architecture (Phase 1) +# As-built architecture (Phase 1 + post-ingest Grok) -**Source of truth for the implemented system.** Supersedes design-doc details when they differ. +**Source of truth for the implemented system.** Supersedes design-doc details when they differ. +**Unbuilt / discussed ideas:** [`BACKLOG.md`](BACKLOG.md) — do not invent features from backlog into this file. | Field | Value | |-------|--------| -| Status | Implemented and manually exercised (Windows, 2026-08-10) | +| Status | Ingest exercised (Windows, 2026-08-10); Grok/Ranked in code + unit tests (2026-08-11) | | Package | `opportunity_ingest` | | Python | 3.11+ (verified on 3.14 in local venv) | @@ -12,69 +13,105 @@ ## 1. Pipeline +### 1.1 Ingest (system of record) + ```text CanadaBuys CSV → download (UA headers, 1 retry) → parse (bilingual headers, EN/FR coalesce) → filter_keywords (any-match; short terms word-boundary; span-aware nested suppress) - → score (0–100, UTC) + → score (0–100, UTC) # rule-based RelevanceScore only → map_fields (OpportunityFields) → load_existing_keys (store) → for each new candidate while attempts < MAX_CREATE: - store.create() + store.create() # create-only; never updates Status/Notes → streak state (write runs) → optional Teams notify → exit code matrix - → optional: export-csv | sync-sheets + → optional: export-csv | sync-sheets (Ingest tab) +``` + +### 1.2 Post-ingest Grok rank (derived views only) + +```text +SQLite contract_opportunities + → filter optional Status / limit / min rule score + → load config/objectives.yaml + → Grok (xAI OpenAI-compatible API) batch interpret + fit_score + → write data/rankings/interpret-*.json + .md + → optional full-replace Google Sheet tab Ranked + (default ON when GOOGLE_SHEET_ID set; --no-sync-sheets to skip) ``` +**Not in pipeline today:** website-type classifier, per-portal keyword packs, notice-page crawl, AI rewriting `keywords.yaml` (see BACKLOG). + --- ## 2. Components | Module | Path | Responsibility | |--------|------|----------------| -| CLI | `cli.py` | `run`, `download-sample`, `check-store`, `export-csv`, `sync-sheets` | +| CLI | `cli.py` | `run`, `download-sample`, `check-store`, `export-csv`, `sync-sheets`, `interpret-rank`, `sync-rank-sheets` | | Pipeline | `pipeline.py` | Orchestration, metrics, run logs | | Download | `download.py` | HTTP GET + DEFAULT_HEADERS (anti-403) | | Parse | `parse.py` | HEADER_CANDIDATES, TenderRecord | -| Filter | `filter_keywords.py` | keywords.yaml load + match | +| Filter | `filter_keywords.py` | keywords.yaml load + match (global groups only) | | Score | `score.py` | Rule-based RelevanceScore | | Map | `map_fields.py` | Schema mapping, Link required | +| Interpret / rank | `interpret_rank.py` | Grok rephrase + fit rank vs objectives; local reports | | Config | `config.py` | pydantic-settings / `.env` | | State | `state.py` | zero-new streak | | Notify | `notify.py` | Teams Workflows Adaptive Card + GITHUB_OUTPUT | | Exit | `exit_codes.py` | Exit + notify decisions | -| Sheets | `sheets_sync.py` | Full tab replace via gspread | +| Sheets | `sheets_sync.py` | Full tab replace: Ingest (SQLite) + Ranked (Grok JSON) | | Store protocol | `storage/base.py` | `OpportunityStore` | | Factory | `storage/factory.py` | `sqlite` \| `sharepoint` | | SQLite | `storage/sqlite_store.py` | Default backend, export_csv | | SharePoint | `storage/sharepoint_store.py` | Graph client credentials | +### Config files (repo) + +| Path | Role | +|------|------| +| `config/keywords.yaml` | Eng-owned ingest filter terms + weights | +| `config/objectives.yaml` | Eng-owned company objectives for Grok ranking (not used by ingest filter) | + --- -## 3. Storage backends +## 3. Storage backends and views -### 3.1 SQLite (default) +### 3.1 SQLite (default system of record) - Path: `{DATA_DIR}/contract_opportunities.db` (default `data/…`) - Unique indexes on OpportunityID, Link - `list_rows` / `export_csv` for review - No cloud secrets +- **Grok never UPDATEs these rows** ### 3.2 SharePoint (optional) - MSAL client credentials → Graph - Paginated key load; create list items - Requires app registration + Sites.Selected grant (see provision runbook) -- Export-csv / sync-sheets: **sqlite-oriented** today +- Export-csv / sync-sheets / interpret-rank: **sqlite-oriented** today + +### 3.3 Google Sheets (derived views only) -### 3.3 Google Sheets (derived view) +| Tab | Command | Source | Mode | +|-----|---------|--------|------| +| `Ingest` (default) | `sync-sheets` | SQLite opportunities | Full clear + rewrite | +| `Ranked` (default) | `interpret-rank` / `sync-rank-sheets` | Grok ranking report | Full clear + rewrite | +| `Review` (optional) | human | operator notes | Not written by pipeline | - Not a `STORAGE_BACKEND` -- Command: `sync-sheets` - Extra deps: `pip install -e ".[sheets]"` (`gspread`, `google-auth`) -- Full replace of tab (default `Ingest`) +- Rankings **refused** if target tab name is `Ingest` +- Ranked columns: see `RANKING_EXPORT_COLUMNS` in `sheets_sync.py` + +### 3.4 Local ranking reports + +- Directory: `{DATA_DIR}/rankings/` (default `data/rankings/`, gitignored) +- Files: `interpret-{run_id}.json`, `interpret-{run_id}.md` --- @@ -94,9 +131,14 @@ Loaded from environment and optional **repo-root** `.env` (`config.py`). | `PARTIAL_ERROR_EXIT_THRESHOLD` | `5` | | | `TEAMS_WEBHOOK_URL` | unset | optional | | `GOOGLE_SHEET_ID` | unset | for sync-sheets | -| `GOOGLE_SHEET_TAB` | `Ingest` | | +| `GOOGLE_SHEET_TAB` | `Ingest` | opportunity rows | +| `GOOGLE_SHEET_RANK_TAB` | `Ranked` | Grok rankings | | `GOOGLE_SERVICE_ACCOUNT_FILE` | unset | path to JSON key | | `GOOGLE_SERVICE_ACCOUNT_JSON` | unset | inline JSON (CI) | +| `XAI_API_KEY` | unset | Grok interpret-rank | +| `XAI_BASE_URL` | `https://api.x.ai/v1` | OpenAI-compatible | +| `XAI_MODEL` | `grok-4.5` | | +| `OBJECTIVES_PATH` | `config/objectives.yaml` | company objectives for AI rank | | Azure / SharePoint IDs | unset | only for SP backend | --- @@ -115,6 +157,13 @@ python -m opportunity_ingest run --csv data/sample-openTenderNotice.csv python -m opportunity_ingest run --write --csv data/sample-openTenderNotice.csv --max-create 10 python -m opportunity_ingest export-csv python -m opportunity_ingest sync-sheets + +# Optional Grok ranking (post-ingest; does not mutate SQLite) +pip install -e ".[ai]" +# set XAI_API_KEY in root .env (and keep GOOGLE_* if Ranked push wanted) +python -m opportunity_ingest interpret-rank --status New --limit 20 +# Skip Ranked sheet: --no-sync-sheets +# Re-push last JSON: python -m opportunity_ingest sync-rank-sheets ``` **Observed sample dry-run (live CSV ~6MB):** parsed≈893, filtered≈83, mapped≈27, many map skips for empty Link (expected). @@ -128,8 +177,9 @@ python -m opportunity_ingest sync-sheets | CI | `.github/workflows/ci.yml` — ruff + pytest on 3.11/3.12 | | Daily ingest | `.github/workflows/daily-canadabuys-ingest.yml` — cron `0 14 * * *` UTC, sqlite default, cache `data/`+`state/`, MAX_CREATE soft cap | | Local daily + Sheets | Operator: Task Scheduler → `run --write` then `sync-sheets` (see google_sheets_setup.md) | +| Local + Grok Ranked | After write: `interpret-rank` (needs `XAI_API_KEY`; auto Ranked if sheet id set) | -Sheets sync is **not** yet wired into the GitHub Actions YAML by default (add optional step when secrets exist). +Sheets **Ingest** sync and **interpret-rank** are **not** yet wired into the GitHub Actions YAML by default (BACKLOG B-06). --- @@ -141,7 +191,8 @@ ruff check src tests ``` Fixtures: `tests/fixtures/open_tender_*.csv` -Sheets unit tests mock JSON load only (no live Google). +Sheets unit tests: SA JSON load + ranking grid/helpers (no live Google). +Grok unit tests: fake client (no live xAI). SharePoint tests mock HTTP (no live Graph). --- @@ -154,12 +205,17 @@ SharePoint tests mock HTTP (no live Graph). 4. **Windows double extension** `file.json.json` breaks service account path 5. **Map skips** when keyword hits but notice URL empty — by design 6. Quotes in `.env` values usually unnecessary +7. **Grok fit_score ≠ RelevanceScore** — do not treat Ranked fit as store score +8. **Do not put Grok output on Ingest** — use Ranked (code enforces) --- ## 9. Related docs +- **Human plug-and-play (keys, functions, roadmap):** [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) - Data rules: [`DATA_UPDATE_DIRECTIVES.md`](DATA_UPDATE_DIRECTIVES.md) - Change recipes: [`CHANGE_PLAYBOOK.md`](CHANGE_PLAYBOOK.md) - Decisions: [`DECISIONS.md`](DECISIONS.md) +- Backlog (not built): [`BACKLOG.md`](BACKLOG.md) +- Status snapshot: [`STATUS.md`](STATUS.md) - Original design: [`phase1-canadabuys-sharepoint-implementation-schema.md`](phase1-canadabuys-sharepoint-implementation-schema.md) diff --git a/docs/BACKLOG.md b/docs/BACKLOG.md new file mode 100644 index 0000000..63b060d --- /dev/null +++ b/docs/BACKLOG.md @@ -0,0 +1,179 @@ +# Backlog — discussed, deferred, or not yet built + +**Purpose:** Track ideas and designs that were **discussed or proposed** but are **not implemented** (or only partially documented). +**Do not treat this as as-built.** For what exists in code, use [`AS_BUILT.md`](AS_BUILT.md) and [`STATUS.md`](STATUS.md). + +| Field | Value | +|-------|--------| +| **Last updated** | 2026-08-11 | +| **Owner** | Engineering (product decisions via issues / review) | + +--- + +## How to use this file + +| Status | Meaning | +|--------|---------| +| **Proposed** | Discussed; no design decision locked | +| **Designed (not built)** | Approach agreed in conversation/docs; no production code | +| **Deferred** | Explicitly out of current phase | +| **Blocked** | Waiting on secrets, product answer, or dependency | + +When an item ships: move a one-line note to `STATUS.md` / `AS_BUILT.md` / `DECISIONS.md`, then mark the backlog row **Done** or remove it. + +--- + +## 1. Built recently (do not re-open as “missing”) + +These **are implemented** in code as of 2026-08-11. Listed here only so reviewers do not confuse them with backlog. + +| ID | Item | Where | +|----|------|--------| +| B-DONE-1 | Grok post-ingest interpret + fit rank | `interpret_rank.py`, CLI `interpret-rank` | +| B-DONE-2 | Company objectives frame of reference | `config/objectives.yaml` | +| B-DONE-3 | Local ranking reports | `data/rankings/interpret-*.{json,md}` (gitignored) | +| B-DONE-4 | Sheets **Ranked** tab full-replace | `sheets_sync.sync_rankings_to_sheet`, CLI `sync-rank-sheets` | +| B-DONE-5 | Auto sheet push when `GOOGLE_SHEET_ID` set | `interpret-rank` default; `--no-sync-sheets` to skip | +| B-DONE-6 | Hard refuse rankings → `Ingest` tab | `sync_rankings_to_sheet` | +| B-DONE-7 | Optional extras | `pip install -e ".[ai]"`, `".[sheets]"` | +| B-DONE-8 | Atlas Flow Group site-tuned keywords + objectives | `config/keywords.yaml`, `config/objectives.yaml` (crawl 2026-08-11) | +| B-DONE-9 | Teams high-match capture pipeline (≥40 score) | `config/notify.yaml`, `notify_match_alerts`, ingest write + interpret-rank | + +Normative rules: [`DATA_UPDATE_DIRECTIVES.md`](DATA_UPDATE_DIRECTIVES.md) §2.8, §3.2. + +**Note (2026-08-11):** Keyword/objective packs were manually derived from a full crawl of +https://atlasflowgroupwebsite.vikrant-singh1.workers.dev/ (home, services, approach, about, contact). +Automated site crawl → YAML (B-04) remains unbuilt; this was a one-off eng tuning pass. + +--- + +## 2. Active backlog (discussed, not built) + +### B-01 — Source / website-type classifier +| | | +|--|--| +| **Status** | **Designed (not built)** | +| **Discussed** | 2026-08-11 — model keywords by interpreting “website type” / notice context | +| **Intent** | Classify each notice by portal (CanadaBuys / MERX / Ariba host), instrument (RFI/RFP/RFSA/ITQ), and category family (services vs construction) **before** keyword matching | +| **Proposed approach** | Cheap rules on URL + title/category cues → `SourceType` labels; no LLM required for daily path | +| **Depends on** | Product answers: which “type” axes matter first (portal vs instrument vs company site) | +| **Out of scope for now** | Full multi-source crawlers | + +### B-02 — Keyword packs by type +| | | +|--|--| +| **Status** | **Designed (not built)** | +| **Discussed** | 2026-08-11 — “most appropriate keywords from website type” | +| **Intent** | Split `keywords.yaml` into `global` + packs (`by_portal`, `by_category`, `by_instrument`); active terms = union of packs for classified type | +| **Proposed approach** | Extend `filter_keywords.py` loader; keep rule-based match engine | +| **Constraint** | Keywords remain **eng-owned** (D-10); no silent AI rewrite of production YAML | + +### B-03 — Grok `suggest-keywords` (human-in-the-loop) +| | | +|--|--| +| **Status** | **Proposed** | +| **Discussed** | 2026-08-11 — AI for discovery, not silent filter mutation | +| **Intent** | Offline/on-demand CLI that reads recent opportunities + false positives + `objectives.yaml` and **proposes** a keywords.yaml patch (PR), does not auto-merge | +| **Depends on** | B-01/B-02 optional; `XAI_API_KEY` already used by interpret-rank | +| **Must not** | Auto-edit `config/keywords.yaml` in the daily pipeline | + +### B-04 — Company website → capability vocabulary +| | | +|--|--| +| **Status** | **Proposed** | +| **Discussed** | 2026-08-11 — alternate “compare against” design: crawl *our* site, not only tender portals | +| **Intent** | Extract services language from company website sections → seed objectives + keyword suggestions | +| **Depends on** | Product: real company URL + which pages are authoritative | +| **Note** | Distinct from tender-portal type packs (B-01/B-02) | + +### B-05 — Notice `Link` page crawl / enrichment +| | | +|--|--| +| **Status** | **Proposed** | +| **Discussed** | 2026-08-11 — CSV description often thin; some rows already point at MERX/Ariba | +| **Intent** | Optional fetch of notice abstract HTML to improve filter/rank input | +| **Risks** | ToS, rate limits, auth walls, non-determinism; create-only store must not overwrite existing rows with “refresh” without new design | +| **Constraint** | Never invent Links; never silent-truncate Links | + +### B-06 — Wire Grok rank + Ranked Sheets into GitHub Actions +| | | +|--|--| +| **Status** | **Deferred** (ops) | +| **Intent** | After `run --write` (+ optional `sync-sheets`), run `interpret-rank` in Actions with `XAI_API_KEY` + sheet secrets | +| **Depends on** | Live Grok validation, cost/budget policy, secrets on repo | + +### B-07 — Live E2E validation of interpret-rank + Ranked tab +| | | +|--|--| +| **Status** | **Blocked** (operator secrets / credits) | +| **Intent** | Document a successful live run: `XAI_API_KEY` + real Grok call + Ranked tab visible on shared sheet | +| **Current** | Unit tests mock Grok; Sheets ranking helpers tested without network; live path not recorded as validated in STATUS §2 | + +--- + +## 3. Explicit phase non-goals (still deferred) + +Carried from Phase 1 design / AGENTS hard rules. Not “forgotten”—**intentionally not building now**. + +| ID | Item | Notes | +|----|------|--------| +| N-1 | Multi-source first-class adapters (MERX/municipal as ingest SoR) | CanadaBuys open-data CSV remains primary | +| N-2 | Learning-to-rank / AI scoring from human Status feedback | Separate from post-ingest Grok fit report | +| N-3 | Two-way Sheets ↔ SQLite Status sync | Sheets remain views | +| N-4 | Automatic amendment / closing-date row refresh | Operator opens `Link` | +| N-5 | Dual-write SQLite + SharePoint | Single `STORAGE_BACKEND` | +| N-6 | Auto-bidding / proposal generation | Out of scope | +| N-7 | Historical backfill campaigns | Day-forward ingest | +| N-8 | AI auto-mutates production `keywords.yaml` on schedule | Conflicts with D-10 / eng ownership | + +--- + +## 4. Open product questions (need answers before B-01–B-04) + +Recorded from 2026-08-11 design discussion. Update when answered. + +1. **Website type definition:** portal host vs notice instrument vs category family vs company website — which axis first? +2. **AI role on keywords:** suggest-only (PR) vs any automatic weight tweak? (Default recommendation: **suggest-only**.) +3. **Primary pain:** fewer false positives vs more true positives vs multi-source readiness? +4. **Ingest surface:** stay on CanadaBuys CSV only, or also crawl notice `Link` pages? +5. **Company identity:** keep template `objectives.yaml` or lock real firm name/services for ranking quality? + +--- + +## 5. Roadmap summary (humans) + +**Already built — needs only keys:** see [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) (Sheets, Grok, Teams, SQLite ingest). + +| Horizon | Focus | +|---------|--------| +| **This week (ops)** | Plug `TEAMS_WEBHOOK_URL`; prove match card; GitHub secrets + one scheduled dry run | +| **This month (eng+ops)** | B-06 optional: Actions step for interpret-rank + Ranked; tune objectives from real Ranked outcomes | +| **Next** | B-01/B-02 type-aware keywords if false positives remain high | +| **Later** | B-03 suggest-keywords; B-04/B-05 crawls only with product OK | +| **Not planned now** | N-* auto-bid, dual-write, two-way Status (see §3) | + +### Suggested implementation order (when prioritized) + +```text +0) Human: PLUG_AND_PLAY — Teams webhook + Actions secrets (no code) +1) B-07 already largely done for Grok+Ranked; finish Teams live proof +2) B-06 Actions wire-up (if daily AI cost OK) +3) B-01 source type classifier (rules only) +4) B-02 keyword packs by type +5) B-03 suggest-keywords (Grok → PR diff) +6) B-04 / B-05 only after product answers +``` + +--- + +## 6. Related docs + +| Doc | Role | +|-----|------| +| [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) | Human keys, function list, go-live sequence | +| [`STATUS.md`](STATUS.md) | What works / validated | +| [`AS_BUILT.md`](AS_BUILT.md) | Architecture that exists | +| [`DECISIONS.md`](DECISIONS.md) | Locked ADRs | +| [`DATA_UPDATE_DIRECTIVES.md`](DATA_UPDATE_DIRECTIVES.md) | MUST / MUST NOT | +| [`CHANGE_PLAYBOOK.md`](CHANGE_PLAYBOOK.md) | How to change keywords / Grok objectives | +| [`../AGENTS.md`](../AGENTS.md) | Hard rules for agents | diff --git a/docs/CHANGE_PLAYBOOK.md b/docs/CHANGE_PLAYBOOK.md index 64dcf51..c19a637 100644 --- a/docs/CHANGE_PLAYBOOK.md +++ b/docs/CHANGE_PLAYBOOK.md @@ -54,6 +54,21 @@ How to implement common changes safely. Follow [`DATA_UPDATE_DIRECTIVES.md`](DAT --- +## E2. Tune Grok interpret-rank / company objectives + +1. Edit `config/objectives.yaml` (eng-owned; align with keyword groups when possible). +2. Prompt / schema / merge logic: `interpret_rank.py` + `tests/test_interpret_rank.py`. +3. **Never** write fit scores into SQLite opportunity rows. +4. Local run: `pip install -e ".[ai]"`, set `XAI_API_KEY`, then + `python -m opportunity_ingest interpret-rank --status New --limit 20`. +5. Review `data/rankings/interpret-*.md` and Sheets **Ranked** tab; adjust objectives, not Status automation. +6. Re-push without Grok: `python -m opportunity_ingest sync-rank-sheets`. +7. Skip sheet: `--no-sync-sheets`. + +**Not yet implemented** (do not invent code paths): website-type keyword packs, `suggest-keywords` CLI — track in [`BACKLOG.md`](BACKLOG.md) B-01–B-03. + +--- + ## F. Google Sheets behavior change 1. Default remains **full replace** of one tab unless product explicitly requests upsert. diff --git a/docs/DATA_UPDATE_DIRECTIVES.md b/docs/DATA_UPDATE_DIRECTIVES.md index 9fb08b2..a949a8c 100644 --- a/docs/DATA_UPDATE_DIRECTIVES.md +++ b/docs/DATA_UPDATE_DIRECTIVES.md @@ -3,7 +3,7 @@ **Normative rules for any agent or developer that writes, migrates, or syncs opportunity data.** Violating these breaks dedupe, human review, or external views. -Last updated: 2026-08-10. +Last updated: 2026-08-11. --- @@ -14,6 +14,8 @@ Last updated: 2026-08-10. | **SQLite** `data/contract_opportunities.db` | **System of record (day-1)** | `STORAGE_BACKEND=sqlite` | | **SharePoint list** | Alternate store when activated | `STORAGE_BACKEND=sharepoint` | | **Google Sheets `Ingest` tab** | **Derived view only** (full replace) | `sync-sheets` after writes | +| **Google Sheets `Ranked` tab** | **Derived Grok ranking view** (full replace) | `interpret-rank` / `sync-rank-sheets` | +| **`data/rankings/*`** | Local Grok reports (not SoR) | `interpret-rank` | | **CSV export** | Snapshot for Excel / import | `export-csv` | **MUST:** Treat SQLite (or active SharePoint backend) as authoritative for create/dedupe. @@ -34,6 +36,8 @@ Last updated: 2026-08-10. | `export-csv` | No store write; writes CSV file | | `sync-sheets` | No SQLite write; **replaces** Sheet tab | | `download-sample` | Writes sample CSV file only | +| `interpret-rank` | **No SQLite write**; writes `data/rankings/*`; may full-replace Sheets **Ranked** tab | +| `sync-rank-sheets` | **No SQLite write**; full-replace Sheets **Ranked** tab from ranking JSON | **MUST:** Use `--write` for persistence. **MUST NOT:** Rely on `DRY_RUN` env to enable writes (it never does). @@ -94,10 +98,22 @@ normalize_link(url) = strip → lower → rstrip('/') **MUST NOT:** Overwrite Status on later ingest. **Human Status values:** `New` | `Reviewing` | `Relevant` | `Bidding` | `Discarded`. +### 2.8 Grok interpret-rank (post-ingest) + +**MUST:** Treat AI ranking as a **derived report**, not the system of record. +**MUST NOT:** UPDATE `contract_opportunities` Status, Notes, RelevanceScore, Title, Description, or Link from Grok output. +**MUST:** Persist local interpret-rank files under `data/rankings/` (JSON + Markdown) or an explicit `--out-dir`. +**MUST:** Keep company objectives in engineering-owned `config/objectives.yaml` (not free-text secrets). +**MUST NOT:** Commit `XAI_API_KEY` or ranking report dumps that contain secrets. +**MUST:** Google Sheets ranking push targets the **Ranked** tab only (or another non-`Ingest` name via `GOOGLE_SHEET_RANK_TAB`). +**MUST NOT:** Write Grok rankings into the opportunity **`Ingest`** tab (code refuses this). + --- ## 3. Google Sheets sync directives +### 3.1 Opportunity tab (`sync-sheets`) + Command: `python -m opportunity_ingest sync-sheets` | Rule | Detail | @@ -110,10 +126,20 @@ Command: `python -m opportunity_ingest sync-sheets` | **MUST** | File name exactly `*.json` (Windows double-extension `.json.json` is a known failure mode) | | **MUST** | Load `.env` from **repo root** (not `scripts/.env`) | +### 3.2 Ranked tab (`interpret-rank` / `sync-rank-sheets`) + +| Rule | Detail | +|------|--------| +| **MUST** | Full **clear + rewrite** of target tab (default `Ranked` / `GOOGLE_SHEET_RANK_TAB`) | +| **MUST** | Same spreadsheet + service account as opportunity sync (`GOOGLE_SHEET_ID`) | +| **MUST NOT** | Target the `Ingest` tab for rankings | +| **Default** | When `GOOGLE_SHEET_ID` is set, `interpret-rank` syncs Ranked unless `--no-sync-sheets` | +| **Re-push** | `sync-rank-sheets` uses latest `data/rankings/interpret-*.json` (no Grok call) | + **After ingest recommended sequence:** ```text -run --write → sync-sheets → (optional) export-csv +run --write → sync-sheets → (optional) interpret-rank → (optional) export-csv ``` **MUST NOT:** Two-way sync Status from Sheets back to SQLite in Phase 1 (out of scope). diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index 9c9ab4f..a6f41a2 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -22,13 +22,28 @@ Condensed ADRs for Phase 1. See also design-doc Key Decisions (rev 4). | D-16 | `sync-sheets` optional extra (`[sheets]`) | Keep core install light | | D-17 | EN preferred, FR fallback on bilingual CSV | Maximize non-empty Title/Description/Link | | D-18 | Closing naive times as fixed UTC−05:00 | Deterministic vs DST-dependent ZoneInfo | +| D-19 | Grok ranking is **post-ingest only** (optional) | Ingest stays explainable rule-based; AI never blocks create path | +| D-20 | Grok never UPDATEs SQLite Status/Notes/RelevanceScore | Protects human triage; AI is a report, not SoR | +| D-21 | Company objectives in `config/objectives.yaml` (eng-owned) | Same review model as keywords; frame for fit_score | +| D-22 | Optional `[ai]` extra + `XAI_API_KEY` (xAI / Grok) | Keep core install light; OpenAI-compatible `https://api.x.ai/v1` | +| D-23 | Google Sheets **Ranked** tab separate from **Ingest** | Full-replace both; refuse writing rankings into Ingest | +| D-24 | Auto Ranked sync when `GOOGLE_SHEET_ID` set (`--no-sync-sheets` opt-out) | Live sheet path without extra flags; explicit skip for offline | +| D-25 | `sync-rank-sheets` re-push from local JSON | Avoid re-calling Grok to refresh the sheet | --- ## Explicit non-decisions (deferred) -- Multi-source adapters (MERX, municipal) -- Learning-to-rank / AI scoring from Status feedback +Tracked in detail: [`BACKLOG.md`](BACKLOG.md). + +- Multi-source adapters (MERX, municipal) as first-class ingest +- Learning-to-rank / AI scoring from Status feedback into store - Two-way Sheets ↔ SQLite Status sync - Automatic amendment/closing-date refresh - Dual-write SQLite + SharePoint +- Website/source-type classifier + keyword packs by portal (B-01, B-02) +- Grok auto-suggest keywords into production YAML without human PR (forbidden; B-03 is suggest-only if built) +- Company-website crawl for capability vocabulary (B-04) +- Notice `Link` HTML crawl enrichment (B-05) +- Auto-bidding / proposal generation +- Historical backfill campaigns diff --git a/docs/INDEX.md b/docs/INDEX.md index 86ecdcd..6e9903f 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -10,7 +10,9 @@ Central map for humans and LLMs. **Start at [`../AGENTS.md`](../AGENTS.md).** |-----|-------------| | [`../AGENTS.md`](../AGENTS.md) | Entry point: hard rules, module map, task routing | | [`STATUS.md`](STATUS.md) | Living implementation / ops readiness snapshot | +| [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) | **Human keys + next steps** to turn on built features | | [`AS_BUILT.md`](AS_BUILT.md) | Implemented architecture (source of truth for “what exists now”) | +| [`BACKLOG.md`](BACKLOG.md) | Roadmap / **discussed but not built** | | [`DATA_UPDATE_DIRECTIVES.md`](DATA_UPDATE_DIRECTIVES.md) | Normative data write/sync rules (MUST / MUST NOT) | | [`CHANGE_PLAYBOOK.md`](CHANGE_PLAYBOOK.md) | Step recipes for common development changes | | [`DECISIONS.md`](DECISIONS.md) | Key architectural decisions (ADR-style, as-built) | @@ -41,7 +43,9 @@ Central map for humans and LLMs. **Start at [`../AGENTS.md`](../AGENTS.md).** |------|-------------| | [`../.env.example`](../.env.example) | Root env template | | [`../config/settings.example.env`](../config/settings.example.env) | Settings mirror | -| [`../config/keywords.yaml`](../config/keywords.yaml) | Keyword groups (eng-owned) | +| [`../config/keywords.yaml`](../config/keywords.yaml) | Keyword groups for ingest filter (eng-owned) | +| [`../config/objectives.yaml`](../config/objectives.yaml) | Company objectives for Grok interpret-rank (eng-owned) | +| [`../config/notify.yaml`](../config/notify.yaml) | Teams match threshold / card limits | --- @@ -50,3 +54,4 @@ Central map for humans and LLMs. **Start at [`../AGENTS.md`](../AGENTS.md).** | Doc | Description | |-----|-------------| | [`../README.md`](../README.md) | Quick start, CLI, secrets table, links | +| [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) | Operator plug-in checklist (keys → working stack) | diff --git a/docs/PLUG_AND_PLAY.md b/docs/PLUG_AND_PLAY.md new file mode 100644 index 0000000..35dd295 --- /dev/null +++ b/docs/PLUG_AND_PLAY.md @@ -0,0 +1,338 @@ +# Plug-and-play guide — human keys & next steps + +**Audience:** operators and humans who will turn on what engineering already built. +**As-of:** 2026-08-11 +**Code status:** features below are **implemented**; they stay idle until you plug secrets/config. + +| Related docs | | +|--------------|--| +| What works now | [`STATUS.md`](STATUS.md) | +| Architecture | [`AS_BUILT.md`](AS_BUILT.md) | +| Future / not built | [`BACKLOG.md`](BACKLOG.md) · roadmap § below | +| Data rules | [`DATA_UPDATE_DIRECTIVES.md`](DATA_UPDATE_DIRECTIVES.md) | +| Daily ops | [`../scripts/ops_runbook.md`](../scripts/ops_runbook.md) | + +--- + +## 1. Big picture (what the product does) + +```text +CanadaBuys open tenders (CSV) + │ + ▼ + keyword filter + rule score (0–100) + │ + ▼ + SQLite create-only ◄── system of record + │ + ├──► export-csv + ├──► Google Sheets tab Ingest (view) + ├──► Grok interpret-rank (plain English + fit 0–100) + │ ├──► data/rankings/*.md + │ └──► Google Sheets tab Ranked + └──► Teams + ├── ops alerts (fail / streak) + └── capture pings when score ≥ 40 +``` + +**Not auto-bidding.** Humans still set Status and decide pursue/pass. + +--- + +## 2. Function list (what you can run) + +| CLI command | What it does | Needs human key? | +|-------------|--------------|------------------| +| `download-sample` | Download CanadaBuys open-tender CSV sample | No (public; User-Agent in code) | +| `run` | Dry-run ingest (default) | No | +| `run --write` | Create **new** opportunities in SQLite | No for local SQLite | +| `check-store` | Health + key counts | No (sqlite) | +| `export-csv` | Dump SQLite → CSV | No | +| `sync-sheets` | Full-replace Sheets **Ingest** | **Yes** — Google SA + sheet ID | +| `interpret-rank` | Grok rephrase + rank; optional Ranked tab + Teams | **Yes** — `XAI_API_KEY` (+ Sheets/Teams optional) | +| `sync-rank-sheets` | Re-push last ranking JSON → **Ranked** | **Yes** — Google SA + sheet ID | +| *(internal)* Teams match notify | Card when RelevanceScore or Grok fit ≥ threshold | **Yes** — Teams webhook | + +**Install extras (one-time on the machine):** + +```powershell +pip install -e . +pip install -e ".[sheets]" # Google Sheets +pip install -e ".[ai]" # Grok / xAI +``` + +--- + +## 3. Keys & secrets checklist (you provide) + +Copy `.env.example` → **repo-root** `.env` (never `scripts/.env`). Never commit `.env` or `secrets/`. + +| # | Secret / config | Where to get it | Unlocks | +|---|-----------------|-----------------|---------| +| 1 | *(none)* | — | Local ingest + SQLite + export-csv | +| 2 | `GOOGLE_SHEET_ID` | Spreadsheet URL `…/d//edit` | Sheets sync | +| 3 | `GOOGLE_SERVICE_ACCOUNT_FILE` or `GOOGLE_SERVICE_ACCOUNT_JSON` | Google Cloud SA key JSON | Sheets API | +| 4 | Share sheet **Editor** with SA email | Sheets UI Share | Sheets write works | +| 5 | `XAI_API_KEY` | [console.x.ai](https://console.x.ai) API keys | `interpret-rank` | +| 6 | `TEAMS_WEBHOOK_URL` | Teams channel → Workflows → “Post when webhook received” | Ops alerts + match fallback | +| 7 | `TEAMS_MATCH_WEBHOOK_URL` *(optional)* | Second Workflows webhook | Dedicated capture channel | +| 8 | `TEAMS_MATCH_SCORE_THRESHOLD` | Default **40** via env or `config/notify.yaml` | When to ping | +| 9 | Azure / SharePoint secrets | Only if `STORAGE_BACKEND=sharepoint` | Alternate store (not required day-1) | +| 10 | GitHub Actions secrets/vars | Repo Settings → Secrets | Unattended daily schedule | + +### Minimal `.env` templates + +**A — Local SQLite only (works immediately after `pip install -e .`)** + +```env +STORAGE_BACKEND=sqlite +DATA_DIR=data +MAX_CREATE=50 +LOG_LEVEL=INFO +``` + +**B — + Google Sheets (Ingest + Ranked)** + +```env +GOOGLE_SHEET_ID=1yourSpreadsheetId +GOOGLE_SHEET_TAB=Ingest +GOOGLE_SHEET_RANK_TAB=Ranked +GOOGLE_SERVICE_ACCOUNT_FILE=secrets/google-service-account.json +``` + +**C — + Grok ranking** + +```env +XAI_API_KEY=xai-... +XAI_MODEL=grok-4.5 +OBJECTIVES_PATH=config/objectives.yaml +``` + +**D — + Teams** + +```env +TEAMS_WEBHOOK_URL=https://...powerautomate.../invoke +# optional separate capture channel: +# TEAMS_MATCH_WEBHOOK_URL=https://... +TEAMS_MATCH_NOTIFY_ENABLED=true +TEAMS_MATCH_SCORE_THRESHOLD=40 +``` + +Detailed Sheets steps: [`../scripts/google_sheets_setup.md`](../scripts/google_sheets_setup.md). + +--- + +## 4. Human plug-and-play sequence (recommended order) + +Do these in order. Skip optional blocks you do not need yet. + +### Step 0 — One-time machine setup (no keys) + +```powershell +cd "" +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +pip install -r requirements.txt -r requirements-dev.txt +pip install -e . +copy .env.example .env +# edit .env with STORAGE_BACKEND=sqlite at minimum +``` + +**Verify:** + +```powershell +python -m opportunity_ingest --help +python -m opportunity_ingest check-store +``` + +### Step 1 — Ingest works offline + +```powershell +python -m opportunity_ingest download-sample +python -m opportunity_ingest run --csv data/sample-openTenderNotice.csv +python -m opportunity_ingest run --write --csv data/sample-openTenderNotice.csv --max-create 25 +python -m opportunity_ingest export-csv +python -m opportunity_ingest check-store +``` + +**You should see:** SQLite file under `data/`, CSV export, non-zero opportunity count. + +### Step 2 — Google Sheets (human actions) + +1. Create a Google Cloud **service account** → download JSON key. +2. Save as `secrets/google-service-account.json` (**not** `.json.json`). +3. Create a Google Sheet; rename first tab **`Ingest`** (optional tab **`Ranked`** / **`Review`**). +4. **Share** the sheet with the service account email as **Editor**. +5. Put `GOOGLE_SHEET_ID` + `GOOGLE_SERVICE_ACCOUNT_FILE` in root `.env`. +6. `pip install -e ".[sheets]"` +7. Run: + +```powershell +python -m opportunity_ingest sync-sheets +``` + +**You should see:** Ingest tab filled with opportunity rows (full replace each run). + +### Step 3 — Grok ranking (human actions) + +1. Create key at [console.x.ai](https://console.x.ai); add credits. +2. Set `XAI_API_KEY` in root `.env`. +3. Review/edit **`config/objectives.yaml`** (company frame — currently Atlas Flow Group). +4. `pip install -e ".[ai]"` +5. Run: + +```powershell +python -m opportunity_ingest interpret-rank --status New --limit 27 +``` + +**You should see:** + +- `data/rankings/interpret-*.md` +- Sheets tab **`Ranked`** updated (if Sheets configured) +- Console: `top #1 fit=…` + +Skip sheet: `--no-sync-sheets`. Skip Teams: `--no-teams`. + +### Step 4 — Teams channel pings (human actions) + +1. In the target **Teams channel** (or chat via Workflows that posts there): + **Workflows** → template *Post to a channel when a webhook request is received* → copy URL. +2. Set `TEAMS_WEBHOOK_URL` (and optionally `TEAMS_MATCH_WEBHOOK_URL` for a capture-only channel). +3. Confirm `config/notify.yaml` or env: threshold **40**, `match_notify_enabled: true`. +4. Trigger: + +```powershell +# New high rule-score creates: +python -m opportunity_ingest run --write --max-create 10 + +# Or Grok fits ≥ 40: +python -m opportunity_ingest interpret-rank --limit 27 +``` + +**You should see:** Adaptive Card in the channel with summaries + **Open notice** buttons. + +Ops-only alerts (hard fail / zero-new streak) also use `TEAMS_WEBHOOK_URL` without needing a separate match URL. + +### Step 5 — Daily automation (pick one) + +| Path | Human work | +|------|------------| +| **Windows Task Scheduler** | Point at `scripts/daily_sync.ps1` (or extend it with `interpret-rank`) | PC must be on | +| **GitHub Actions** | Push `dev`/`main`; set secrets below; run `workflow_dispatch` once | Unattended | + +**GitHub secrets / vars to plug:** + +| Name | Required for | +|------|----------------| +| `TEAMS_WEBHOOK_URL` | Alerts (recommended) | +| `INGEST_MAX_CREATE` (variable) | Soft create budget (default 50) | +| `GOOGLE_SHEET_ID` + `GOOGLE_SERVICE_ACCOUNT_JSON` | Sheets in CI (optional; add step) | +| `XAI_API_KEY` | interpret-rank in CI (optional; not in workflow by default yet — BACKLOG B-06) | + +Workflow file: [`.github/workflows/daily-canadabuys-ingest.yml`](../.github/workflows/daily-canadabuys-ingest.yml). + +### Step 6 — Human triage (ongoing, no more keys) + +1. Open Sheets **Ingest** or **Ranked**, or `export-csv`. +2. Set **Status** / Notes on rows you care about (SQLite or Review tab — **not** relying on Ingest if it is auto-replaced). +3. Act on Teams CTAs (open Link, decide pursue/pass). +4. Request keyword/objective changes via eng (`config/keywords.yaml`, `config/objectives.yaml`). + +--- + +## 5. Config files you may edit (no rebuild) + +| File | Who | Purpose | +|------|-----|---------| +| `.env` | Ops | Secrets + paths (gitignored) | +| `config/keywords.yaml` | Eng | Ingest keyword groups / weights | +| `config/objectives.yaml` | Eng | Grok ranking company frame | +| `config/notify.yaml` | Eng/Ops | Match threshold, max items, enable flag | +| `secrets/google-service-account.json` | Ops | Sheets key (gitignored) | + +--- + +## 6. End-to-end “happy path” (everything on) + +After Steps 0–4: + +```powershell +.\.venv\Scripts\Activate.ps1 +python -m opportunity_ingest run --write --max-create 50 +python -m opportunity_ingest sync-sheets +python -m opportunity_ingest interpret-rank --status New --limit 50 +``` + +| Output | Where | +|--------|--------| +| New rows | SQLite `data/contract_opportunities.db` | +| Opportunity grid | Sheets **Ingest** | +| Grok brief | `data/rankings/interpret-*.md` + Sheets **Ranked** | +| Capture ping | Teams channel if any score ≥ 40 | + +--- + +## 7. Feature readiness vs human plug + +| Capability | Code | Human must plug | Live-validated? | +|------------|------|-----------------|-----------------| +| CanadaBuys download + filter + score | Yes | — | Yes (local) | +| SQLite create-only store | Yes | — | Yes | +| export-csv | Yes | — | Yes | +| Sheets **Ingest** | Yes | SA + sheet share + ID | Yes (local) | +| Grok **interpret-rank** | Yes | `XAI_API_KEY` | Yes (local) | +| Sheets **Ranked** | Yes | same Sheets as Ingest | Yes (local) | +| Teams ops alerts | Yes | `TEAMS_WEBHOOK_URL` | Code ready; needs your webhook | +| Teams match ≥40 | Yes | webhook + threshold | Code ready; needs your webhook | +| GitHub daily schedule | Yes | repo secrets + enable schedule | Not fully go-live | +| SharePoint SoR | Yes | Entra + Sites.Selected | Not activated | +| Type-based keyword packs / auto crawl | No | — | See roadmap | + +--- + +## 8. Roadmap (what’s next — not plug-and-play yet) + +Full detail: [`BACKLOG.md`](BACKLOG.md). Summary for humans: + +| Priority | ID | Item | Blocked on | +|----------|-----|------|------------| +| Now (ops) | B-07 | Confirm Teams webhook live in channel | Human Workflows URL | +| Now (ops) | — | GitHub Actions secrets + first scheduled run | Human repo settings | +| Near | B-06 | Wire interpret-rank + Ranked + Teams into Actions YAML | Secrets + cost policy | +| Near | B-01 / B-02 | Source-type classifier + keyword packs | Product priority | +| Later | B-03 | Grok suggest-keywords (PR only) | Design sign-off | +| Later | B-04 / B-05 | Company-site crawl / notice page crawl | Product + ToS | +| Deferred | N-* | Multi-source SoR, auto-bid, two-way Status, dual-write | Phase non-goals | + +**Suggested human order after keys work:** + +1. Plug Teams webhook → prove one match card. +2. Enable Actions schedule with small `INGEST_MAX_CREATE`. +3. Weekly: review Ranked + Status triage. +4. Eng: tune keywords/objectives from real false positives. + +--- + +## 9. Quick troubleshooting + +| Symptom | Check | +|---------|--------| +| CanadaBuys **403** | Code has User-Agent; retry; network block | +| `set XAI_API_KEY` | Root `.env`; restart shell; `pip install -e ".[ai]"` | +| Sheets permission error | Share sheet with **SA email** as Editor | +| SA file not found | Exact path; not `.json.json` | +| Teams silent | Webhook URL set; Workflows enabled; threshold too high vs scores | +| No new SQLite rows | Create-only + dedupe; already ingested IDs skip | +| Ranked empty | Run `interpret-rank` with key; not the same as `sync-sheets` | + +--- + +## 10. One-page “who does what” + +| Role | Does | +|------|------| +| **Human (you)** | Keys, sheet share, Teams Workflow, Status triage, bid/no-bid | +| **Engineering** | keywords.yaml, objectives.yaml, code, tests, Actions YAML | +| **System (already built)** | Download, filter, score, store, export, Sheets replace, Grok rank, Teams cards | + +When something is built but idle, the fix is almost always **this guide §3–4**, not new code. diff --git a/docs/STATUS.md b/docs/STATUS.md index ed6f810..781c432 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -5,12 +5,12 @@ Living snapshot of implementation and ops readiness. | Field | Value | |-------|--------| -| **Last updated** | 2026-08-10 | -| **Phase** | Phase 1 — CanadaBuys open tender ingest | -| **Overall status** | **Implemented & manually validated (local); code on GitHub** | +| **Last updated** | 2026-08-11 | +| **Phase** | Phase 1 ingest + optional Grok interpret-rank (post-ingest) | +| **Overall status** | **Ingest implemented & locally validated; Grok/Ranked coded + unit-tested; live Grok E2E pending** | | **GitHub** | https://github.com/vikiscoding/contractbiddingsystem (**private**) | | **Default branch** | `main` | -| **Recommended next** | Configure Actions secrets/vars → `workflow_dispatch` smoke → enable schedule | +| **Recommended next** | **Human plug-in:** [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) — Teams webhook, Actions secrets, confirm daily path. Eng roadmap: [`BACKLOG.md`](BACKLOG.md) | --- @@ -19,30 +19,49 @@ Living snapshot of implementation and ops readiness. | Capability | Status | Notes | |------------|--------|--------| | Download CanadaBuys open tender CSV | **Working** | Browser User-Agent required (403 without) | -| Bilingual parse + keyword filter + score | **Working** | `config/keywords.yaml` eng-owned | +| Bilingual parse + keyword filter + score | **Working** | `config/keywords.yaml` eng-owned; **single global pack** (no per-site packs yet) | | SQLite create-only store | **Working** | System of record (day-1) | | Dedupe OpportunityID + Link | **Working** | Unique indexes + pre-check | | CLI dry-run / `--write` / export-csv | **Working** | Only `--write` persists | -| Google Sheets full-replace sync | **Working (local)** | `sync-sheets`; service account + shared sheet | +| Google Sheets `Ingest` full-replace | **Working (local)** | `sync-sheets`; service account + shared sheet | +| Google Sheets `Ranked` full-replace | **Implemented** | From Grok report; same sheet/SA; **never** writes `Ingest` | +| Grok interpret-rank (post-ingest) | **Implemented (optional)** | `interpret-rank`; needs `XAI_API_KEY` + `.[ai]`; does **not** mutate SQLite | +| Re-push rankings without Grok | **Implemented** | `sync-rank-sheets` from latest `data/rankings/interpret-*.json` | | SharePoint Graph adapter | **Implemented, not activated** | Needs Entra + Sites.Selected grant | -| Teams Workflows notify | **Code ready** | Needs `TEAMS_WEBHOOK_URL` | -| GitHub Actions CI (`ci.yml`) | **In repo** | Runs on push/PR once remote exists | -| GitHub Actions daily ingest | **In repo, not yet scheduled live** | Needs remote + vars/secrets; enable after calibration | -| Multi-source / AI ranking | **Out of scope** | Phase 1 non-goals | +| Teams Workflows ops notify | **Code ready** | hard fail / partial / streak → `TEAMS_WEBHOOK_URL` | +| Teams high-match capture ping | **Implemented** | score ≥40 (config); Adaptive Card + OpenUrl CTA; `config/notify.yaml` | +| GitHub Actions CI (`ci.yml`) | **In repo** | ruff + pytest | +| GitHub Actions daily ingest | **In repo, not yet scheduled live** | Needs vars/secrets; enable after calibration | +| Source/website-type keyword packs | **Not built** | Tracked: [`BACKLOG.md`](BACKLOG.md) B-01, B-02 | +| AI auto-suggest keywords CLI | **Not built** | Tracked: B-03 | +| Multi-source / auto-bidding | **Out of scope** | Non-goals; see BACKLOG §3 | --- -## 2. Validated local runs (2026-08-10) +## 2. Validated local runs + +### 2.1 Ingest + Sheets Ingest (2026-08-10) | Step | Result | |------|--------| | `pip install -e .` / `.[sheets]` | OK | | `download-sample` | OK (~6.2 MB open tender CSV) after UA fix | | Dry-run on sample | parsed≈**893**, filtered≈**83**, mapped≈**27** (many map skips: empty Link) | -| `run --write` (capped) | SQLite rows created | +| `run --write` (capped) | SQLite rows created (~27 local) | | `export-csv` | OK → `data/export-opportunities.csv` (local, gitignored) | -| `sync-sheets` | OK → **27 rows** written to Google Sheet tab `Ingest` | -| Unit tests | **pytest** green (package + sheets unit tests) | +| `sync-sheets` | OK → rows written to Google Sheet tab **`Ingest`** | +| Unit tests (ingest era) | **pytest** green | + +### 2.2 Grok interpret-rank + Ranked tab (2026-08-11) + +| Step | Result | +|------|--------| +| Code: `interpret_rank.py`, CLI, `objectives.yaml` | **In tree** | +| Code: `sync_rankings_to_sheet` / `sync-rank-sheets` | **In tree** | +| Docs: DATA_UPDATE, AGENTS, README, playbook | **Updated** | +| Unit tests (mocked Grok, no live xAI) | **pytest green** (210 as of doc update) | +| Live `XAI_API_KEY` → real Grok ranking | **Not recorded as validated** (see BACKLOG B-07) | +| Live push to sheet tab **`Ranked`** | **Not recorded as validated** (B-07) | --- @@ -54,7 +73,10 @@ Living snapshot of implementation and ops readiness. | Root `.env` | Present (gitignored) | **Not** `scripts/.env` | | `STORAGE_BACKEND` | `sqlite` | Day-1 default | | `secrets/google-service-account.json` | Present (gitignored) | Exact `.json` name (not `.json.json`) | -| `GOOGLE_SHEET_ID` / tab | Configured | Sheet shared Editor with SA email | +| `GOOGLE_SHEET_ID` / `Ingest` tab | Configured | Sheet shared Editor with SA email | +| `GOOGLE_SHEET_RANK_TAB` | `Ranked` (default) | Created on first rank sync if missing | +| `XAI_API_KEY` | Optional | Required only for `interpret-rank` | +| `config/objectives.yaml` | Present | Eng-owned frame for Grok (template until customized) | | `TEAMS_WEBHOOK_URL` | Optional | Alerts silent if unset | | SharePoint secrets | Not required | Until SP activation | @@ -65,13 +87,15 @@ Living snapshot of implementation and ops readiness. | Store / view | Role | Live? | |--------------|------|-------| | SQLite `data/contract_opportunities.db` | **System of record** | Local yes | -| Google Sheets `Ingest` | Derived view (full replace) | Yes (user sheet) | +| Google Sheets `Ingest` | Derived opportunity view (full replace) | Yes (user sheet) | +| Google Sheets `Ranked` | Derived Grok ranking view (full replace) | Code ready; live E2E pending | | Google Sheets `Review` (if used) | Human notes | Operator-owned | +| `data/rankings/*` | Local Grok reports | Local only; gitignored | | SharePoint list | Alternate SoR | Not activated | | Teams | Failure / streak notify | Not configured | **Data rules:** see [`DATA_UPDATE_DIRECTIVES.md`](DATA_UPDATE_DIRECTIVES.md). -Create-only ingest; never silent-truncate Link; Sheets is not SoR. +Create-only ingest; never silent-truncate Link; Sheets is not SoR; Grok never rewrites SQLite Status/Notes/scores. --- @@ -81,8 +105,11 @@ Create-only ingest; never silent-truncate Link; Sheets is not SoR. 2. Windows **double extension** on service account file (`.json.json`). 3. Keyword hits with **empty notice URL** → map skip (by design). 4. `MAX_CREATE=0` means **unlimited**, not “block writes”. -5. Daily Actions workflow does **not** run Sheets sync by default (add step + secrets when ready). -6. `export-csv` / `sync-sheets` are **sqlite-oriented**. +5. Daily Actions workflow does **not** run Sheets sync or interpret-rank by default. +6. `export-csv` / `sync-sheets` / `interpret-rank` are **sqlite-oriented**. +7. Rule keywords are **global only** — false positives (e.g. construction “service desk”) still possible; type packs not built (BACKLOG B-01/B-02). +8. `objectives.yaml` still template company name until customized. +9. Without `XAI_API_KEY`, `interpret-rank` exits usage error (expected). --- @@ -93,7 +120,9 @@ Create-only ingest; never silent-truncate Link; Sheets is not SoR. - [ ] Repo variable `INGEST_MAX_CREATE=50` (or desired) - [ ] Secret `TEAMS_WEBHOOK_URL` (recommended) - [ ] Optional Sheets: secrets `GOOGLE_SHEET_ID`, `GOOGLE_SERVICE_ACCOUNT_JSON` + workflow step +- [ ] Optional Grok: secret `XAI_API_KEY` + workflow step (BACKLOG B-06) - [ ] Manual `workflow_dispatch` dry-run / capped write successful +- [ ] Live interpret-rank + Ranked tab smoke (BACKLOG B-07) - [ ] Enable schedule confidence after calibration week - [ ] Operators following [`scripts/ops_runbook.md`](../scripts/ops_runbook.md) @@ -103,10 +132,13 @@ Create-only ingest; never silent-truncate Link; Sheets is not SoR. | Need | Doc | |------|-----| +| **Human keys + plug-and-play + function list + roadmap** | [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) | | LLM / agent context | [`../AGENTS.md`](../AGENTS.md) | -| Architecture | [`AS_BUILT.md`](AS_BUILT.md) | +| Architecture (what exists) | [`AS_BUILT.md`](AS_BUILT.md) | +| **Discussed / not built** | [`BACKLOG.md`](BACKLOG.md) | | Data MUST/MUST NOT | [`DATA_UPDATE_DIRECTIVES.md`](DATA_UPDATE_DIRECTIVES.md) | | How to change code | [`CHANGE_PLAYBOOK.md`](CHANGE_PLAYBOOK.md) | +| Decisions (ADR) | [`DECISIONS.md`](DECISIONS.md) | | Doc index | [`INDEX.md`](INDEX.md) | | Sheets setup | [`../scripts/google_sheets_setup.md`](../scripts/google_sheets_setup.md) | | Ops | [`../scripts/ops_runbook.md`](../scripts/ops_runbook.md) | @@ -118,9 +150,11 @@ Create-only ingest; never silent-truncate Link; Sheets is not SoR. When something material changes, edit §1–§6 and bump **Last updated**. Examples: - First successful Actions scheduled run +- Live Grok + Ranked tab validation (close B-07) - SharePoint activation -- Sheets wired into Actions +- Sheets / interpret-rank wired into Actions - Keyword/policy changes that affect typical filtered volume - New known issues +- Backlog item shipped → mark Done in BACKLOG + capability matrix -Keep STATUS factual and short; put deep rules in DATA_UPDATE_DIRECTIVES / AS_BUILT. +Keep STATUS factual and short; put deep rules in DATA_UPDATE_DIRECTIVES / AS_BUILT; put unbuilt designs in BACKLOG. diff --git a/pyproject.toml b/pyproject.toml index ae356e5..083575c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,9 @@ sheets = [ "gspread>=6.0", "google-auth>=2.0", ] +ai = [ + "openai>=1.40", +] [project.scripts] opportunity-ingest = "opportunity_ingest.cli:main" diff --git a/scripts/google_sheets_setup.md b/scripts/google_sheets_setup.md index d71aa6a..754037d 100644 --- a/scripts/google_sheets_setup.md +++ b/scripts/google_sheets_setup.md @@ -96,7 +96,8 @@ You will share the Sheet with this address. 1. Open [Google Sheets](https://sheets.google.com) → **Blank** spreadsheet. 2. Rename the file, e.g. **Contract Opportunities**. 3. Rename the first tab to **`Ingest`** (or leave default and set `GOOGLE_SHEET_TAB` later). -4. Optional: add a second tab **`Review`** for human notes (job will not touch it if you only sync `Ingest`). +4. Optional: add a **`Ranked`** tab for Grok interpret-rank output (created automatically on first rank sync if missing). Do **not** put rankings on `Ingest`. +5. Optional: add a **`Review`** tab for human notes (pipeline will not touch it if you only sync `Ingest` / `Ranked`). ### B2. Share the sheet with the service account @@ -139,7 +140,9 @@ DATA_DIR=data GOOGLE_SHEET_ID=1abc...your_id...xyz GOOGLE_SHEET_TAB=Ingest +GOOGLE_SHEET_RANK_TAB=Ranked GOOGLE_SERVICE_ACCOUNT_FILE=secrets/google-service-account.json +# Optional Grok (separate extra): XAI_API_KEY=... ``` **Do not** commit `.env` or the JSON key. @@ -165,7 +168,7 @@ Or override flags: python -m opportunity_ingest sync-sheets --sheet-id "1abc..." --tab Ingest ``` -Expected success line: +Expected success line (Ingest): ```text Synced 10 rows to sheet 1abc... tab 'Ingest' @@ -174,6 +177,24 @@ Synced 10 rows to sheet 1abc... tab 'Ingest' Open the Google Sheet in the browser → **Ingest** tab should show headers + rows (Title, OpportunityID, Link, RelevanceScore, Status, …). +### D2. Grok Ranked tab (optional) + +Same spreadsheet and service account. Rankings never write to **Ingest**. + +```powershell +pip install -e ".[ai]" +# XAI_API_KEY in root .env +python -m opportunity_ingest interpret-rank --status New --limit 20 +# or re-push last local report without calling Grok: +python -m opportunity_ingest sync-rank-sheets +``` + +- Creates/replaces tab **`Ranked`** (or `GOOGLE_SHEET_RANK_TAB`). +- Local copies: `data/rankings/interpret-*.md`. +- Skip sheet push: `--no-sync-sheets`. +- Full human checklist (all features): [`docs/PLUG_AND_PLAY.md`](../docs/PLUG_AND_PLAY.md). +- Architecture: [`docs/AS_BUILT.md`](../docs/AS_BUILT.md). Unbuilt ideas: [`docs/BACKLOG.md`](../docs/BACKLOG.md). + --- ## Part E — Daily automatic update diff --git a/scripts/ops_runbook.md b/scripts/ops_runbook.md index c684b92..3aa28b9 100644 --- a/scripts/ops_runbook.md +++ b/scripts/ops_runbook.md @@ -4,8 +4,9 @@ Operator guide for day-1 SQLite storage, Status triage, calibration, create caps **Audience:** operators who review opportunities and manage the daily schedule. **Engineering owns:** `config/keywords.yaml`, pipeline code, GitHub workflow YAML. +**First-time plug-in (keys, Grok, Teams, function list, roadmap):** **[PLUG_AND_PLAY.md](../docs/PLUG_AND_PLAY.md)**. **LLM / data rules:** [AGENTS.md](../AGENTS.md), [DATA_UPDATE_DIRECTIVES](../docs/DATA_UPDATE_DIRECTIVES.md), [AS_BUILT](../docs/AS_BUILT.md). -**Related:** [README.md](../README.md), [google_sheets_setup.md](google_sheets_setup.md), [provision_sharepoint_list.md](provision_sharepoint_list.md), [daily_sync.ps1](daily_sync.ps1). +**Related:** [README.md](../README.md), [google_sheets_setup.md](google_sheets_setup.md), [provision_sharepoint_list.md](provision_sharepoint_list.md), [daily_sync.ps1](daily_sync.ps1), [BACKLOG.md](../docs/BACKLOG.md). --- @@ -14,11 +15,17 @@ Operator guide for day-1 SQLite storage, Status triage, calibration, create caps ```text 1. run --write → SQLite system of record (create-only) 2. sync-sheets → optional full-replace of Google tab "Ingest" -3. export-csv → optional Excel snapshot -4. Human Status triage → SQLite / export / Sheet Review tab (not Ingest if auto-synced) +3. interpret-rank → optional Grok plain-English + fit rank + (writes data/rankings/*; full-replaces Sheet tab "Ranked" + when GOOGLE_SHEET_ID set — use --no-sync-sheets to skip) +4. export-csv → optional Excel snapshot +5. Human Status triage → SQLite / export / Sheet Review tab (not Ingest if auto-synced) + Use Ranked tab for capture prioritization; do not treat fit_score + as store RelevanceScore ``` -Windows automation sample: [`daily_sync.ps1`](daily_sync.ps1) (Task Scheduler). +Windows automation sample: [`daily_sync.ps1`](daily_sync.ps1) (Task Scheduler). +**Not built yet:** website-type keyword packs, auto keyword suggestions — see [`docs/BACKLOG.md`](../docs/BACKLOG.md). --- @@ -57,7 +64,25 @@ pip install -e ".[sheets]" python -m opportunity_ingest sync-sheets ``` -- Setup: [google_sheets_setup.md](google_sheets_setup.md) +- Setup: [google_sheets_setup.md](google_sheets_setup.md) +- Tab **`Ingest`**: opportunity rows from SQLite (full replace). +- Tab **`Ranked`**: Grok ranking (full replace via `interpret-rank` / `sync-rank-sheets`). +- Tab **`Review`**: human-only if you create it — pipeline does not write it. + +### Grok interpret-rank (optional) + +Requires `pip install -e ".[ai]"`, root `.env` `XAI_API_KEY`, and eng-owned `config/objectives.yaml`. + +```bash +python -m opportunity_ingest interpret-rank --status New --limit 20 +# Offline reports only: +python -m opportunity_ingest interpret-rank --limit 20 --no-sync-sheets +# Re-push last ranking JSON to Ranked tab (no Grok call): +python -m opportunity_ingest sync-rank-sheets +``` + +- Local files: `data/rankings/interpret-*.md` (and `.json`). +- **Does not** change SQLite Status, Notes, or RelevanceScore. - **Ingest** tab is **full-replaced** every sync — put manual notes on a separate **Review** tab - SQLite remains system of record; Sheets is a derived view - Normative rules: [DATA_UPDATE_DIRECTIVES.md](../docs/DATA_UPDATE_DIRECTIVES.md) §3 @@ -406,21 +431,33 @@ python -m opportunity_ingest run [--write | --dry-run] [--csv PATH] [--max-creat python -m opportunity_ingest download-sample [--out PATH] python -m opportunity_ingest check-store python -m opportunity_ingest export-csv [--out PATH] +python -m opportunity_ingest sync-sheets [--sheet-id ID] [--tab NAME] +python -m opportunity_ingest interpret-rank [--status S] [--limit N] [--sync-sheets|--no-sync-sheets] +python -m opportunity_ingest sync-rank-sheets [--from-json PATH] [--rank-tab NAME] ``` -Only **`--write`** persists. Default (and `--dry-run`) never writes. `DRY_RUN` env never enables write alone. +Only **`--write`** on `run` persists opportunities. Default (and `--dry-run`) never writes store rows. `DRY_RUN` env never enables write alone. +`interpret-rank` / sheet commands never UPDATE opportunity rows. ## Quick reference — key env / vars | Name | Default | Notes | |------|---------|--------| | `STORAGE_BACKEND` | `sqlite` | `sharepoint` when activated | -| `DATA_DIR` | `data` | SQLite + exports | +| `DATA_DIR` | `data` | SQLite + exports + rankings | | `MAX_CREATE` / `INGEST_MAX_CREATE` | `50` | Attempt budget; `0` = unlimited | | `ZERO_NEW_STREAK_THRESHOLD` | `3` | Teams after N zero-new UTC days | | `PARTIAL_ERROR_EXIT_THRESHOLD` | `5` | Exit 1 + notify | -| `TEAMS_WEBHOOK_URL` | — | Strongly recommended day-1 | -| `KEYWORDS_PATH` | `config/keywords.yaml` | Eng-owned | +| `TEAMS_WEBHOOK_URL` | — | Ops alerts (fail/streak); also match fallback | +| `TEAMS_MATCH_WEBHOOK_URL` | — | Optional dedicated capture channel | +| `TEAMS_MATCH_SCORE_THRESHOLD` | `40` | Ping when RelevanceScore or Grok fit ≥ N | +| `TEAMS_MATCH_NOTIFY_ENABLED` | `true` | Master switch for match pings | +| `KEYWORDS_PATH` | `config/keywords.yaml` | Eng-owned ingest filter | +| `OBJECTIVES_PATH` | `config/objectives.yaml` | Eng-owned Grok frame | +| `XAI_API_KEY` | — | Required for interpret-rank | +| `GOOGLE_SHEET_ID` | — | Ingest + Ranked sheet | +| `GOOGLE_SHEET_TAB` | `Ingest` | Opportunity full-replace | +| `GOOGLE_SHEET_RANK_TAB` | `Ranked` | Grok full-replace | | `STATE_PATH` | `state/zero_new_streak.json` | Streak JSON | --- diff --git a/src/opportunity_ingest/cli.py b/src/opportunity_ingest/cli.py index 29bf584..93ea1ef 100644 --- a/src/opportunity_ingest/cli.py +++ b/src/opportunity_ingest/cli.py @@ -7,9 +7,15 @@ python -m opportunity_ingest check-store python -m opportunity_ingest export-csv [--out PATH] python -m opportunity_ingest sync-sheets [--sheet-id ID] [--tab NAME] + python -m opportunity_ingest interpret-rank [options] [--sync-sheets] + python -m opportunity_ingest sync-rank-sheets [--from-json PATH] [--tab NAME] Write gating: only ``--write`` persists. ``DRY_RUN`` env never enables write and does not disable ``--write``. Default (neither flag) is dry-run. + +``interpret-rank`` is post-ingest only: it never updates store Status/Notes/scores. +Grok rankings may full-replace a separate Google Sheet tab (default ``Ranked``), +never the opportunity ``Ingest`` tab. """ from __future__ import annotations @@ -21,9 +27,22 @@ from opportunity_ingest.config import Settings, get_settings from opportunity_ingest.download import DEFAULT_SAMPLE_PATH, DownloadError, download_to_path from opportunity_ingest.exit_codes import EXIT_FAILURE, EXIT_USAGE +from opportunity_ingest.interpret_rank import ( + DEFAULT_BATCH_SIZE, + InterpretRankError, + run_interpret_rank, +) from opportunity_ingest.logging_setup import setup_logging +from opportunity_ingest.notify import match_items_from_ranked, notify_match_alerts from opportunity_ingest.pipeline import resolve_write_mode, run_pipeline -from opportunity_ingest.sheets_sync import SheetsSyncError, sync_sqlite_to_sheet +from opportunity_ingest.sheets_sync import ( + DEFAULT_RANK_TAB, + SheetsSyncError, + find_latest_rankings_json, + load_rankings_json, + sync_rankings_to_sheet, + sync_sqlite_to_sheet, +) from opportunity_ingest.storage.base import StoreError from opportunity_ingest.storage.factory import build_store from opportunity_ingest.storage.sqlite_store import SqliteOpportunityStore @@ -92,7 +111,9 @@ def cmd_run( f"errors={metrics.error_count} skipped_dup={metrics.skipped_duplicate_count} " f"skipped_max={metrics.skipped_max_create_count} " f"streak={metrics.consecutive_zero_new_days} " - f"exit={metrics.exit_code} notified={metrics.notified}" + f"exit={metrics.exit_code} notified={metrics.notified} " + f"match_notified={metrics.match_notified} " + f"match_count={metrics.match_notify_count}" ) if metrics.hard_fail and metrics.hard_fail_reason: print(f"error: {metrics.hard_fail_reason}", file=sys.stderr) @@ -242,6 +263,225 @@ def cmd_sync_sheets( return 0 +def _resolve_rank_sheet_sync( + args: argparse.Namespace, + settings: Settings, +) -> bool: + """Whether to push rankings to Google Sheets. + + ``--sync-sheets`` forces on; ``--no-sync-sheets`` forces off. + Default (neither flag): on when GOOGLE_SHEET_ID is configured. + """ + if getattr(args, "no_sync_sheets", False): + return False + if getattr(args, "sync_sheets", False): + return True + return bool((settings.google_sheet_id or "").strip()) + + +def _push_rankings_to_sheet( + *, + ranked_rows: list[dict[str, object]], + run_id: str, + model: str, + settings: Settings, + sheet_id: str | None, + rank_tab: str | None, +) -> tuple[int, str, str]: + """Sync rankings; return (count, spreadsheet_id, tab). Raises SheetsSyncError.""" + resolved_id = (sheet_id or settings.google_sheet_id or "").strip() + if not resolved_id: + raise SheetsSyncError("set --sheet-id or GOOGLE_SHEET_ID") + resolved_tab = ( + (rank_tab or settings.google_sheet_rank_tab or DEFAULT_RANK_TAB).strip() + or DEFAULT_RANK_TAB + ) + count = sync_rankings_to_sheet( + ranked_rows, + run_id=run_id, + model=model, + spreadsheet_id=resolved_id, + worksheet_title=resolved_tab, + service_account_file=settings.google_service_account_file, + service_account_json=settings.google_service_account_json, + ) + return count, resolved_id, resolved_tab + + +def cmd_interpret_rank( + args: argparse.Namespace, + settings: Settings | None = None, +) -> int: + """Grok rephrase + fit-rank stored opportunities (report only; no store mutate).""" + try: + settings = settings or get_settings() + except Exception as exc: + print(f"error: invalid settings: {exc}", file=sys.stderr) + return EXIT_USAGE + setup_logging(settings.log_level) + + backend = (settings.storage_backend or "sqlite").strip().lower() + if backend != "sqlite": + print( + "error: interpret-rank requires STORAGE_BACKEND=sqlite " + f"(current={backend!r})", + file=sys.stderr, + ) + return EXIT_FAILURE + + api_key = (settings.xai_api_key or "").strip() + if not api_key: + print( + "error: set XAI_API_KEY for interpret-rank " + '(and pip install -e ".[ai]")', + file=sys.stderr, + ) + return EXIT_USAGE + + objectives_path = Path(args.objectives) if args.objectives else settings.objectives_path + out_dir = ( + Path(args.out_dir) + if args.out_dir + else settings.data_dir / "rankings" + ) + limit = args.limit + if limit is not None and limit < 0: + return _usage_error("--limit must be >= 0") + min_score = args.min_score + if min_score is not None and min_score < 0: + return _usage_error("--min-score must be >= 0") + batch_size = args.batch_size if args.batch_size is not None else DEFAULT_BATCH_SIZE + if batch_size < 1: + return _usage_error("--batch-size must be >= 1") + + try: + store = build_store(settings) + except (ValueError, NotImplementedError) as exc: + print(f"error: store config: {exc}", file=sys.stderr) + return EXIT_FAILURE + if not isinstance(store, SqliteOpportunityStore): + print("error: interpret-rank requires SqliteOpportunityStore", file=sys.stderr) + return EXIT_FAILURE + + try: + store.health_check() + rows = store.list_rows() + except StoreError as exc: + print(f"error: store failed: {exc}", file=sys.stderr) + return EXIT_FAILURE + + status = (args.status or "").strip() or None + try: + result = run_interpret_rank( + rows, + objectives_path=objectives_path, + api_key=api_key, + base_url=settings.xai_base_url, + model=settings.xai_model, + out_dir=out_dir, + status=status, + limit=limit, + min_rule_score=min_score, + batch_size=batch_size, + ) + except InterpretRankError as exc: + print(f"error: interpret-rank failed: {exc}", file=sys.stderr) + return EXIT_FAILURE + + top = result.ranked[0] if result.ranked else None + print( + f"interpret-rank model={result.model} input={result.input_count} " + f"ranked={len(result.ranked)} run_id={result.run_id}" + ) + if top: + print( + f"top #{top.rank} fit={top.fit_score} rec={top.recommendation} " + f"id={top.opportunity_id} title={top.title[:80]}" + ) + if result.markdown_path: + print(f"Wrote {result.markdown_path}") + if result.json_path: + print(f"Wrote {result.json_path}") + + if _resolve_rank_sheet_sync(args, settings): + try: + count, sid, tab = _push_rankings_to_sheet( + ranked_rows=[r.as_dict() for r in result.ranked], + run_id=result.run_id, + model=result.model, + settings=settings, + sheet_id=getattr(args, "sheet_id", None), + rank_tab=getattr(args, "rank_tab", None), + ) + except SheetsSyncError as exc: + print(f"error: rank sheets sync failed: {exc}", file=sys.stderr) + return EXIT_FAILURE + print(f"Synced {count} ranking rows to sheet {sid} tab {tab!r}") + + # Teams capture ping for Grok fits at/above threshold (default 40). + skip_teams = bool(getattr(args, "no_teams", False)) + if settings.teams_match_notify_enabled and not skip_teams: + thr = int(settings.teams_match_score_threshold) + match_items = match_items_from_ranked(result.ranked, threshold=thr) + posted = notify_match_alerts( + settings.resolved_match_webhook_url(), + match_items, + threshold=thr, + source="interpret-rank", + run_url=settings.github_run_url, + max_items=int(settings.teams_match_max_items), + enabled=True, + ) + print( + f"teams match_notify posted={posted} " + f"above_threshold={len(match_items)} threshold={thr}" + ) + return 0 + + +def cmd_sync_rank_sheets( + args: argparse.Namespace, + settings: Settings | None = None, +) -> int: + """Push an existing interpret-rank JSON report to the Ranked Google Sheet tab.""" + try: + settings = settings or get_settings() + except Exception as exc: + print(f"error: invalid settings: {exc}", file=sys.stderr) + return EXIT_USAGE + setup_logging(settings.log_level) + + from_json = Path(args.from_json) if args.from_json else None + if from_json is None: + latest = find_latest_rankings_json(settings.data_dir / "rankings") + if latest is None: + print( + "error: no interpret-*.json under data/rankings; " + "run interpret-rank first or pass --from-json", + file=sys.stderr, + ) + return EXIT_USAGE + from_json = latest + print(f"Using latest rankings file {from_json}") + + try: + run_id, model, ranked = load_rankings_json(from_json) + count, sid, tab = _push_rankings_to_sheet( + ranked_rows=ranked, + run_id=run_id, + model=model, + settings=settings, + sheet_id=args.sheet_id, + rank_tab=args.rank_tab, + ) + except SheetsSyncError as exc: + print(f"error: rank sheets sync failed: {exc}", file=sys.stderr) + return EXIT_FAILURE + + print(f"Synced {count} ranking rows to sheet {sid} tab {tab!r}") + return 0 + + def build_parser() -> argparse.ArgumentParser: # Omit prog= so usage reflects the invoked name (module or console script). parser = argparse.ArgumentParser( @@ -336,6 +576,126 @@ def build_parser() -> argparse.ArgumentParser: help="Worksheet tab name to replace (default: Ingest / GOOGLE_SHEET_TAB)", ) + # interpret-rank — Grok rephrase + fit ranking (report only) + ir_p = sub.add_parser( + "interpret-rank", + help=( + "Grok-assisted plain-English rewrite + rank stored opportunities " + "against config/objectives.yaml (writes reports; does not mutate store)" + ), + ) + ir_p.add_argument( + "--objectives", + dest="objectives", + metavar="PATH", + default=None, + help="Company objectives YAML (default: OBJECTIVES_PATH / config/objectives.yaml)", + ) + ir_p.add_argument( + "--out-dir", + dest="out_dir", + metavar="DIR", + default=None, + help="Directory for JSON/Markdown reports (default: data/rankings)", + ) + ir_p.add_argument( + "--status", + dest="status", + metavar="STATUS", + default=None, + help="Only interpret rows with this Status (e.g. New)", + ) + ir_p.add_argument( + "--limit", + dest="limit", + type=int, + metavar="N", + default=None, + help="Max opportunities to send to Grok (after sort by rule score)", + ) + ir_p.add_argument( + "--min-score", + dest="min_score", + type=int, + metavar="N", + default=None, + help="Only rows with rule RelevanceScore >= N", + ) + ir_p.add_argument( + "--batch-size", + dest="batch_size", + type=int, + metavar="N", + default=None, + help=f"Opportunities per Grok call (default {DEFAULT_BATCH_SIZE})", + ) + ir_sync = ir_p.add_mutually_exclusive_group() + ir_sync.add_argument( + "--sync-sheets", + dest="sync_sheets", + action="store_true", + help=( + "Full-replace Google Sheet Ranked tab after ranking " + "(default when GOOGLE_SHEET_ID is set)" + ), + ) + ir_sync.add_argument( + "--no-sync-sheets", + dest="no_sync_sheets", + action="store_true", + help="Do not push rankings to Google Sheets even if GOOGLE_SHEET_ID is set", + ) + ir_p.add_argument( + "--sheet-id", + dest="sheet_id", + metavar="ID", + default=None, + help="Spreadsheet ID for rank sync (default: GOOGLE_SHEET_ID)", + ) + ir_p.add_argument( + "--rank-tab", + dest="rank_tab", + metavar="NAME", + default=None, + help="Rankings worksheet tab (default: Ranked / GOOGLE_SHEET_RANK_TAB)", + ) + ir_p.add_argument( + "--no-teams", + dest="no_teams", + action="store_true", + help="Skip Teams high-match capture ping after ranking", + ) + + # sync-rank-sheets — push existing ranking JSON without re-calling Grok + srs_p = sub.add_parser( + "sync-rank-sheets", + help=( + "Full-replace Google Sheet Ranked tab from an interpret-rank JSON " + "(latest under data/rankings or --from-json); does not call Grok" + ), + ) + srs_p.add_argument( + "--from-json", + dest="from_json", + metavar="PATH", + default=None, + help="Path to interpret-*.json (default: newest in data/rankings)", + ) + srs_p.add_argument( + "--sheet-id", + dest="sheet_id", + metavar="ID", + default=None, + help="Spreadsheet ID (default: GOOGLE_SHEET_ID env)", + ) + srs_p.add_argument( + "--rank-tab", + dest="rank_tab", + metavar="NAME", + default=None, + help=f"Worksheet tab to replace (default: {DEFAULT_RANK_TAB} / GOOGLE_SHEET_RANK_TAB)", + ) + return parser @@ -359,6 +719,10 @@ def main(argv: list[str] | None = None) -> int: return cmd_export_csv(args.out_path) if args.command == "sync-sheets": return cmd_sync_sheets(args.sheet_id, args.tab) + if args.command == "interpret-rank": + return cmd_interpret_rank(args) + if args.command == "sync-rank-sheets": + return cmd_sync_rank_sheets(args) raise AssertionError(f"unhandled command: {args.command!r}") diff --git a/src/opportunity_ingest/config.py b/src/opportunity_ingest/config.py index 1f06aca..c8cb92c 100644 --- a/src/opportunity_ingest/config.py +++ b/src/opportunity_ingest/config.py @@ -29,8 +29,25 @@ class Settings(BaseSettings): description="Explicit SQLite path; default {DATA_DIR}/contract_opportunities.db", ) keywords_path: Path = Field(default=Path("config/keywords.yaml")) + objectives_path: Path = Field( + default=Path("config/objectives.yaml"), + description="Company objectives for interpret-rank (Grok)", + ) state_path: Path = Field(default=Path("state/zero_new_streak.json")) log_level: str = Field(default="INFO") + # Grok / xAI (optional; required only for interpret-rank) + xai_api_key: str | None = Field( + default=None, + description="xAI API key (XAI_API_KEY); never commit", + ) + xai_base_url: str = Field( + default="https://api.x.ai/v1", + description="OpenAI-compatible xAI base URL", + ) + xai_model: str = Field( + default="grok-4.5", + description="Grok model id for interpret-rank", + ) # Soft create-attempt budget (KD-17). 0 = unlimited; negatives rejected. max_create: int | None = Field( default=50, @@ -47,7 +64,33 @@ class Settings(BaseSettings): http_timeout_seconds: float = Field(default=120.0) zero_new_streak_threshold: int = Field(default=3) partial_error_exit_threshold: int = Field(default=5) - teams_webhook_url: str | None = Field(default=None) + teams_webhook_url: str | None = Field( + default=None, + description="Teams Workflows webhook for ops alerts (and match fallback)", + ) + teams_match_webhook_url: str | None = Field( + default=None, + description=( + "Optional dedicated webhook for high-match opportunity pings; " + "falls back to TEAMS_WEBHOOK_URL when unset" + ), + ) + teams_match_notify_enabled: bool = Field( + default=True, + description="Post Teams card when new matches meet score threshold", + ) + teams_match_score_threshold: int = Field( + default=40, + description="Min score 0–100 to ping Teams (RelevanceScore or Grok fit)", + ) + teams_match_max_items: int = Field( + default=8, + description="Max opportunities listed on one match Adaptive Card", + ) + notify_config_path: Path = Field( + default=Path("config/notify.yaml"), + description="Optional YAML for Teams match notify defaults", + ) github_run_url: str | None = Field( default=None, description="GitHub Actions run URL for Teams cards (GITHUB_RUN_URL)", @@ -67,7 +110,11 @@ class Settings(BaseSettings): ) google_sheet_tab: str = Field( default="Ingest", - description="Worksheet tab name fully replaced on each sync", + description="Worksheet tab name fully replaced on each opportunity sync", + ) + google_sheet_rank_tab: str = Field( + default="Ranked", + description="Worksheet tab full-replaced by Grok interpret-rank sync", ) google_service_account_file: Path | None = Field( default=None, @@ -93,13 +140,91 @@ def _max_create_non_negative(cls, v: int | None) -> int | None: raise ValueError("max_create must be >= 0 (0=unlimited)") return v + @field_validator("teams_match_score_threshold") + @classmethod + def _threshold_0_100(cls, v: int) -> int: + n = int(v) + if n < 0 or n > 100: + raise ValueError("teams_match_score_threshold must be 0–100") + return n + + @field_validator("teams_match_max_items") + @classmethod + def _max_items_positive(cls, v: int) -> int: + n = int(v) + if n < 1: + raise ValueError("teams_match_max_items must be >= 1") + return n + def resolved_sqlite_path(self) -> Path: """Return SQLITE_PATH or ``{DATA_DIR}/contract_opportunities.db``.""" if self.sqlite_path is not None: return self.sqlite_path return self.data_dir / "contract_opportunities.db" + def resolved_match_webhook_url(self) -> str | None: + """Webhook for high-match pings (dedicated URL or ops fallback).""" + for raw in (self.teams_match_webhook_url, self.teams_webhook_url): + if raw and str(raw).strip(): + return str(raw).strip() + return None + + +def load_notify_yaml_overrides(path: Path | str | None = None) -> dict[str, object]: + """Load optional config/notify.yaml teams section (soft; empty if missing).""" + import yaml + + p = Path(path) if path is not None else Path("config/notify.yaml") + if not p.is_file(): + return {} + try: + raw = yaml.safe_load(p.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError): + return {} + if not isinstance(raw, dict): + return {} + teams = raw.get("teams") + if not isinstance(teams, dict): + return {} + out: dict[str, object] = {} + if "match_notify_enabled" in teams: + out["teams_match_notify_enabled"] = bool(teams["match_notify_enabled"]) + if "match_score_threshold" in teams: + try: + out["teams_match_score_threshold"] = int(teams["match_score_threshold"]) + except (TypeError, ValueError): + pass + if "max_items_per_message" in teams: + try: + out["teams_match_max_items"] = int(teams["max_items_per_message"]) + except (TypeError, ValueError): + pass + return out + def get_settings() -> Settings: - """Load settings from environment (and optional ``.env``).""" - return Settings() + """Load settings from environment / ``.env``, with soft notify.yaml overrides. + + Env vars always win for secrets and when ``TEAMS_MATCH_*`` is set. + ``config/notify.yaml`` fills threshold/enabled/max items when those env + keys are unset. + """ + import os + + base = Settings() + yaml_over = load_notify_yaml_overrides(base.notify_config_path) + if not yaml_over: + return base + + updates: dict[str, object] = {} + env_map = { + "teams_match_notify_enabled": "TEAMS_MATCH_NOTIFY_ENABLED", + "teams_match_score_threshold": "TEAMS_MATCH_SCORE_THRESHOLD", + "teams_match_max_items": "TEAMS_MATCH_MAX_ITEMS", + } + for field_name, env_name in env_map.items(): + if env_name not in os.environ and field_name in yaml_over: + updates[field_name] = yaml_over[field_name] + if not updates: + return base + return base.model_copy(update=updates) diff --git a/src/opportunity_ingest/interpret_rank.py b/src/opportunity_ingest/interpret_rank.py new file mode 100644 index 0000000..99f5144 --- /dev/null +++ b/src/opportunity_ingest/interpret_rank.py @@ -0,0 +1,671 @@ +"""Grok-assisted interpretation and ranking of stored opportunities. + +Post-ingest only. Reads SQLite (or row dicts), never updates contract_opportunities +Status / Notes / RelevanceScore. Writes ranked JSON + Markdown reports under +``data/rankings/`` (or a caller-chosen path). + +Requires optional extra: ``pip install -e ".[ai]"`` and ``XAI_API_KEY``. +""" + +from __future__ import annotations + +import html +import json +import logging +import re +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Mapping, Protocol, Sequence + +import yaml + +UTC = timezone.utc +logger = logging.getLogger(__name__) + +DEFAULT_BATCH_SIZE = 12 +DEFAULT_DESCRIPTION_CHARS = 1200 +RANKINGS_SUBDIR = "rankings" + +SYSTEM_PROMPT = """\ +You are a capture analyst for a Canadian professional-services / IT firm. +You rank public tender opportunities against the firm's stated objectives. + +Rules: +1. Use ONLY the opportunity text and company objectives provided. Do not invent + buyers, budgets, or requirements not supported by the text. +2. Interpret the buyer's likely objective in plain language. +3. Rephrase the opportunity so a busy capture lead can scan it in 30 seconds. +4. Score fit_score 0–100 against the company's objectives (not keyword weight alone). +5. Be candid: false keyword hits and off-scope work get low scores. +6. Prefer technology / managed services / advisory that matches objectives; + deprioritize construction, facilities, food design, pure creative ads, etc. +7. Return valid JSON only, matching the schema in the user message. +""" + + +class InterpretRankError(Exception): + """User-facing failure for interpret-rank (config, API, parse).""" + + +class GrokClient(Protocol): + """Minimal protocol so tests can inject a fake client.""" + + def complete_json(self, *, system: str, user: str) -> dict[str, Any]: + """Return parsed JSON object from the model.""" + ... + + +@dataclass(frozen=True, slots=True) +class CompanyObjectives: + """Loaded from config/objectives.yaml.""" + + version: int + company: dict[str, Any] + objectives: list[dict[str, Any]] + guidance: dict[str, Any] + raw: dict[str, Any] = field(repr=False) + + def prompt_block(self) -> str: + """Compact YAML-ish block for the model prompt.""" + payload = { + "company": self.company, + "objectives": self.objectives, + "guidance": self.guidance, + } + return yaml.safe_dump(payload, sort_keys=False, allow_unicode=True) + + +@dataclass(frozen=True, slots=True) +class RankedOpportunity: + """One interpreted + ranked opportunity (report row).""" + + rank: int + opportunity_id: str + title: str + buyer: str | None + link: str + closing_date: str | None + keywords_matched: str | None + rule_relevance_score: int | None + fit_score: int + recommendation: str + plain_english: str + interpreted_objective: str + matched_objectives: list[str] + why_it_fits: str + risks_or_mismatches: str + next_action: str + + def as_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True, slots=True) +class InterpretRankResult: + """Full run output (in-memory + paths when written).""" + + run_id: str + model: str + ranked: list[RankedOpportunity] + input_count: int + json_path: Path | None = None + markdown_path: Path | None = None + objectives_path: Path | None = None + + def as_dict(self) -> dict[str, Any]: + return { + "run_id": self.run_id, + "model": self.model, + "input_count": self.input_count, + "ranked_count": len(self.ranked), + "objectives_path": str(self.objectives_path) if self.objectives_path else None, + "json_path": str(self.json_path) if self.json_path else None, + "markdown_path": str(self.markdown_path) if self.markdown_path else None, + "ranked": [r.as_dict() for r in self.ranked], + } + + +def load_objectives(path: Path | str) -> CompanyObjectives: + """Load and validate company objectives YAML.""" + p = Path(path) + if not p.is_file(): + raise InterpretRankError(f"objectives file not found: {p}") + try: + raw = yaml.safe_load(p.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError) as exc: + raise InterpretRankError(f"failed to load objectives from {p}: {exc}") from exc + if not isinstance(raw, dict): + raise InterpretRankError(f"objectives root must be a mapping: {p}") + company = raw.get("company") or {} + objectives = raw.get("objectives") or [] + guidance = raw.get("guidance") or {} + if not isinstance(company, dict): + raise InterpretRankError("objectives.company must be a mapping") + if not isinstance(objectives, list) or not objectives: + raise InterpretRankError("objectives.objectives must be a non-empty list") + if not isinstance(guidance, dict): + raise InterpretRankError("objectives.guidance must be a mapping") + version = int(raw.get("version") or 1) + return CompanyObjectives( + version=version, + company=company, + objectives=objectives, + guidance=guidance, + raw=raw, + ) + + +def _clean_text(value: object | None, max_chars: int | None = None) -> str: + if value is None: + return "" + text = html.unescape(str(value)).strip() + # Collapse whitespace / HTML-ish noise lightly. + text = re.sub(r"\s+", " ", text) + if max_chars is not None and len(text) > max_chars: + text = text[: max_chars - 1].rstrip() + "…" + return text + + +def prepare_opportunity_payload( + rows: Sequence[Mapping[str, Any]], + *, + description_chars: int = DEFAULT_DESCRIPTION_CHARS, +) -> list[dict[str, Any]]: + """Normalize store rows into compact model inputs.""" + out: list[dict[str, Any]] = [] + for row in rows: + oid = _clean_text(row.get("OpportunityID")) + title = _clean_text(row.get("Title")) + if not oid or not title: + continue + out.append( + { + "opportunity_id": oid, + "title": title, + "buyer": _clean_text(row.get("Buyer")) or None, + "category": _clean_text(row.get("Category")) or None, + "description": _clean_text(row.get("Description"), description_chars) or None, + "keywords_matched": _clean_text(row.get("KeywordsMatched")) or None, + "rule_relevance_score": _as_int(row.get("RelevanceScore")), + "status": _clean_text(row.get("Status")) or None, + "published_date": _clean_text(row.get("PublishedDate")) or None, + "closing_date": _clean_text(row.get("ClosingDate")) or None, + "link": _clean_text(row.get("Link")) or None, + } + ) + return out + + +def _as_int(value: object | None) -> int | None: + if value is None or value == "": + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def filter_rows( + rows: Sequence[Mapping[str, Any]], + *, + status: str | None = None, + limit: int | None = None, + min_rule_score: int | None = None, +) -> list[dict[str, Any]]: + """Filter/sort opportunity rows before AI ranking. + + Sort seed: RelevanceScore DESC, then OpportunityID for stability. + """ + items = [dict(r) for r in rows] + if status: + status_l = status.strip().lower() + items = [ + r + for r in items + if _clean_text(r.get("Status")).lower() == status_l + ] + if min_rule_score is not None: + items = [ + r + for r in items + if (_as_int(r.get("RelevanceScore")) or 0) >= min_rule_score + ] + + def sort_key(r: Mapping[str, Any]) -> tuple[int, str]: + score = _as_int(r.get("RelevanceScore")) or 0 + oid = _clean_text(r.get("OpportunityID")) + return (-score, oid) + + items.sort(key=sort_key) + if limit is not None and limit >= 0: + items = items[:limit] + return items + + +def build_user_prompt( + objectives: CompanyObjectives, + batch: Sequence[Mapping[str, Any]], +) -> str: + """User message asking for structured JSON ranking of one batch.""" + schema_hint = { + "rankings": [ + { + "opportunity_id": "string (must match input)", + "fit_score": "integer 0-100", + "recommendation": "pursue | watch | pass", + "plain_english": "2-4 sentence human rewrite", + "interpreted_objective": "what the buyer is trying to achieve", + "matched_objectives": ["objective id from company list"], + "why_it_fits": "1-3 sentences", + "risks_or_mismatches": "1-3 sentences", + "next_action": "concrete next step for capture lead", + } + ] + } + return ( + "## Company objectives (frame of reference)\n" + f"{objectives.prompt_block()}\n" + "## Opportunities to interpret and rank\n" + f"{json.dumps(list(batch), ensure_ascii=False, indent=2)}\n\n" + "## Required JSON schema\n" + f"{json.dumps(schema_hint, indent=2)}\n\n" + "Return a single JSON object with key `rankings` covering EVERY opportunity " + "in the batch (same opportunity_id values). Sort rankings by fit_score " + "descending within this batch." + ) + + +def _extract_json_object(text: str) -> dict[str, Any]: + """Parse JSON object from model text (tolerates fenced markdown).""" + raw = text.strip() + if not raw: + raise InterpretRankError("empty model response") + # Strip ```json ... ``` fences if present. + fence = re.search(r"```(?:json)?\s*([\s\S]*?)\s*```", raw, re.IGNORECASE) + if fence: + raw = fence.group(1).strip() + try: + data = json.loads(raw) + except json.JSONDecodeError: + # Last resort: first { ... last } + start = raw.find("{") + end = raw.rfind("}") + if start < 0 or end <= start: + raise InterpretRankError( + f"model response is not JSON (first 200 chars): {text[:200]!r}" + ) + try: + data = json.loads(raw[start : end + 1]) + except json.JSONDecodeError as exc: + raise InterpretRankError(f"failed to parse model JSON: {exc}") from exc + if not isinstance(data, dict): + raise InterpretRankError("model JSON root must be an object") + return data + + +class OpenAIGrokClient: + """xAI Grok client via OpenAI-compatible Chat Completions API.""" + + def __init__( + self, + *, + api_key: str, + base_url: str = "https://api.x.ai/v1", + model: str = "grok-4.5", + timeout: float = 120.0, + ) -> None: + try: + from openai import OpenAI + except ImportError as exc: + raise InterpretRankError( + 'openai package not installed; run: pip install -e ".[ai]"' + ) from exc + if not api_key or not api_key.strip(): + raise InterpretRankError("XAI_API_KEY is required for interpret-rank") + self.model = model + self._client = OpenAI(api_key=api_key.strip(), base_url=base_url, timeout=timeout) + + def complete_json(self, *, system: str, user: str) -> dict[str, Any]: + try: + resp = self._client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + temperature=0.2, + response_format={"type": "json_object"}, + ) + except Exception as exc: # noqa: BLE001 — surface any SDK/HTTP error + raise InterpretRankError(f"Grok API call failed: {exc}") from exc + try: + content = resp.choices[0].message.content or "" + except (AttributeError, IndexError, TypeError) as exc: + raise InterpretRankError(f"unexpected Grok response shape: {exc}") from exc + return _extract_json_object(content) + + +def _clamp_score(value: object) -> int: + try: + n = int(value) # type: ignore[arg-type] + except (TypeError, ValueError): + return 0 + return max(0, min(100, n)) + + +def _normalize_recommendation(value: object) -> str: + s = _clean_text(value).lower() + if s in {"pursue", "bid", "go", "relevant"}: + return "pursue" + if s in {"watch", "monitor", "maybe", "review"}: + return "watch" + if s in {"pass", "skip", "discard", "no"}: + return "pass" + return s or "watch" + + +def merge_model_rankings( + inputs: Sequence[Mapping[str, Any]], + model_payload: Mapping[str, Any], +) -> list[RankedOpportunity]: + """Join model rankings back to inputs; stable global rank by fit_score.""" + by_id = {str(item["opportunity_id"]): item for item in inputs} + raw_list = model_payload.get("rankings") + if not isinstance(raw_list, list): + raise InterpretRankError("model JSON missing rankings[] array") + + seen: set[str] = set() + partial: list[dict[str, Any]] = [] + for entry in raw_list: + if not isinstance(entry, dict): + continue + oid = _clean_text(entry.get("opportunity_id")) + if not oid or oid not in by_id or oid in seen: + continue + seen.add(oid) + src = by_id[oid] + matched = entry.get("matched_objectives") or [] + if isinstance(matched, str): + matched_list = [matched] + elif isinstance(matched, list): + matched_list = [str(x) for x in matched if x is not None] + else: + matched_list = [] + partial.append( + { + "opportunity_id": oid, + "title": src.get("title") or "", + "buyer": src.get("buyer"), + "link": src.get("link") or "", + "closing_date": src.get("closing_date"), + "keywords_matched": src.get("keywords_matched"), + "rule_relevance_score": src.get("rule_relevance_score"), + "fit_score": _clamp_score(entry.get("fit_score")), + "recommendation": _normalize_recommendation(entry.get("recommendation")), + "plain_english": _clean_text(entry.get("plain_english")) + or _clean_text(src.get("description"), 400) + or (src.get("title") or ""), + "interpreted_objective": _clean_text(entry.get("interpreted_objective")) + or "Not stated clearly in the notice text.", + "matched_objectives": matched_list, + "why_it_fits": _clean_text(entry.get("why_it_fits")) or "", + "risks_or_mismatches": _clean_text(entry.get("risks_or_mismatches")) or "", + "next_action": _clean_text(entry.get("next_action")) + or "Open the notice Link and review eligibility.", + } + ) + + # Any inputs the model skipped get a low default so the report is complete. + for oid, src in by_id.items(): + if oid in seen: + continue + logger.warning("model omitted opportunity_id=%s; applying default pass", oid) + partial.append( + { + "opportunity_id": oid, + "title": src.get("title") or "", + "buyer": src.get("buyer"), + "link": src.get("link") or "", + "closing_date": src.get("closing_date"), + "keywords_matched": src.get("keywords_matched"), + "rule_relevance_score": src.get("rule_relevance_score"), + "fit_score": 0, + "recommendation": "watch", + "plain_english": _clean_text(src.get("description"), 400) + or (src.get("title") or ""), + "interpreted_objective": "Model did not return an interpretation.", + "matched_objectives": [], + "why_it_fits": "", + "risks_or_mismatches": "Missing model output for this row.", + "next_action": "Re-run interpret-rank or review manually via Link.", + } + ) + + partial.sort( + key=lambda r: (-int(r["fit_score"]), str(r["opportunity_id"])), + ) + ranked: list[RankedOpportunity] = [] + for i, row in enumerate(partial, start=1): + ranked.append( + RankedOpportunity( + rank=i, + opportunity_id=str(row["opportunity_id"]), + title=str(row["title"]), + buyer=row.get("buyer"), # type: ignore[arg-type] + link=str(row.get("link") or ""), + closing_date=row.get("closing_date"), # type: ignore[arg-type] + keywords_matched=row.get("keywords_matched"), # type: ignore[arg-type] + rule_relevance_score=row.get("rule_relevance_score"), # type: ignore[arg-type] + fit_score=int(row["fit_score"]), + recommendation=str(row["recommendation"]), + plain_english=str(row["plain_english"]), + interpreted_objective=str(row["interpreted_objective"]), + matched_objectives=list(row["matched_objectives"]), + why_it_fits=str(row["why_it_fits"]), + risks_or_mismatches=str(row["risks_or_mismatches"]), + next_action=str(row["next_action"]), + ) + ) + return ranked + + +def _chunk(items: Sequence[Any], size: int) -> list[list[Any]]: + if size <= 0: + return [list(items)] + return [list(items[i : i + size]) for i in range(0, len(items), size)] + + +def rank_opportunities( + rows: Sequence[Mapping[str, Any]], + objectives: CompanyObjectives, + client: GrokClient, + *, + model_name: str = "grok-4.5", + batch_size: int = DEFAULT_BATCH_SIZE, + description_chars: int = DEFAULT_DESCRIPTION_CHARS, + run_id: str | None = None, +) -> InterpretRankResult: + """Call Grok in batches and return a globally ranked result (in-memory).""" + prepared = prepare_opportunity_payload(rows, description_chars=description_chars) + if not prepared: + rid = run_id or datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + return InterpretRankResult( + run_id=rid, + model=model_name, + ranked=[], + input_count=0, + ) + + all_partial: list[RankedOpportunity] = [] + for batch in _chunk(prepared, batch_size): + user = build_user_prompt(objectives, batch) + payload = client.complete_json(system=SYSTEM_PROMPT, user=user) + batch_ranked = merge_model_rankings(batch, payload) + all_partial.extend(batch_ranked) + + # Re-rank across batches. + all_partial.sort(key=lambda r: (-r.fit_score, r.opportunity_id)) + final: list[RankedOpportunity] = [] + for i, r in enumerate(all_partial, start=1): + final.append( + RankedOpportunity( + rank=i, + opportunity_id=r.opportunity_id, + title=r.title, + buyer=r.buyer, + link=r.link, + closing_date=r.closing_date, + keywords_matched=r.keywords_matched, + rule_relevance_score=r.rule_relevance_score, + fit_score=r.fit_score, + recommendation=r.recommendation, + plain_english=r.plain_english, + interpreted_objective=r.interpreted_objective, + matched_objectives=list(r.matched_objectives), + why_it_fits=r.why_it_fits, + risks_or_mismatches=r.risks_or_mismatches, + next_action=r.next_action, + ) + ) + + rid = run_id or datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + return InterpretRankResult( + run_id=rid, + model=model_name, + ranked=final, + input_count=len(prepared), + ) + + +def render_markdown(result: InterpretRankResult, objectives: CompanyObjectives) -> str: + """Human-readable ranked brief for capture leads.""" + company_name = _clean_text((objectives.company or {}).get("name")) or "Company" + lines: list[str] = [ + f"# Opportunity ranking brief — {result.run_id}", + "", + f"**Frame of reference:** {company_name} ", + f"**Model:** `{result.model}` ", + f"**Opportunities interpreted:** {result.input_count} ", + "", + "Scores are **Grok fit** against `config/objectives.yaml`, not the rule-based " + "`RelevanceScore` from ingest. Ingest rows are **not** modified.", + "", + "---", + "", + ] + if not result.ranked: + lines.append("_No opportunities to rank._") + lines.append("") + return "\n".join(lines) + + for r in result.ranked: + buyer = r.buyer or "Unknown buyer" + rule = r.rule_relevance_score if r.rule_relevance_score is not None else "—" + matched = ", ".join(r.matched_objectives) if r.matched_objectives else "—" + lines.extend( + [ + f"## #{r.rank} — {r.title}", + "", + f"- **Fit score:** {r.fit_score}/100 ", + f"- **Recommendation:** `{r.recommendation}` ", + f"- **Buyer:** {buyer} ", + f"- **OpportunityID:** `{r.opportunity_id}` ", + f"- **Closing:** {r.closing_date or '—'} ", + f"- **Keywords (ingest):** {r.keywords_matched or '—'} ", + f"- **Rule RelevanceScore:** {rule} ", + f"- **Matched objectives:** {matched} ", + f"- **Link:** {r.link or '—'} ", + "", + "### Plain English", + r.plain_english or "—", + "", + "### Interpreted buyer objective", + r.interpreted_objective or "—", + "", + "### Why it fits", + r.why_it_fits or "—", + "", + "### Risks / mismatches", + r.risks_or_mismatches or "—", + "", + "### Next action", + r.next_action or "—", + "", + "---", + "", + ] + ) + return "\n".join(lines) + + +def write_reports( + result: InterpretRankResult, + objectives: CompanyObjectives, + *, + out_dir: Path, + objectives_path: Path | None = None, +) -> InterpretRankResult: + """Write JSON + Markdown under out_dir; return result with paths set.""" + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + json_path = out_dir / f"interpret-{result.run_id}.json" + md_path = out_dir / f"interpret-{result.run_id}.md" + payload = result.as_dict() + payload["company"] = objectives.company + payload["objectives"] = objectives.objectives + json_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + md_path.write_text(render_markdown(result, objectives), encoding="utf-8") + return InterpretRankResult( + run_id=result.run_id, + model=result.model, + ranked=result.ranked, + input_count=result.input_count, + json_path=json_path, + markdown_path=md_path, + objectives_path=objectives_path, + ) + + +def run_interpret_rank( + rows: Sequence[Mapping[str, Any]], + *, + objectives_path: Path, + api_key: str, + base_url: str = "https://api.x.ai/v1", + model: str = "grok-4.5", + out_dir: Path, + status: str | None = None, + limit: int | None = None, + min_rule_score: int | None = None, + batch_size: int = DEFAULT_BATCH_SIZE, + client_factory: Callable[..., GrokClient] | None = None, +) -> InterpretRankResult: + """End-to-end: filter rows → Grok rank → write reports.""" + objectives = load_objectives(objectives_path) + filtered = filter_rows( + rows, + status=status, + limit=limit, + min_rule_score=min_rule_score, + ) + if client_factory is not None: + client = client_factory(api_key=api_key, base_url=base_url, model=model) + else: + client = OpenAIGrokClient(api_key=api_key, base_url=base_url, model=model) + result = rank_opportunities( + filtered, + objectives, + client, + model_name=model, + batch_size=batch_size, + ) + return write_reports( + result, + objectives, + out_dir=out_dir, + objectives_path=objectives_path, + ) diff --git a/src/opportunity_ingest/notify.py b/src/opportunity_ingest/notify.py index 2a97049..4c7713e 100644 --- a/src/opportunity_ingest/notify.py +++ b/src/opportunity_ingest/notify.py @@ -1,8 +1,15 @@ """Teams Workflows webhook notifications (Adaptive Card). -Python owns primary notify for hard fail, high partial errors, and zero-new -streak. On successful Teams POST, sets GitHub Actions step output -``notified=true`` so the workflow backup step can skip double-notify +Two pipelines share this module: + +1. **Ops alerts** — hard fail, partial errors, zero-new streak + (``notify_ingest_alert`` / ``TEAMS_WEBHOOK_URL``). +2. **Match alerts** — new opportunities or Grok ranks at/above score threshold + (``notify_match_alerts`` / ``TEAMS_MATCH_WEBHOOK_URL`` or ops fallback). + Call-to-action cards include truncated summaries + OpenUrl actions. + +On successful ops notify POST, sets GitHub Actions step output ``notified=true`` +so the workflow backup step can skip double-notify (``failure() && steps.ingest.outputs.notified != 'true'``). **GITHUB_OUTPUT ownership (KD-18):** When ``GITHUB_OUTPUT`` is set, notify @@ -12,6 +19,8 @@ still double-notify on job failure — operators should treat GITHUB_OUTPUT I/O errors as critical. When ``GITHUB_OUTPUT`` is unset (local runs), POST success alone is enough. + +Match alerts do **not** set ``notified=true`` (ops ownership handoff only). """ from __future__ import annotations @@ -19,19 +28,41 @@ import json import logging import os +import re import urllib.error import urllib.request +from dataclasses import dataclass from typing import Any, Mapping, Sequence logger = logging.getLogger(__name__) DEFAULT_TIMEOUT_SECONDS = 30.0 +DEFAULT_MATCH_THRESHOLD = 40 +DEFAULT_MATCH_MAX_ITEMS = 8 +DEFAULT_TITLE_MAX = 120 +DEFAULT_SUMMARY_MAX = 280 class NotifyError(Exception): """Teams webhook or GITHUB_OUTPUT handoff failure (non-fatal to exit matrix).""" +@dataclass(frozen=True, slots=True) +class MatchAlertItem: + """One high-match opportunity for a Teams capture ping.""" + + title: str + opportunity_id: str + link: str + score: int + score_kind: str # e.g. RelevanceScore | GrokFit + buyer: str | None = None + keywords: str | None = None + summary: str | None = None + recommendation: str | None = None + closing_date: str | None = None + + def set_github_output_notified(notified: bool = True) -> bool: """Append ``notified=true|false`` to ``GITHUB_OUTPUT`` when running in Actions. @@ -67,11 +98,24 @@ def set_github_output_notified(notified: bool = True) -> bool: ) +def _truncate(text: str | None, max_chars: int) -> str: + if not text: + return "" + s = re.sub(r"\s+", " ", str(text)).strip() + if len(s) <= max_chars: + return s + if max_chars <= 1: + return s[:max_chars] + return s[: max_chars - 1].rstrip() + "…" + + def build_adaptive_card_payload( *, title: str, facts: Sequence[Mapping[str, str]] | None = None, body_text: str | None = None, + actions: Sequence[Mapping[str, Any]] | None = None, + extra_body: Sequence[Mapping[str, Any]] | None = None, ) -> dict[str, Any]: """Build a Teams Workflows message with an Adaptive Card attachment.""" body: list[dict[str, Any]] = [ @@ -85,6 +129,8 @@ def build_adaptive_card_payload( ] if body_text: body.append({"type": "TextBlock", "text": body_text, "wrap": True}) + if extra_body: + body.extend(dict(b) for b in extra_body) if facts: body.append( { @@ -95,23 +141,137 @@ def build_adaptive_card_payload( ], } ) + card: dict[str, Any] = { + "$schema": "http://adaptivecards.io/schemas/adaptive-card.json", + "type": "AdaptiveCard", + "version": "1.4", + "body": body, + } + if actions: + card["actions"] = [dict(a) for a in actions] return { "type": "message", "attachments": [ { "contentType": "application/vnd.microsoft.card.adaptive", "contentUrl": None, - "content": { - "$schema": "http://adaptivecards.io/schemas/adaptive-card.json", - "type": "AdaptiveCard", - "version": "1.4", - "body": body, - }, + "content": card, } ], } +def filter_match_items( + items: Sequence[MatchAlertItem], + *, + threshold: int = DEFAULT_MATCH_THRESHOLD, +) -> list[MatchAlertItem]: + """Return items with score >= threshold, highest score first.""" + thr = max(0, min(100, int(threshold))) + filtered = [i for i in items if int(i.score) >= thr and (i.link or "").strip()] + filtered.sort(key=lambda i: (-int(i.score), i.opportunity_id)) + return filtered + + +def build_match_alert_payload( + items: Sequence[MatchAlertItem], + *, + threshold: int = DEFAULT_MATCH_THRESHOLD, + source: str = "ingest", + run_url: str | None = None, + max_items: int = DEFAULT_MATCH_MAX_ITEMS, + title_max_chars: int = DEFAULT_TITLE_MAX, + summary_max_chars: int = DEFAULT_SUMMARY_MAX, + card_title: str | None = None, + cta_label: str = "Open notice", +) -> dict[str, Any]: + """Adaptive Card: high-match opportunities with summary + OpenUrl CTAs. + + ``source`` is ``ingest`` (rule score) or ``interpret-rank`` (Grok fit). + """ + matched = filter_match_items(items, threshold=threshold) + if not matched: + raise NotifyError("no match items at or above threshold") + + cap = max(1, int(max_items)) + shown = matched[:cap] + hidden = len(matched) - len(shown) + + if card_title: + title = card_title + elif source == "interpret-rank": + title = f"Grok fit matches (≥{threshold})" + else: + title = f"New opportunity matches (≥{threshold})" + + intro = ( + f"**{len(matched)}** high-match opportunit" + f"{'y' if len(matched) == 1 else 'ies'} " + f"(threshold **{threshold}/100**, source `{source}`). " + "Review links and act — Status stays human-owned in the store." + ) + if hidden > 0: + intro += f" Showing top {len(shown)}; **{hidden}** more not listed." + + extra_body: list[dict[str, Any]] = [] + actions: list[dict[str, Any]] = [] + for idx, item in enumerate(shown, start=1): + t = _truncate(item.title, title_max_chars) or item.opportunity_id + summary = _truncate(item.summary, summary_max_chars) + buyer = _truncate(item.buyer, 80) or "—" + kws = _truncate(item.keywords, 100) or "—" + rec = (item.recommendation or "").strip() + lines = [ + f"**#{idx} · {item.score}/100** ({item.score_kind})" + + (f" · `{rec}`" if rec else ""), + t, + f"Buyer: {buyer} · ID: `{item.opportunity_id}`", + f"Keywords: {kws}", + ] + if item.closing_date: + lines.append(f"Closing: {item.closing_date}") + if summary: + lines.append(summary) + extra_body.append( + { + "type": "TextBlock", + "text": "\n\n".join(lines), + "wrap": True, + "separator": True, + } + ) + link = (item.link or "").strip() + if link.startswith(("http://", "https://")): + # Teams shows a limited number of card actions; label uniquely. + label = f"{cta_label} #{idx}" + if len(label) > 40: + label = f"Open #{idx}" + actions.append( + { + "type": "Action.OpenUrl", + "title": label, + "url": link, + } + ) + + facts: list[dict[str, str]] = [ + {"title": "Threshold", "value": str(threshold)}, + {"title": "Matches", "value": str(len(matched))}, + {"title": "Source", "value": source}, + ] + if run_url: + facts.append({"title": "Run", "value": run_url}) + + # Adaptive Cards: too many actions can be dropped by Teams; keep first 6. + return build_adaptive_card_payload( + title=title, + body_text=intro, + extra_body=extra_body, + facts=facts, + actions=actions[:6] if actions else None, + ) + + def build_ingest_alert_payload( *, reason: str, @@ -224,3 +384,167 @@ def notify_ingest_alert( ) return False return True + + +def notify_match_alerts( + webhook_url: str | None, + items: Sequence[MatchAlertItem], + *, + threshold: int = DEFAULT_MATCH_THRESHOLD, + source: str = "ingest", + run_url: str | None = None, + max_items: int = DEFAULT_MATCH_MAX_ITEMS, + enabled: bool = True, + timeout: float = DEFAULT_TIMEOUT_SECONDS, + title_max_chars: int = DEFAULT_TITLE_MAX, + summary_max_chars: int = DEFAULT_SUMMARY_MAX, + card_title: str | None = None, + cta_label: str = "Open notice", +) -> bool: + """Post a capture-channel Adaptive Card for high-match opportunities. + + Returns True when a card was POSTed successfully. + Returns False when disabled, no webhook, no items above threshold, or POST fails. + Does **not** write GITHUB_OUTPUT (ops double-notify handoff is separate). + """ + if not enabled: + logger.info("Match notify disabled; skipping (source=%s)", source) + return False + if not webhook_url or not str(webhook_url).strip(): + logger.warning( + "Match notify skipped (source=%s): no TEAMS_MATCH_WEBHOOK_URL / " + "TEAMS_WEBHOOK_URL", + source, + ) + return False + + matched = filter_match_items(items, threshold=threshold) + if not matched: + logger.info( + "Match notify: no items with score>=%s (source=%s, candidates=%s)", + threshold, + source, + len(items), + ) + return False + + try: + payload = build_match_alert_payload( + matched, + threshold=threshold, + source=source, + run_url=run_url, + max_items=max_items, + title_max_chars=title_max_chars, + summary_max_chars=summary_max_chars, + card_title=card_title, + cta_label=cta_label, + ) + post_teams_webhook(str(webhook_url), payload, timeout=timeout) + except NotifyError as exc: + logger.error("Teams match notify failed (source=%s): %s", source, exc) + return False + + logger.info( + "Teams match notify sent (source=%s, matches=%s, threshold=%s)", + source, + len(matched), + threshold, + ) + return True + + +def match_items_from_opportunity_fields( + fields_list: Sequence[Any], + *, + threshold: int = DEFAULT_MATCH_THRESHOLD, +) -> list[MatchAlertItem]: + """Build match items from ``OpportunityFields`` (or duck-typed) for ingest.""" + items: list[MatchAlertItem] = [] + for f in fields_list: + try: + score = int(getattr(f, "RelevanceScore", 0) or 0) + except (TypeError, ValueError): + score = 0 + if score < threshold: + continue + link = str(getattr(f, "Link", "") or "").strip() + if not link: + continue + desc = getattr(f, "Description", None) + items.append( + MatchAlertItem( + title=str(getattr(f, "Title", "") or ""), + opportunity_id=str(getattr(f, "OpportunityID", "") or ""), + link=link, + score=score, + score_kind="RelevanceScore", + buyer=getattr(f, "Buyer", None), + keywords=getattr(f, "KeywordsMatched", None), + summary=str(desc) if desc else None, + recommendation=None, + closing_date=( + str(getattr(f, "ClosingDate")) + if getattr(f, "ClosingDate", None) is not None + else None + ), + ) + ) + return items + + +def match_items_from_ranked( + ranked: Sequence[Any], + *, + threshold: int = DEFAULT_MATCH_THRESHOLD, +) -> list[MatchAlertItem]: + """Build match items from ``RankedOpportunity`` (or dicts) for interpret-rank.""" + items: list[MatchAlertItem] = [] + for r in ranked: + if isinstance(r, Mapping): + score = int(r.get("fit_score") or 0) + link = str(r.get("link") or "").strip() + if score < threshold or not link: + continue + items.append( + MatchAlertItem( + title=str(r.get("title") or ""), + opportunity_id=str(r.get("opportunity_id") or ""), + link=link, + score=score, + score_kind="GrokFit", + buyer=r.get("buyer"), # type: ignore[arg-type] + keywords=r.get("keywords_matched"), # type: ignore[arg-type] + summary=str(r.get("plain_english") or r.get("interpreted_objective") or ""), + recommendation=str(r.get("recommendation") or "") or None, + closing_date=str(r.get("closing_date") or "") or None, + ) + ) + continue + try: + score = int(getattr(r, "fit_score", 0) or 0) + except (TypeError, ValueError): + score = 0 + link = str(getattr(r, "link", "") or "").strip() + if score < threshold or not link: + continue + items.append( + MatchAlertItem( + title=str(getattr(r, "title", "") or ""), + opportunity_id=str(getattr(r, "opportunity_id", "") or ""), + link=link, + score=score, + score_kind="GrokFit", + buyer=getattr(r, "buyer", None), + keywords=getattr(r, "keywords_matched", None), + summary=str( + getattr(r, "plain_english", None) + or getattr(r, "interpreted_objective", None) + or "" + ) + or None, + recommendation=getattr(r, "recommendation", None), + closing_date=getattr(r, "closing_date", None), + ) + ) + return items diff --git a/src/opportunity_ingest/pipeline.py b/src/opportunity_ingest/pipeline.py index 3a9a014..a81c0c3 100644 --- a/src/opportunity_ingest/pipeline.py +++ b/src/opportunity_ingest/pipeline.py @@ -27,7 +27,11 @@ ) from opportunity_ingest.map_fields import MapError, map_to_opportunity_fields from opportunity_ingest.models import ExistingKeys, OpportunityFields -from opportunity_ingest.notify import notify_ingest_alert +from opportunity_ingest.notify import ( + match_items_from_opportunity_fields, + notify_ingest_alert, + notify_match_alerts, +) from opportunity_ingest.parse import ParseError, parse_csv_file, parse_csv_text from opportunity_ingest.score import compute_score from opportunity_ingest.state import ( @@ -77,6 +81,8 @@ class RunMetrics: exit_code: int = EXIT_OK notified: bool = False notify_reason: str | None = None + match_notified: bool = False + match_notify_count: int = 0 csv_source: str | None = None extra: dict[str, Any] = field(default_factory=dict) @@ -201,14 +207,19 @@ def _process_candidates( keys: ExistingKeys, budget: AttemptBudget, write: bool, -) -> dict[str, int]: - """Dedupe + create loop. On dry-run, counts would-create without store writes.""" +) -> dict[str, Any]: + """Dedupe + create loop. On dry-run, counts would-create without store writes. + + Write path also returns ``created``: list of successfully created OpportunityFields + (for Teams high-match pings). + """ added = 0 errors = 0 skipped_dup = 0 skipped_max = 0 would_create = 0 attempts = 0 + created: list[OpportunityFields] = [] for fields in candidates: if is_duplicate(fields.OpportunityID, fields.Link, keys): @@ -264,6 +275,7 @@ def _process_candidates( else: register_created(keys, fields.OpportunityID, fields.Link) added += 1 + created.append(fields) logger.info( "Created opportunity %s (score=%s)", fields.OpportunityID, @@ -277,6 +289,7 @@ def _process_candidates( "skipped_max_create": skipped_max, "would_create": would_create, "attempts": attempts, + "created": created, } @@ -395,6 +408,7 @@ def run_pipeline( metrics.skipped_max_create_count = stats["skipped_max_create"] metrics.would_create_count = stats["would_create"] metrics.create_attempts = stats["attempts"] + created_fields: list[OpportunityFields] = list(stats.get("created") or []) # Streak: write runs only (UTC calendar-day semantics). streak_reached = False @@ -451,9 +465,28 @@ def run_pipeline( extra_facts=extra, ) + # High-match capture pings (new creates only; dry-run never posts). + if write and created_fields and settings.teams_match_notify_enabled: + thr = int(settings.teams_match_score_threshold) + match_items = match_items_from_opportunity_fields( + created_fields, threshold=thr + ) + metrics.match_notify_count = len(match_items) + if match_items: + metrics.match_notified = notify_match_alerts( + settings.resolved_match_webhook_url(), + match_items, + threshold=thr, + source="ingest", + run_url=settings.github_run_url, + max_items=int(settings.teams_match_max_items), + enabled=True, + ) + logger.info( "Run complete: write=%s added=%s errors=%s would_create=%s " - "skipped_dup=%s skipped_max=%s exit=%s notified=%s", + "skipped_dup=%s skipped_max=%s exit=%s notified=%s " + "match_notified=%s match_count=%s", write, metrics.added_count, metrics.error_count, @@ -462,6 +495,8 @@ def run_pipeline( metrics.skipped_max_create_count, metrics.exit_code, metrics.notified, + metrics.match_notified, + metrics.match_notify_count, ) except PipelineHardFail as exc: diff --git a/src/opportunity_ingest/sheets_sync.py b/src/opportunity_ingest/sheets_sync.py index aff8ac7..6e54763 100644 --- a/src/opportunity_ingest/sheets_sync.py +++ b/src/opportunity_ingest/sheets_sync.py @@ -1,6 +1,9 @@ -"""Sync SQLite opportunities to Google Sheets (service account + Sheets API). +"""Sync SQLite opportunities and Grok rankings to Google Sheets. + +Free path: full replace of worksheet tabs via service account + Sheets API. +- ``Ingest`` (default): opportunity rows from SQLite +- ``Ranked`` (default for interpret-rank): Grok fit-rank report (separate tab) -Free path: full replace of one worksheet tab (default ``Ingest``). Requires optional deps: ``pip install gspread google-auth`` (or ``pip install -e ".[sheets]"``). """ @@ -11,13 +14,36 @@ import logging import os from pathlib import Path -from typing import Any +from typing import Any, Mapping, Sequence from opportunity_ingest.storage.sqlite_store import SqliteOpportunityStore logger = logging.getLogger(__name__) DEFAULT_TAB = "Ingest" +DEFAULT_RANK_TAB = "Ranked" + +# Columns written to the Ranked tab (full replace each interpret-rank sync). +RANKING_EXPORT_COLUMNS: tuple[str, ...] = ( + "RunId", + "Model", + "Rank", + "FitScore", + "Recommendation", + "Title", + "OpportunityID", + "Buyer", + "ClosingDate", + "KeywordsMatched", + "RuleRelevanceScore", + "MatchedObjectives", + "PlainEnglish", + "InterpretedObjective", + "WhyItFits", + "RisksOrMismatches", + "NextAction", + "Link", +) class SheetsSyncError(Exception): @@ -72,18 +98,13 @@ def _import_gspread(): return gspread, Credentials -def sync_sqlite_to_sheet( - store: SqliteOpportunityStore, - *, +def _open_spreadsheet( spreadsheet_id: str, - worksheet_title: str = DEFAULT_TAB, + *, service_account_file: str | Path | None = None, service_account_json: str | None = None, -) -> int: - """Replace worksheet contents with all SQLite opportunity rows. - - Returns number of data rows written (excluding header). - """ +): + """Authorize and open a spreadsheet by id. Returns (gspread_module, spreadsheet).""" if not spreadsheet_id or not spreadsheet_id.strip(): raise SheetsSyncError("GOOGLE_SHEET_ID is required") @@ -103,24 +124,23 @@ def sync_sqlite_to_sheet( f"Failed to open spreadsheet {spreadsheet_id!r}. " f"Share the sheet with the service account email as Editor. Detail: {exc}" ) from exc + return gspread, spreadsheet - title = (worksheet_title or DEFAULT_TAB).strip() or DEFAULT_TAB + +def _get_or_create_worksheet(spreadsheet: Any, title: str, *, cols: int = 20) -> Any: + gspread, _ = _import_gspread() try: try: - worksheet = spreadsheet.worksheet(title) + return spreadsheet.worksheet(title) except gspread.WorksheetNotFound: - worksheet = spreadsheet.add_worksheet(title=title, rows=1000, cols=20) + worksheet = spreadsheet.add_worksheet(title=title, rows=1000, cols=cols) logger.info("Created worksheet tab %r", title) + return worksheet except Exception as exc: raise SheetsSyncError(f"Cannot open/create worksheet {title!r}: {exc}") from exc - store.health_check() - rows = store.list_rows() - headers = list(SqliteOpportunityStore.EXPORT_COLUMNS) - values: list[list[Any]] = [headers] - for row in rows: - values.append([_cell(row.get(col)) for col in headers]) +def _replace_worksheet_values(worksheet: Any, values: list[list[Any]]) -> None: try: worksheet.clear() # Batch update is one efficient API call for the full grid. @@ -128,19 +148,169 @@ def sync_sqlite_to_sheet( except Exception as exc: raise SheetsSyncError(f"Failed to write sheet values: {exc}") from exc + +def sync_sqlite_to_sheet( + store: SqliteOpportunityStore, + *, + spreadsheet_id: str, + worksheet_title: str = DEFAULT_TAB, + service_account_file: str | Path | None = None, + service_account_json: str | None = None, +) -> int: + """Replace worksheet contents with all SQLite opportunity rows. + + Returns number of data rows written (excluding header). + """ + gspread_mod, spreadsheet = _open_spreadsheet( + spreadsheet_id, + service_account_file=service_account_file, + service_account_json=service_account_json, + ) + del gspread_mod # only needed for WorksheetNotFound inside helper + + title = (worksheet_title or DEFAULT_TAB).strip() or DEFAULT_TAB + worksheet = _get_or_create_worksheet( + spreadsheet, title, cols=max(20, len(SqliteOpportunityStore.EXPORT_COLUMNS)) + ) + + store.health_check() + rows = store.list_rows() + headers = list(SqliteOpportunityStore.EXPORT_COLUMNS) + values: list[list[Any]] = [headers] + for row in rows: + values.append([_cell(row.get(col)) for col in headers]) + + _replace_worksheet_values(worksheet, values) + data_rows = len(rows) logger.info( - "Synced %s rows to sheet %s tab %r", + "Synced %s opportunity rows to sheet %s tab %r", + data_rows, + spreadsheet_id, + title, + ) + return data_rows + + +def rankings_to_sheet_values( + ranked_rows: Sequence[Mapping[str, Any]], + *, + run_id: str, + model: str, +) -> list[list[Any]]: + """Build header + data grid for the Ranked tab from ranked dicts.""" + headers = list(RANKING_EXPORT_COLUMNS) + values: list[list[Any]] = [headers] + for row in ranked_rows: + matched = row.get("matched_objectives") + if isinstance(matched, list): + matched_s = ", ".join(str(x) for x in matched) + else: + matched_s = matched or "" + values.append( + [ + _cell(run_id), + _cell(model), + _cell(row.get("rank")), + _cell(row.get("fit_score")), + _cell(row.get("recommendation")), + _cell(row.get("title")), + _cell(row.get("opportunity_id")), + _cell(row.get("buyer")), + _cell(row.get("closing_date")), + _cell(row.get("keywords_matched")), + _cell(row.get("rule_relevance_score")), + _cell(matched_s), + _cell(row.get("plain_english")), + _cell(row.get("interpreted_objective")), + _cell(row.get("why_it_fits")), + _cell(row.get("risks_or_mismatches")), + _cell(row.get("next_action")), + _cell(row.get("link")), + ] + ) + return values + + +def sync_rankings_to_sheet( + ranked_rows: Sequence[Mapping[str, Any]], + *, + run_id: str, + model: str, + spreadsheet_id: str, + worksheet_title: str = DEFAULT_RANK_TAB, + service_account_file: str | Path | None = None, + service_account_json: str | None = None, +) -> int: + """Full-replace the Ranked tab with Grok interpret-rank output. + + **Never** targets the opportunity ``Ingest`` tab (refused). + Returns number of data rows written (excluding header). + """ + title = (worksheet_title or DEFAULT_RANK_TAB).strip() or DEFAULT_RANK_TAB + if title.lower() == DEFAULT_TAB.lower(): + raise SheetsSyncError( + f"Refusing to write Grok rankings to the {DEFAULT_TAB!r} tab " + f"(opportunity view). Use {DEFAULT_RANK_TAB!r} or another tab name." + ) + + _gspread, spreadsheet = _open_spreadsheet( + spreadsheet_id, + service_account_file=service_account_file, + service_account_json=service_account_json, + ) + worksheet = _get_or_create_worksheet( + spreadsheet, title, cols=max(20, len(RANKING_EXPORT_COLUMNS)) + ) + values = rankings_to_sheet_values(ranked_rows, run_id=run_id, model=model) + _replace_worksheet_values(worksheet, values) + + data_rows = max(0, len(values) - 1) + logger.info( + "Synced %s ranking rows to sheet %s tab %r (run_id=%s)", data_rows, spreadsheet_id, title, + run_id, ) return data_rows +def load_rankings_json(path: str | Path) -> tuple[str, str, list[dict[str, Any]]]: + """Load interpret-rank JSON report → (run_id, model, ranked list).""" + p = Path(path) + if not p.is_file(): + raise SheetsSyncError(f"Rankings JSON not found: {p}") + try: + data = json.loads(p.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise SheetsSyncError(f"Cannot read rankings JSON {p}: {exc}") from exc + if not isinstance(data, dict): + raise SheetsSyncError("Rankings JSON root must be an object") + run_id = str(data.get("run_id") or p.stem) + model = str(data.get("model") or "") + ranked = data.get("ranked") + if not isinstance(ranked, list): + raise SheetsSyncError("Rankings JSON missing ranked[] array") + rows: list[dict[str, Any]] = [] + for item in ranked: + if isinstance(item, dict): + rows.append(item) + return run_id, model, rows + + +def find_latest_rankings_json(directory: str | Path) -> Path | None: + """Return newest ``interpret-*.json`` under directory, or None.""" + d = Path(directory) + if not d.is_dir(): + return None + files = sorted(d.glob("interpret-*.json"), key=lambda p: p.stat().st_mtime, reverse=True) + return files[0] if files else None + + def _cell(value: Any) -> str | int | float: if value is None: return "" - if isinstance(value, (int, float)): + if isinstance(value, (int, float)) and not isinstance(value, bool): return value return str(value) diff --git a/tests/test_filter.py b/tests/test_filter.py index 12eb383..e19b047 100644 --- a/tests/test_filter.py +++ b/tests/test_filter.py @@ -91,17 +91,20 @@ def test_load_shipped_keywords_yaml(): group_ids = {t.group for t in cfg.terms} assert group_ids == { "microsoft_cloud", - "managed_services", - "itsm_servicenow", + "operating_authority", "advisory_consulting", "automation_process", "ai_operations", + "knowledge_flow", } - # No bare noisy terms in shipped defaults + # No bare noisy terms in shipped defaults (Atlas-tuned) bare = {t.term.lower() for t in cfg.terms} assert "teams" not in bare assert "strategy" not in bare + assert "service desk" not in bare # classic MSP noise; not Atlas offer assert "microsoft teams" in bare + assert "operating model" in bare + assert "copilot" in bare @pytest.mark.parametrize( @@ -285,13 +288,15 @@ def test_suppress_nested_terms_helper(): ] -def test_shipped_config_managed_services_not_double_counted(): +def test_shipped_config_nested_operating_terms_not_double_counted(): + """Longer phrase wins when nested shorter is fully covered by span rules.""" cfg = load_keyword_config(KEYWORDS_PATH) - hit = _tender(title="managed services for IT") + hit = _tender(title="operating contracts and registries for governance") result = match_keywords(hit, cfg) - assert "managed services" in result.terms - assert "managed service" not in result.terms - assert result.weights["managed services"] == 16 + assert "operating contracts" in result.terms + # singular nested form should not also count when only plural phrase hit + assert result.matched is True + assert "operating_authority" in result.groups def test_filter_tenders_drops_non_matches(): diff --git a/tests/test_interpret_rank.py b/tests/test_interpret_rank.py new file mode 100644 index 0000000..7e54a4e --- /dev/null +++ b/tests/test_interpret_rank.py @@ -0,0 +1,252 @@ +"""Unit tests for Grok interpret-rank (mocked client; no live API).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from opportunity_ingest.interpret_rank import ( + CompanyObjectives, + InterpretRankError, + _extract_json_object, + build_user_prompt, + filter_rows, + load_objectives, + merge_model_rankings, + prepare_opportunity_payload, + rank_opportunities, + render_markdown, + run_interpret_rank, + write_reports, +) + + +class FakeGrok: + """Deterministic client for tests.""" + + def __init__(self, *args, **kwargs) -> None: # noqa: ANN002, ANN003 + self.calls = 0 + + def complete_json(self, *, system: str, user: str) -> dict: + self.calls += 1 + # Pull opportunity ids from the JSON block in the user prompt. + start = user.find("## Opportunities to interpret") + assert start >= 0 + # Find first [ after that header for the opportunities array. + arr_start = user.find("[", start) + arr_end = user.find("\n## Required JSON schema", arr_start) + blob = user[arr_start:arr_end].strip() + opps = json.loads(blob) + rankings = [] + for i, o in enumerate(opps): + oid = o["opportunity_id"] + # Higher for known tech-ish ids in fixtures. + fit = 90 - i * 5 if "TECH" in oid else 40 - i + rankings.append( + { + "opportunity_id": oid, + "fit_score": fit, + "recommendation": "pursue" if fit >= 70 else "watch", + "plain_english": f"Plain summary for {oid}", + "interpreted_objective": f"Buyer wants outcome for {oid}", + "matched_objectives": ["microsoft_cloud"], + "why_it_fits": "Matches cloud skills.", + "risks_or_mismatches": "Limited description.", + "next_action": "Open the Link and confirm scope.", + } + ) + return {"rankings": rankings} + + +def _sample_objectives(tmp_path: Path) -> Path: + data = { + "version": 1, + "company": {"name": "Test Co", "summary": "Cloud services"}, + "objectives": [ + { + "id": "microsoft_cloud", + "label": "M365", + "priority": "critical", + "description": "Microsoft work", + } + ], + "guidance": {"prefer": ["IT"], "deprioritize": ["construction"]}, + } + path = tmp_path / "objectives.yaml" + path.write_text(yaml.safe_dump(data), encoding="utf-8") + return path + + +def test_load_objectives_ok(tmp_path: Path): + path = _sample_objectives(tmp_path) + obj = load_objectives(path) + assert obj.version == 1 + assert obj.company["name"] == "Test Co" + assert obj.objectives[0]["id"] == "microsoft_cloud" + + +def test_load_objectives_missing(tmp_path: Path): + with pytest.raises(InterpretRankError, match="not found"): + load_objectives(tmp_path / "nope.yaml") + + +def test_filter_rows_status_limit_and_score(): + rows = [ + {"OpportunityID": "A", "Status": "New", "RelevanceScore": 10}, + {"OpportunityID": "B", "Status": "New", "RelevanceScore": 50}, + {"OpportunityID": "C", "Status": "Discarded", "RelevanceScore": 99}, + ] + out = filter_rows(rows, status="New", min_rule_score=20, limit=1) + assert len(out) == 1 + assert out[0]["OpportunityID"] == "B" + + +def test_prepare_and_merge_roundtrip(): + rows = [ + { + "Title": "Cloud RFP", + "OpportunityID": "TECH-1", + "Buyer": "Agency", + "Description": "Need SharePoint & Power Apps", + "KeywordsMatched": "sharepoint", + "RelevanceScore": 40, + "Link": "https://example.com/a", + "ClosingDate": "2026-09-01T00:00:00Z", + } + ] + prepared = prepare_opportunity_payload(rows) + assert prepared[0]["description"] == "Need SharePoint & Power Apps" + model = { + "rankings": [ + { + "opportunity_id": "TECH-1", + "fit_score": 88, + "recommendation": "pursue", + "plain_english": "Buyer needs Power Platform help.", + "interpreted_objective": "Modernize collaboration tools.", + "matched_objectives": ["microsoft_cloud"], + "why_it_fits": "Direct stack match.", + "risks_or_mismatches": "None major.", + "next_action": "Download full package.", + } + ] + } + ranked = merge_model_rankings(prepared, model) + assert len(ranked) == 1 + assert ranked[0].rank == 1 + assert ranked[0].fit_score == 88 + assert ranked[0].recommendation == "pursue" + + +def test_merge_fills_missing_model_rows(): + prepared = [ + {"opportunity_id": "X", "title": "T", "buyer": None, "link": "u", + "closing_date": None, "keywords_matched": None, "rule_relevance_score": 1, + "description": "d"}, + ] + ranked = merge_model_rankings(prepared, {"rankings": []}) + assert ranked[0].fit_score == 0 + assert "Model did not return" in ranked[0].interpreted_objective + + +def test_extract_json_fenced(): + data = _extract_json_object('```json\n{"rankings": []}\n```') + assert data == {"rankings": []} + + +def test_rank_and_write_reports(tmp_path: Path): + path = _sample_objectives(tmp_path) + objectives = load_objectives(path) + rows = [ + { + "Title": "Tech deal", + "OpportunityID": "TECH-9", + "Buyer": "B", + "Description": "Azure migration", + "KeywordsMatched": "azure", + "RelevanceScore": 30, + "Link": "https://example.com/t", + "Status": "New", + }, + { + "Title": "Other", + "OpportunityID": "OTH-1", + "Buyer": "B", + "Description": "Something else", + "KeywordsMatched": "service desk", + "RelevanceScore": 10, + "Link": "https://example.com/o", + "Status": "New", + }, + ] + client = FakeGrok() + result = rank_opportunities(rows, objectives, client, model_name="fake", batch_size=10) + assert client.calls == 1 + assert result.input_count == 2 + assert result.ranked[0].opportunity_id == "TECH-9" + assert result.ranked[0].fit_score >= result.ranked[1].fit_score + + written = write_reports(result, objectives, out_dir=tmp_path / "rankings", objectives_path=path) + assert written.json_path is not None and written.json_path.is_file() + assert written.markdown_path is not None and written.markdown_path.is_file() + md = written.markdown_path.read_text(encoding="utf-8") + assert "Fit score" in md + assert "TECH-9" in md + payload = json.loads(written.json_path.read_text(encoding="utf-8")) + assert payload["ranked_count"] == 2 + + +def test_run_interpret_rank_with_factory(tmp_path: Path): + path = _sample_objectives(tmp_path) + rows = [ + { + "Title": "Tech deal", + "OpportunityID": "TECH-1", + "Buyer": "B", + "Description": "Power Platform", + "KeywordsMatched": "power platform", + "RelevanceScore": 50, + "Link": "https://example.com/a", + "Status": "New", + } + ] + result = run_interpret_rank( + rows, + objectives_path=path, + api_key="test-key", + model="fake-model", + out_dir=tmp_path / "out", + client_factory=FakeGrok, + ) + assert result.ranked[0].plain_english.startswith("Plain summary") + assert result.markdown_path is not None + + +def test_build_user_prompt_includes_objectives(): + obj = CompanyObjectives( + version=1, + company={"name": "Co"}, + objectives=[{"id": "x", "label": "X"}], + guidance={}, + raw={}, + ) + prompt = build_user_prompt( + obj, + [{"opportunity_id": "1", "title": "T"}], + ) + assert "Company objectives" in prompt + assert "opportunity_id" in prompt + + +def test_render_markdown_empty(): + from opportunity_ingest.interpret_rank import InterpretRankResult + + obj = CompanyObjectives(1, {"name": "Co"}, [{"id": "a"}], {}, {}) + md = render_markdown( + InterpretRankResult("rid", "m", [], 0), + obj, + ) + assert "No opportunities" in md diff --git a/tests/test_notify.py b/tests/test_notify.py index 519613c..52bf448 100644 --- a/tests/test_notify.py +++ b/tests/test_notify.py @@ -9,10 +9,15 @@ import pytest from opportunity_ingest.notify import ( + MatchAlertItem, NotifyError, build_adaptive_card_payload, build_ingest_alert_payload, + build_match_alert_payload, + filter_match_items, + match_items_from_ranked, notify_ingest_alert, + notify_match_alerts, post_teams_webhook, set_github_output_notified, ) @@ -163,3 +168,97 @@ def test_notify_returns_false_when_github_output_handoff_fails( ) assert post.called assert ok is False + + +def _sample_match(score: int = 55, oid: str = "ID-1") -> MatchAlertItem: + return MatchAlertItem( + title="Cloud RFP for Copilot enablement", + opportunity_id=oid, + link=f"https://example.com/notice/{oid}", + score=score, + score_kind="RelevanceScore", + buyer="PSPC", + keywords="copilot, power platform", + summary="Buyer wants Microsoft-native AI under human governance.", + recommendation="pursue", + closing_date="2026-09-01T00:00:00Z", + ) + + +def test_filter_match_items_threshold(): + items = [_sample_match(55), _sample_match(30, "LOW"), _sample_match(90, "HI")] + out = filter_match_items(items, threshold=40) + assert [i.opportunity_id for i in out] == ["HI", "ID-1"] + + +def test_build_match_alert_payload_has_cta_and_truncation(): + items = [_sample_match(52, "WS-AI"), _sample_match(41, "WS-IAM")] + payload = build_match_alert_payload(items, threshold=40, source="ingest") + card = payload["attachments"][0]["content"] + assert card["type"] == "AdaptiveCard" + assert "actions" in card + assert card["actions"][0]["type"] == "Action.OpenUrl" + assert "https://example.com/notice/WS-AI" in card["actions"][0]["url"] + body_text = " ".join( + str(b.get("text", "")) for b in card["body"] if isinstance(b, dict) + ) + assert "52/100" in body_text or "52" in body_text + + +def test_notify_match_alerts_success(): + with patch("opportunity_ingest.notify.post_teams_webhook") as post: + ok = notify_match_alerts( + "https://example.com/hook", + [_sample_match(60)], + threshold=40, + source="ingest", + ) + assert ok is True + post.assert_called_once() + + +def test_notify_match_alerts_below_threshold_skips(): + with patch("opportunity_ingest.notify.post_teams_webhook") as post: + ok = notify_match_alerts( + "https://example.com/hook", + [_sample_match(10)], + threshold=40, + ) + assert ok is False + post.assert_not_called() + + +def test_notify_match_alerts_disabled(): + with patch("opportunity_ingest.notify.post_teams_webhook") as post: + ok = notify_match_alerts( + "https://example.com/hook", + [_sample_match(90)], + enabled=False, + ) + assert ok is False + post.assert_not_called() + + +def test_match_items_from_ranked_dicts(): + ranked = [ + { + "fit_score": 52, + "title": "AI ITQ", + "opportunity_id": "X", + "link": "https://example.com/x", + "plain_english": "Source list for AI", + "recommendation": "watch", + "buyer": "PSPC", + "keywords_matched": "artificial intelligence", + }, + { + "fit_score": 5, + "title": "Elevator", + "opportunity_id": "Y", + "link": "https://example.com/y", + }, + ] + items = match_items_from_ranked(ranked, threshold=40) + assert len(items) == 1 + assert items[0].score == 52 + assert items[0].score_kind == "GrokFit" diff --git a/tests/test_sheets_sync.py b/tests/test_sheets_sync.py index 22e4b1c..9ed2705 100644 --- a/tests/test_sheets_sync.py +++ b/tests/test_sheets_sync.py @@ -8,8 +8,14 @@ import pytest from opportunity_ingest.sheets_sync import ( + DEFAULT_RANK_TAB, + RANKING_EXPORT_COLUMNS, SheetsSyncError, _load_service_account_info, + find_latest_rankings_json, + load_rankings_json, + rankings_to_sheet_values, + sync_rankings_to_sheet, ) @@ -37,3 +43,95 @@ def test_load_service_account_missing_raises() -> None: def test_load_service_account_bad_json() -> None: with pytest.raises(SheetsSyncError, match="not valid JSON"): _load_service_account_info(None, "{not-json") + + +def test_rankings_to_sheet_values_shape() -> None: + rows = [ + { + "rank": 1, + "fit_score": 88, + "recommendation": "pursue", + "title": "Cloud RFP", + "opportunity_id": "TECH-1", + "buyer": "Agency", + "closing_date": "2026-09-01", + "keywords_matched": "azure", + "rule_relevance_score": 40, + "matched_objectives": ["microsoft_cloud", "ai_operations"], + "plain_english": "Plain text", + "interpreted_objective": "Modernize", + "why_it_fits": "Stack match", + "risks_or_mismatches": "None", + "next_action": "Open link", + "link": "https://example.com/a", + } + ] + values = rankings_to_sheet_values(rows, run_id="run1", model="grok-4.5") + assert values[0] == list(RANKING_EXPORT_COLUMNS) + assert len(values) == 2 + assert values[1][0] == "run1" + assert values[1][1] == "grok-4.5" + assert values[1][2] == 1 + assert values[1][3] == 88 + assert values[1][11] == "microsoft_cloud, ai_operations" + assert values[1][-1] == "https://example.com/a" + + +def test_sync_rankings_refuses_ingest_tab() -> None: + with pytest.raises(SheetsSyncError, match="Refusing"): + sync_rankings_to_sheet( + [], + run_id="r", + model="m", + spreadsheet_id="sheet123", + worksheet_title="Ingest", + service_account_json=json.dumps({"type": "service_account"}), + ) + + +def test_load_rankings_json_and_latest(tmp_path: Path) -> None: + older = tmp_path / "interpret-20260101T000000Z.json" + newer = tmp_path / "interpret-20260811T120000Z.json" + payload = { + "run_id": "20260811T120000Z", + "model": "grok-4.5", + "ranked": [ + { + "rank": 1, + "opportunity_id": "X", + "title": "T", + "fit_score": 50, + "recommendation": "watch", + "matched_objectives": [], + "plain_english": "p", + "interpreted_objective": "i", + "why_it_fits": "w", + "risks_or_mismatches": "r", + "next_action": "n", + "link": "https://example.com", + } + ], + } + older.write_text(json.dumps({"run_id": "old", "model": "m", "ranked": []}), encoding="utf-8") + newer.write_text(json.dumps(payload), encoding="utf-8") + # Ensure mtime ordering + import os + import time + + os.utime(older, (time.time() - 100, time.time() - 100)) + os.utime(newer, (time.time(), time.time())) + + run_id, model, ranked = load_rankings_json(newer) + assert run_id == "20260811T120000Z" + assert model == "grok-4.5" + assert ranked[0]["opportunity_id"] == "X" + + latest = find_latest_rankings_json(tmp_path) + assert latest == newer + assert find_latest_rankings_json(tmp_path / "missing") is None + assert DEFAULT_RANK_TAB == "Ranked" + + +def test_load_rankings_json_missing(tmp_path: Path) -> None: + with pytest.raises(SheetsSyncError, match="not found"): + load_rankings_json(tmp_path / "nope.json") diff --git a/tests/test_smoke.py b/tests/test_smoke.py index 2695226..e297582 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -32,6 +32,8 @@ def test_cli_parser_has_frozen_subcommands(): "check-store", "export-csv", "sync-sheets", + "interpret-rank", + "sync-rank-sheets", } From 9476c59d2fef7b3285156422c240ae7a65313281 Mon Sep 17 00:00:00 2001 From: Execute Plan Date: Sat, 15 Aug 2026 10:40:43 -0400 Subject: [PATCH 2/2] feat: Slack match notifications and professionalize repo docs Add Slack Web API match alerts (bot token + channel) with setup guide, processor overview, and notify tests. Remove personal paths, usernames, and informal wording from docs and scripts. Resolve repo root in daily_sync.ps1 and point company config at atlasflowgroup.com. --- .env.example | 12 + AGENTS.md | 3 +- README.md | 6 +- config/keywords.yaml | 4 +- config/notify.yaml | 41 +-- config/objectives.yaml | 20 +- config/settings.example.env | 11 + docs/BACKLOG.md | 147 ++++++++- docs/INDEX.md | 5 +- docs/PLUG_AND_PLAY.md | 73 +++-- docs/PROCESSOR_OVERVIEW.md | 220 +++++++++++++ docs/STATUS.md | 10 +- pyproject.toml | 3 + scripts/daily_sync.ps1 | 3 +- scripts/google_sheets_setup.md | 17 +- scripts/ops_runbook.md | 10 +- scripts/slack_cli_setup.md | 189 +++++++++++ src/opportunity_ingest/cli.py | 34 +- src/opportunity_ingest/config.py | 112 ++++++- src/opportunity_ingest/notify.py | 501 +++++++++++++++++++++++++++-- src/opportunity_ingest/pipeline.py | 26 +- tests/test_notify.py | 155 +++++++++ 22 files changed, 1456 insertions(+), 146 deletions(-) create mode 100644 docs/PROCESSOR_OVERVIEW.md create mode 100644 scripts/slack_cli_setup.md diff --git a/.env.example b/.env.example index 2a05c1f..9214424 100644 --- a/.env.example +++ b/.env.example @@ -27,6 +27,7 @@ LOG_LEVEL=INFO # Optional Google Sheets sync (free service account + Sheets API): # pip install -e ".[sheets]" # GOOGLE_SHEET_ID= # from docs.google.com/spreadsheets/d//edit +# GOOGLE_SHEET_URL= # optional full share/deep link for Teams/Slack cards (else built from ID) # GOOGLE_SHEET_TAB=Ingest # GOOGLE_SHEET_RANK_TAB=Ranked # Grok interpret-rank full-replace tab (never Ingest) # GOOGLE_SERVICE_ACCOUNT_FILE=secrets/google-service-account.json @@ -47,3 +48,14 @@ LOG_LEVEL=INFO # TEAMS_MATCH_SCORE_THRESHOLD=40 # 0–100; RelevanceScore (ingest) or Grok fit # TEAMS_MATCH_MAX_ITEMS=8 # NOTIFY_CONFIG_PATH=config/notify.yaml + +# Slack match pings — prefer Slack CLI / Bolt tokens (https://docs.slack.dev/tools/slack-cli/): +# slack login → create/install app with chat:write → paste: +# SLACK_BOT_TOKEN=xoxb-... # Bot User OAuth Token (required for Web API) +# SLACK_CHANNEL_ID=C... # or #channel-name +# SLACK_APP_TOKEN=xapp-... # optional; Socket Mode / slack run only +# SLACK_MATCH_NOTIFY_ENABLED=true +# Setup guide: scripts/slack_cli_setup.md +# Legacy Incoming Webhook fallback (discouraged): +# SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... +# SLACK_MATCH_WEBHOOK_URL= diff --git a/AGENTS.md b/AGENTS.md index 12242f1..8cf9572 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,6 +37,7 @@ Daily (or on-demand) pipeline: | 1 | **This file (`AGENTS.md`)** | Non-negotiable rules + orientation | | 2 | [`docs/STATUS.md`](docs/STATUS.md) | Living readiness snapshot (what works / next) | | 3 | [`docs/PLUG_AND_PLAY.md`](docs/PLUG_AND_PLAY.md) | Human keys + plug-and-play steps for built features | +| 3b | [`docs/PROCESSOR_OVERVIEW.md`](docs/PROCESSOR_OVERVIEW.md) | Plain language: this app as one processor — triggers, I/O, system place | | 4 | [`docs/DATA_UPDATE_DIRECTIVES.md`](docs/DATA_UPDATE_DIRECTIVES.md) | **MUST / MUST NOT** for data writes, schema, sync | | 5 | [`docs/AS_BUILT.md`](docs/AS_BUILT.md) | Current architecture, modules, CLI, env | | 6 | [`docs/BACKLOG.md`](docs/BACKLOG.md) | Roadmap / discussed but **not built** | @@ -80,7 +81,7 @@ src/opportunity_ingest/ models.py # TenderRecord, OpportunityFields, ExistingKeys config.py # pydantic-settings state.py # zero-new streak JSON - notify.py # Teams ops alerts + high-match capture cards (CTA links) + notify.py # Teams + Slack match pings (score≥threshold); Teams ops alerts exit_codes.py # exit + notify matrix sheets_sync.py # SQLite → Google Sheets full tab replace interpret_rank.py # Grok rephrase + fit rank (report only) diff --git a/README.md b/README.md index e482473..4061739 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Phase 1 pipeline: download CanadaBuys **Open Tender Notices** CSV, filter by key | **Optional view** | Google Sheets tab full-replace (`sync-sheets`) | | **LLM / agent entry** | **[`AGENTS.md`](AGENTS.md)** ← start here for AI-assisted development | | **Human plug-in** | **[`docs/PLUG_AND_PLAY.md`](docs/PLUG_AND_PLAY.md)** ← keys, steps, roadmap to go live | +| **Processor overview** | **[`docs/PROCESSOR_OVERVIEW.md`](docs/PROCESSOR_OVERVIEW.md)** ← role, triggers, I/O in the wider system | --- @@ -16,6 +17,7 @@ Phase 1 pipeline: download CanadaBuys **Open Tender Notices** CSV, filter by key | Doc | Audience | Purpose | |-----|----------|---------| +| [`docs/PROCESSOR_OVERVIEW.md`](docs/PROCESSOR_OVERVIEW.md) | **Everyone** | This app as **one key processor** — role, triggers, outputs | | [`docs/PLUG_AND_PLAY.md`](docs/PLUG_AND_PLAY.md) | **Operators / humans** | **Keys to plug, function list, go-live sequence, roadmap** | | [`AGENTS.md`](AGENTS.md) | LLMs + devs | Hard rules, module map, task routing | | [`docs/STATUS.md`](docs/STATUS.md) | Everyone | Living readiness / what works now | @@ -49,7 +51,7 @@ If design doc and as-built conflict → **prefer code + AS_BUILT + DATA_UPDATE_D → **[`docs/PLUG_AND_PLAY.md`](docs/PLUG_AND_PLAY.md)** ```powershell -cd "C:\Users\Vikrant\Documents\Agentic AI Learning\contractbiddingsystem" +cd python -m venv .venv .\.venv\Scripts\Activate.ps1 @@ -93,7 +95,7 @@ python -m opportunity_ingest interpret-rank --status New --limit 20 # Re-push last report without Grok: python -m opportunity_ingest sync-rank-sheets ``` -**Notes learned in setup:** +**Setup notes:** - `pip install -e .` needs the trailing `.` - CanadaBuys may return **403** without a browser User-Agent (handled in `download.py`) diff --git a/config/keywords.yaml b/config/keywords.yaml index 2524786..7f06662 100644 --- a/config/keywords.yaml +++ b/config/keywords.yaml @@ -1,6 +1,6 @@ # Engineering-owned keyword config for CanadaBuys open-tender filtering. -# Tuned for Atlas Flow Group capabilities (site crawl 2026-08-11): -# https://atlasflowgroupwebsite.vikrant-singh1.workers.dev/ +# Tuned for Atlas Flow Group capabilities (public site review 2026-08-11): +# https://www.atlasflowgroup.com/ # Ops request changes via GitHub issues; eng reviews/merges. # # Atlas Flow Group is NOT a classic MSP / ServiceNow SI / construction vendor. diff --git a/config/notify.yaml b/config/notify.yaml index 8755039..7f52f64 100644 --- a/config/notify.yaml +++ b/config/notify.yaml @@ -1,34 +1,35 @@ -# Teams / notification pipeline config (engineering-owned). +# Notification pipeline config (engineering-owned). # Loaded by opportunity_ingest when present; env vars override (see Settings). # -# Ops alerts (hard fail / partial errors / zero-new streak) still use -# TEAMS_WEBHOOK_URL via notify_ingest_alert. +# Ops alerts (hard fail / partial / zero-new streak): TEAMS_WEBHOOK_URL only today. # -# Match alerts: when NEW opportunities clear the score bar, post a channel -# Adaptive Card with summary + Open notice CTA links. +# Match alerts (score ≥ threshold): Teams Adaptive Card and/or Slack Block Kit. +# Ingest write path → rule RelevanceScore +# interpret-rank → Grok fit_score +# When GOOGLE_SHEET_ID (or GOOGLE_SHEET_URL) is set, cards include Open Google Sheet. +# +# Slack preferred auth = Slack CLI / Bolt tokens (not Incoming Webhooks): +# https://docs.slack.dev/tools/slack-cli/ +# SLACK_BOT_TOKEN=xoxb-... (required for chat.postMessage) +# SLACK_CHANNEL_ID=C... (or #channel-name) +# SLACK_APP_TOKEN=xapp-... (optional; used by `slack run` Socket Mode only) version: 1 +match: + score_threshold: 40 + max_items_per_message: 8 + teams: - # Master switch for match pings (ops alerts still follow TEAMS_WEBHOOK_URL). match_notify_enabled: true - - # Score threshold 0–100 (40 = "above 40% match" on our 0–100 scales). - # - ingest write path: rule RelevanceScore - # - interpret-rank path: Grok fit_score match_score_threshold: 40 - - # Prefer TEAMS_MATCH_WEBHOOK_URL env for a dedicated capture channel; - # if unset, falls back to TEAMS_WEBHOOK_URL (same Workflows connector). - # use_ops_webhook_fallback: true # always true in code when match URL empty - - # Cap how many opportunities appear in one Teams card (rest summarized). max_items_per_message: 8 - - # Truncate free-text fields for Adaptive Card readability. title_max_chars: 120 summary_max_chars: 280 - - # Card branding card_title_ingest: "New opportunity matches (≥ threshold)" card_title_rank: "Grok fit matches (≥ threshold)" cta_label: "Open notice" + +slack: + match_notify_enabled: true + # Prefer Web API (bot token). Legacy webhook only if SLACK_BOT_TOKEN unset. + # mode: web_api | webhook # informational; code auto-selects diff --git a/config/objectives.yaml b/config/objectives.yaml index bf42b67..bbf2b3d 100644 --- a/config/objectives.yaml +++ b/config/objectives.yaml @@ -1,6 +1,6 @@ # Engineering-owned company objectives for Grok-assisted interpret-rank. -# Source of truth for WHO we are: Atlas Flow Group public site (crawled 2026-08-11). -# https://atlasflowgroupwebsite.vikrant-singh1.workers.dev/ +# Source of truth for company positioning: Atlas Flow Group public site (reviewed 2026-08-11). +# https://www.atlasflowgroup.com/ # Pages: /, /services, /approach, /about, /contact # Align keyword groups in config/keywords.yaml with these objectives. # @@ -10,7 +10,7 @@ version: 2 company: name: "Atlas Flow Group" - website: "https://atlasflowgroupwebsite.vikrant-singh1.workers.dev/" + website: "https://www.atlasflowgroup.com/" email: "hello@atlasflowgroup.com" summary: > SMB-focused advisory and Microsoft-native enablement firm. We design @@ -18,7 +18,7 @@ company: control, and every material decision is traceable by design. Lead with advisory (authority, operating model, governance readiness), then implement Copilot / Power Platform / M365 workflows under that model — - not tool sprawl, not unowned AI, not large multi-year SI theatre. + not tool sprawl, not unowned AI, not large multi-year systems-integrator programmes. geography: > Canada-oriented public and private buyers are in scope when the work is operating-model, process, Microsoft-native AI enablement, or human-governed @@ -26,7 +26,7 @@ company: footprints are a poor fit. posture: > One Authority · Human Governed · Traceable by Design. Authority before - automation. AI assistance without authority. Implementation over slideware. + automation. AI assistance without authority. Implementation over presentation-only deliverables. Not a pure product vendor, not a classic MSP/service desk outsourcer, not ServiceNow platform SI, not construction/facilities/commodities. @@ -76,7 +76,7 @@ objectives: remain accountable. Strong fit for AI governance, responsible AI, Copilot adoption with decision rights, and anti-patterns of unowned AI. Weak fit for pure GenAI product platforms, AI that replaces ownership, - or hype-driven transformation without governance. + or speculative transformation without governance. - id: knowledge_flow label: "Traceable operating flow & continuous improvement" @@ -98,13 +98,13 @@ guidance: - "Services categories (*SRV) with real ops/IT/process scope" deprioritize: - "Construction, facilities, elevator, snow/ice, food/beverage design, pure advertising creative" - - "Classic MSP / service desk / desktop outsourcing as the core SOW (not our offer)" - - "ServiceNow / ITSM platform implementation as primary scope (not on our site)" - - "Specialized product platforms (e.g. third-party risk intelligence SaaS) unless we partner" + - "Classic MSP / service desk / desktop outsourcing as the core SOW (outside published services)" + - "ServiceNow / ITSM platform implementation as primary scope (not a published capability)" + - "Specialized product platforms (e.g. third-party risk intelligence SaaS) unless a partnership is in scope" - "Military payroll / niche HRIS sustainment without ops-model angle" - "Keyword false positives with no operating-model or Microsoft-native scope" - "Buyers who want AI to replace human ownership" - - "Hype AI transformation without governance" + - "Speculative AI transformation without governance" output_tone: > Plain language for an Atlas Flow Group capture lead. Lead with whether the buyer needs authority/governance design and/or Microsoft-native enablement diff --git a/config/settings.example.env b/config/settings.example.env index 1100a2e..36e0bf8 100644 --- a/config/settings.example.env +++ b/config/settings.example.env @@ -50,6 +50,7 @@ LOG_LEVEL=INFO # pip install -e ".[sheets]" # .env must be at REPO ROOT (not scripts/) # GOOGLE_SHEET_ID= +# GOOGLE_SHEET_URL= # optional; Teams/Slack "Open Google Sheet" CTA (else from ID) # GOOGLE_SHEET_TAB=Ingest # GOOGLE_SHEET_RANK_TAB=Ranked # GOOGLE_SERVICE_ACCOUNT_FILE=secrets/google-service-account.json @@ -70,3 +71,13 @@ LOG_LEVEL=INFO # TEAMS_MATCH_SCORE_THRESHOLD=40 # TEAMS_MATCH_MAX_ITEMS=8 # NOTIFY_CONFIG_PATH=config/notify.yaml + +# --- Slack notify (Slack CLI / Bolt tokens; same match threshold as Teams) --- +# https://docs.slack.dev/tools/slack-cli/ — see scripts/slack_cli_setup.md +# SLACK_BOT_TOKEN=xoxb-... +# SLACK_CHANNEL_ID=C... +# SLACK_APP_TOKEN=xapp-... +# SLACK_MATCH_NOTIFY_ENABLED=true +# Legacy webhook fallback: +# SLACK_WEBHOOK_URL= +# SLACK_MATCH_WEBHOOK_URL= diff --git a/docs/BACKLOG.md b/docs/BACKLOG.md index 63b060d..fef955d 100644 --- a/docs/BACKLOG.md +++ b/docs/BACKLOG.md @@ -38,12 +38,13 @@ These **are implemented** in code as of 2026-08-11. Listed here only so reviewer | B-DONE-7 | Optional extras | `pip install -e ".[ai]"`, `".[sheets]"` | | B-DONE-8 | Atlas Flow Group site-tuned keywords + objectives | `config/keywords.yaml`, `config/objectives.yaml` (crawl 2026-08-11) | | B-DONE-9 | Teams high-match capture pipeline (≥40 score) | `config/notify.yaml`, `notify_match_alerts`, ingest write + interpret-rank | +| B-DONE-10 | Slack high-match engine (CLI/Bolt tokens + legacy webhook) | `SLACK_BOT_TOKEN`/`SLACK_CHANNEL_ID`, `slack-sdk`, `scripts/slack_cli_setup.md` | Normative rules: [`DATA_UPDATE_DIRECTIVES.md`](DATA_UPDATE_DIRECTIVES.md) §2.8, §3.2. -**Note (2026-08-11):** Keyword/objective packs were manually derived from a full crawl of -https://atlasflowgroupwebsite.vikrant-singh1.workers.dev/ (home, services, approach, about, contact). -Automated site crawl → YAML (B-04) remains unbuilt; this was a one-off eng tuning pass. +**Note (2026-08-11):** Keyword and objective packs were derived from a review of the +Atlas Flow Group public site (home, services, approach, about, contact). +Automated site crawl → YAML (B-04) remains unbuilt; this was a one-off engineering tuning pass. --- @@ -113,7 +114,7 @@ Automated site crawl → YAML (B-04) remains unbuilt; this was a one-off eng tun ## 3. Explicit phase non-goals (still deferred) -Carried from Phase 1 design / AGENTS hard rules. Not “forgotten”—**intentionally not building now**. +Carried from Phase 1 design / AGENTS hard rules. These items are **intentionally deferred**. | ID | Item | Notes | |----|------|--------| @@ -142,26 +143,28 @@ Recorded from 2026-08-11 design discussion. Update when answered. ## 5. Roadmap summary (humans) -**Already built — needs only keys:** see [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) (Sheets, Grok, Teams, SQLite ingest). +**Already built — needs only keys:** see [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) (Sheets, Grok, Teams/Slack, SQLite ingest). | Horizon | Focus | |---------|--------| -| **This week (ops)** | Plug `TEAMS_WEBHOOK_URL`; prove match card; GitHub secrets + one scheduled dry run | +| **This week (ops)** | Slack/Teams match cards stable; Actions secrets + one scheduled dry run | | **This month (eng+ops)** | B-06 optional: Actions step for interpret-rank + Ranked; tune objectives from real Ranked outcomes | | **Next** | B-01/B-02 type-aware keywords if false positives remain high | | **Later** | B-03 suggest-keywords; B-04/B-05 crawls only with product OK | +| **Long horizon** | §7 multi-geography **lawful** source mesh + self-improving rank loop (not unrestricted web scrape) | | **Not planned now** | N-* auto-bid, dual-write, two-way Status (see §3) | ### Suggested implementation order (when prioritized) ```text -0) Human: PLUG_AND_PLAY — Teams webhook + Actions secrets (no code) -1) B-07 already largely done for Grok+Ranked; finish Teams live proof +0) Human: PLUG_AND_PLAY — channel webhooks + Actions secrets (no code) +1) B-07 finish remaining live proofs (Teams ops if needed) 2) B-06 Actions wire-up (if daily AI cost OK) 3) B-01 source type classifier (rules only) 4) B-02 keyword packs by type 5) B-03 suggest-keywords (Grok → PR diff) 6) B-04 / B-05 only after product answers +7) Long horizon: V-* items in §7 (adapter mesh, geography packs, feedback loop) ``` --- @@ -171,9 +174,137 @@ Recorded from 2026-08-11 design discussion. Update when answered. | Doc | Role | |-----|------| | [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) | Human keys, function list, go-live sequence | +| [`PROCESSOR_OVERVIEW.md`](PROCESSOR_OVERVIEW.md) | Processor role | | [`STATUS.md`](STATUS.md) | What works / validated | | [`AS_BUILT.md`](AS_BUILT.md) | Architecture that exists | | [`DECISIONS.md`](DECISIONS.md) | Locked ADRs | | [`DATA_UPDATE_DIRECTIVES.md`](DATA_UPDATE_DIRECTIVES.md) | MUST / MUST NOT | | [`CHANGE_PLAYBOOK.md`](CHANGE_PLAYBOOK.md) | How to change keywords / Grok objectives | | [`../AGENTS.md`](../AGENTS.md) | Hard rules for agents | + +--- + +## 7. Long-horizon vision — multi-geography opportunity intelligence + +**Status:** Future goals only (not scheduled for build). +**Recorded:** 2026-08-11 (product discussion: global data lakes / USMCA / self-sustaining system). +**Guardrails:** Only the constraints of **this project** ([`AGENTS.md`](../AGENTS.md), DATA_UPDATE, Phase 1 non-goals N-*). +**Processor context:** [`PROCESSOR_OVERVIEW.md`](PROCESSOR_OVERVIEW.md). + +### 7.1 Intent (restated from first principles) + +**Common informal request:** scrape every public data source for bids, any geography. + +**Goal this project will track (lawful and useful):** + +> Continuously discover **high-signal procurement / opportunity records** from **declared, permitted sources**, normalize them into **one opportunity schema**, rank them against **our objectives**, and improve filters from **human Status feedback** — for **any geography we choose to enable** — without auto-bidding and without violating create-only / eng-owned control planes. + +**Explicitly out of bounds (even as “future”):** + +- Unrestricted scraping of the whole public web +- Bypassing ToS, paywalls, auth, or robots where prohibited +- Inventing Links or silent-truncating URLs +- Auto-mutate production `keywords.yaml` without eng review (N-8) +- Auto-bid / proposal auto-submit (N-6) +- Treating Sheets as system of record + +Opaque translation of signals is allowed **only** as: structured extraction from a **registered source adapter** + optional LLM **interpretation** into the same schema — not unrestricted harvesting of arbitrary pages. + +### 7.2 Future goal register (V-*) + +| ID | Goal | Status | Notes | +|----|------|--------|--------| +| **V-1** | **Source registry** — catalog of lakes/portals per geography (open data APIs, official CSVs, licensed aggregators) | Proposed | Each source: license, rate limit, identity keys, health | +| **V-2** | **Adapter interface** — pluggable fetch→normalize→`TenderRecord`/`OpportunityFields` | Designed (not built) | Generalizes CanadaBuys download/parse; multi-source N-1 becomes real | +| **V-3** | **Geography packs** — keywords + objectives + legal flags per region (CA / US / other) | Proposed | USMCA: mark set-aside / domestic-preference when known | +| **V-4** | **Signal fusion** — same real-world tender from two portals → one OpportunityID/Link family | Proposed | Extends current dedupe; never invent IDs | +| **V-5** | **Enrichment lane** — optional fetch of notice `Link` body when CSV is thin | Proposed | Same as B-05; per-source ToS gate | +| **V-6** | **Opaque → structured** — LLM maps messy HTML/PDF fields into schema (post-fetch) | Proposed | Post-ingest / side table; do not overwrite Status | +| **V-7** | **Human feedback loop** — Status/Discarded/Bidding trains **suggested** keyword/objective diffs | Proposed | B-03 + N-2 hybrid; eng merge only | +| **V-8** | **Self-sustaining ops** — schedule, budgets (MAX_CREATE), cost caps (Grok), source health, Slack/Teams heartbeats | Partial | Daily Actions + notify exist; multi-source health TBD | +| **V-9** | **Advantage layer** — rank not only “keyword hit” but **winnability** (set-aside, clearance, geography, partner-needed) | Proposed | Grok objectives expand; still report-only | +| **V-10** | **Commercial + public** — optional private RFP feeds / partner drops as first-class sources | Proposed | Not only government open data | + +### 7.3 Evolutionary architecture (self-sustaining, under project constraints) + +```text + ┌──────────────────────────┐ + │ Source Registry (V-1) │ + │ CA · US · … (enabled) │ + └────────────┬─────────────┘ + │ lawful adapters only (V-2) + ▼ +┌──────────────┐ normalize ┌────────────────────────────┐ +│ CanadaBuys │─────────────────►│ Opportunity mesh (SQLite) │ +│ + future │ dedupe V-4 │ create-only SoR │ +└──────────────┘ └────────────┬───────────────┘ + │ + optional enrich V-5/V-6 │ + ▼ + ┌────────────────────────────┐ + │ Rank / interpret (Grok) │ + │ geography packs V-3 / V-9 │ + └────────────┬───────────────┘ + │ + views + alerts │ + Sheets · Slack · Teams │ + ▼ + ┌────────────────────────────┐ + │ Human Status (truth) │ + │ pursue / discard / bid │ + └────────────┬───────────────┘ + │ suggest-only V-7 + ▼ + ┌────────────────────────────┐ + │ Eng PR: keywords/objectives│ + │ (never silent AI overwrite)│ + └────────────────────────────┘ +``` + +**Self-sustaining means:** scheduled runs + budget caps + source health + human-in-the-loop learning **suggestions** — not unsupervised self-rewriting of production rules. + +### 7.4 Decision principles + +**Question:** How should this project pursue multi-geography coverage while staying inside project guardrails and remaining operable? + +**Review lenses (advisory):** + +| Seat | Lens | Core claim | +|------|------|------------| +| **Systems engineer** | Interfaces | One schema plus adapters is preferable to a single unrestricted crawler | +| **Procurement** | Markets | Geography is a **policy surface** (set-asides, TAA), not only additional HTTP sources | +| **Security / counsel** | Lawful use | Only sources that can be defended; license and terms of use are product features | +| **Capture lead** | ROI | Alert quality over coverage volume; Status feedback is the training signal | +| **Company positioning** | Fit | Canada public-sector plus partner/commercial sources before large US federal scope | + +**Atomic unit of value:** +A **deduplicated opportunity with a real Link**, a **justified fit score**, and a **human action** (open / pursue / discard) — not additional pages retrieved. + +**Adopted principles:** + +1. **Operating constraint:** Wins come from **timely, relevant, actionable** notices — not from total bytes retrieved. +2. **Exclude:** Unbounded crawl, dual system of record, auto-bid, silent keyword mutation. +3. **Keep / build:** Adapter mesh (V-2), geography packs (V-3), fusion dedupe (V-4), human feedback → engineering PR (V-7), cost-capped rank + alerts (already started). +4. **Expansion rule:** Add geography only when the **Canada loop is stable** and a **named source** has a clear license and identity model. +5. **Iteration:** Each new source is a **module with a disable switch**, not a rewrite of the core processor. + +**Decision:** +Pursue broader coverage via a **lawful, extensible source mesh and a human-feedback rank loop**, not unrestricted scraping. That is the only path that stays inside AGENTS hard rules and can remain operable. + +### 7.5 Phased enablement (when product prioritizes) + +| Phase | Deliver | Exit criteria | +|-------|---------|---------------| +| **G0** (now) | Single source CanadaBuys + rank + Slack | Operators use Status weekly | +| **G1** | V-1 registry + second **official** adapter (e.g. one more CA or licensed feed) | Create-only + dedupe still green | +| **G2** | V-3 geography pack US (flags only) + optional partner feed | Clear “skip set-aside” behavior | +| **G3** | V-5/V-6 enrichment + V-7 suggest-keywords from Status | Eng merges PRs; no silent YAML write | +| **G4** | Multi-geo schedule + cost/health automation V-8 | Budget alarms; dead sources auto-disable | + +### 7.6 Product questions for long horizon + +1. Which **second source** is first (MERX, provincial, SAM unrestricted only, commercial feed)? +2. Who owns **source license** approval? +3. Is **US** a geography pack for Atlas, or only partner-sub signals? +4. What Status values feed **positive** vs **negative** training for V-7? +5. Hard **monthly $ cap** on Grok + fetch? diff --git a/docs/INDEX.md b/docs/INDEX.md index 6e9903f..3110ce0 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -11,8 +11,9 @@ Central map for humans and LLMs. **Start at [`../AGENTS.md`](../AGENTS.md).** | [`../AGENTS.md`](../AGENTS.md) | Entry point: hard rules, module map, task routing | | [`STATUS.md`](STATUS.md) | Living implementation / ops readiness snapshot | | [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) | **Human keys + next steps** to turn on built features | +| [`PROCESSOR_OVERVIEW.md`](PROCESSOR_OVERVIEW.md) | **Plain-language view:** this app as one processor — triggers, I/O, place in system | | [`AS_BUILT.md`](AS_BUILT.md) | Implemented architecture (source of truth for “what exists now”) | -| [`BACKLOG.md`](BACKLOG.md) | Roadmap / **discussed but not built** | +| [`BACKLOG.md`](BACKLOG.md) | Roadmap / not built — includes **§7 long-horizon multi-geo vision** | | [`DATA_UPDATE_DIRECTIVES.md`](DATA_UPDATE_DIRECTIVES.md) | Normative data write/sync rules (MUST / MUST NOT) | | [`CHANGE_PLAYBOOK.md`](CHANGE_PLAYBOOK.md) | Step recipes for common development changes | | [`DECISIONS.md`](DECISIONS.md) | Key architectural decisions (ADR-style, as-built) | @@ -33,6 +34,7 @@ Central map for humans and LLMs. **Start at [`../AGENTS.md`](../AGENTS.md).** |-----|-------------| | [`../scripts/ops_runbook.md`](../scripts/ops_runbook.md) | Daily triage, MAX_CREATE policy, calibration, rollback | | [`../scripts/google_sheets_setup.md`](../scripts/google_sheets_setup.md) | Free Sheets service-account setup + `sync-sheets` | +| [`../scripts/slack_cli_setup.md`](../scripts/slack_cli_setup.md) | Slack CLI / Bolt tokens for match notify | | [`../scripts/provision_sharepoint_list.md`](../scripts/provision_sharepoint_list.md) | SharePoint list + Sites.Selected activation | --- @@ -54,4 +56,5 @@ Central map for humans and LLMs. **Start at [`../AGENTS.md`](../AGENTS.md).** | Doc | Description | |-----|-------------| | [`../README.md`](../README.md) | Quick start, CLI, secrets table, links | +| [`PROCESSOR_OVERVIEW.md`](PROCESSOR_OVERVIEW.md) | Plain-language processor role, triggers, outputs | | [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) | Operator plug-in checklist (keys → working stack) | diff --git a/docs/PLUG_AND_PLAY.md b/docs/PLUG_AND_PLAY.md index 35dd295..99f9655 100644 --- a/docs/PLUG_AND_PLAY.md +++ b/docs/PLUG_AND_PLAY.md @@ -2,10 +2,11 @@ **Audience:** operators and humans who will turn on what engineering already built. **As-of:** 2026-08-11 -**Code status:** features below are **implemented**; they stay idle until you plug secrets/config. +**Code status:** features below are **implemented**; they stay idle until secrets and configuration are provided. | Related docs | | |--------------|--| +| **What this processor is** | [`PROCESSOR_OVERVIEW.md`](PROCESSOR_OVERVIEW.md) | | What works now | [`STATUS.md`](STATUS.md) | | Architecture | [`AS_BUILT.md`](AS_BUILT.md) | | Future / not built | [`BACKLOG.md`](BACKLOG.md) · roadmap § below | @@ -76,7 +77,9 @@ Copy `.env.example` → **repo-root** `.env` (never `scripts/.env`). Never commi | 5 | `XAI_API_KEY` | [console.x.ai](https://console.x.ai) API keys | `interpret-rank` | | 6 | `TEAMS_WEBHOOK_URL` | Teams channel → Workflows → “Post when webhook received” | Ops alerts + match fallback | | 7 | `TEAMS_MATCH_WEBHOOK_URL` *(optional)* | Second Workflows webhook | Dedicated capture channel | -| 8 | `TEAMS_MATCH_SCORE_THRESHOLD` | Default **40** via env or `config/notify.yaml` | When to ping | +| 8 | `TEAMS_MATCH_SCORE_THRESHOLD` | Default **40** via env or `config/notify.yaml` | When to ping (Teams **and** Slack) | +| 8b | `SLACK_BOT_TOKEN` + `SLACK_CHANNEL_ID` | Slack CLI / Bolt app (`xoxb-` + channel) | High-match Slack posts (preferred) | +| 8c | `SLACK_APP_TOKEN` | Optional `xapp-` for Socket Mode only | Not required for match posts | | 9 | Azure / SharePoint secrets | Only if `STORAGE_BACKEND=sharepoint` | Alternate store (not required day-1) | | 10 | GitHub Actions secrets/vars | Repo Settings → Secrets | Unattended daily schedule | @@ -118,7 +121,23 @@ TEAMS_MATCH_NOTIFY_ENABLED=true TEAMS_MATCH_SCORE_THRESHOLD=40 ``` -Detailed Sheets steps: [`../scripts/google_sheets_setup.md`](../scripts/google_sheets_setup.md). +**E — + Slack via Slack CLI / Bolt tokens (same match criteria as Teams)** + +```env +# https://docs.slack.dev/tools/slack-cli/ — full steps: scripts/slack_cli_setup.md +SLACK_BOT_TOKEN=xoxb-... +SLACK_CHANNEL_ID=C... +# SLACK_APP_TOKEN=xapp-... # optional; slack run Socket Mode only +SLACK_MATCH_NOTIFY_ENABLED=true +TEAMS_MATCH_SCORE_THRESHOLD=40 +``` + +```powershell +pip install -e ".[slack]" +``` + +Detailed Sheets steps: [`../scripts/google_sheets_setup.md`](../scripts/google_sheets_setup.md). +Slack CLI paste-config: [`../scripts/slack_cli_setup.md`](../scripts/slack_cli_setup.md). --- @@ -193,13 +212,25 @@ python -m opportunity_ingest interpret-rank --status New --limit 27 Skip sheet: `--no-sync-sheets`. Skip Teams: `--no-teams`. -### Step 4 — Teams channel pings (human actions) +### Step 4 — Teams and/or Slack capture pings (human actions) + +**Teams** + +1. **Workflows** → *Post to a channel when a webhook request is received* → copy URL. +2. Set `TEAMS_WEBHOOK_URL` (and optionally `TEAMS_MATCH_WEBHOOK_URL`). + +**Slack (CLI / Bolt config)** -1. In the target **Teams channel** (or chat via Workflows that posts there): - **Workflows** → template *Post to a channel when a webhook request is received* → copy URL. -2. Set `TEAMS_WEBHOOK_URL` (and optionally `TEAMS_MATCH_WEBHOOK_URL` for a capture-only channel). -3. Confirm `config/notify.yaml` or env: threshold **40**, `match_notify_enabled: true`. -4. Trigger: +1. Install [Slack CLI](https://docs.slack.dev/tools/slack-cli/), run `slack login`, create/install an app with **`chat:write`**. +2. Paste **Bot User OAuth Token** → `SLACK_BOT_TOKEN=xoxb-...` +3. Set **channel** → `SLACK_CHANNEL_ID=C...` (invite bot to channel). +4. `pip install -e ".[slack]"` — see [`../scripts/slack_cli_setup.md`](../scripts/slack_cli_setup.md). + +**Shared criteria** + +3. Threshold **40** (`TEAMS_MATCH_SCORE_THRESHOLD` or `config/notify.yaml` → `match.score_threshold`). +4. Enable flags: `TEAMS_MATCH_NOTIFY_ENABLED` / `SLACK_MATCH_NOTIFY_ENABLED` (default true). +5. Trigger: ```powershell # New high rule-score creates: @@ -209,9 +240,8 @@ python -m opportunity_ingest run --write --max-create 10 python -m opportunity_ingest interpret-rank --limit 27 ``` -**You should see:** Adaptive Card in the channel with summaries + **Open notice** buttons. - -Ops-only alerts (hard fail / zero-new streak) also use `TEAMS_WEBHOOK_URL` without needing a separate match URL. +**You should see:** Teams Adaptive Card and/or Slack Block Kit message with summaries + open-notice buttons. +Ops alerts (hard fail / streak) remain **Teams-only** today (`TEAMS_WEBHOOK_URL`). ### Step 5 — Daily automation (pick one) @@ -252,7 +282,7 @@ Workflow file: [`.github/workflows/daily-canadabuys-ingest.yml`](../.github/work --- -## 6. End-to-end “happy path” (everything on) +## 6. End-to-end verification (all integrations enabled) After Steps 0–4: @@ -282,8 +312,9 @@ python -m opportunity_ingest interpret-rank --status New --limit 50 | Sheets **Ingest** | Yes | SA + sheet share + ID | Yes (local) | | Grok **interpret-rank** | Yes | `XAI_API_KEY` | Yes (local) | | Sheets **Ranked** | Yes | same Sheets as Ingest | Yes (local) | -| Teams ops alerts | Yes | `TEAMS_WEBHOOK_URL` | Code ready; needs your webhook | -| Teams match ≥40 | Yes | webhook + threshold | Code ready; needs your webhook | +| Teams ops alerts | Yes | `TEAMS_WEBHOOK_URL` | Code ready; requires a configured webhook | +| Teams match ≥40 | Yes | webhook + threshold | Code ready; requires a configured webhook | +| Slack match ≥40 | Yes | `SLACK_WEBHOOK_URL` + same threshold | Code ready; requires a configured webhook | | GitHub daily schedule | Yes | repo secrets + enable schedule | Not fully go-live | | SharePoint SoR | Yes | Entra + Sites.Selected | Not activated | | Type-based keyword packs / auto crawl | No | — | See roadmap | @@ -302,14 +333,16 @@ Full detail: [`BACKLOG.md`](BACKLOG.md). Summary for humans: | Near | B-01 / B-02 | Source-type classifier + keyword packs | Product priority | | Later | B-03 | Grok suggest-keywords (PR only) | Design sign-off | | Later | B-04 / B-05 | Company-site crawl / notice page crawl | Product + ToS | -| Deferred | N-* | Multi-source SoR, auto-bid, two-way Status, dual-write | Phase non-goals | +| Long horizon | V-1…V-10 | Multi-geography **lawful** source mesh + self-sustaining loop | [`BACKLOG.md`](BACKLOG.md) **§7** | +| Deferred | N-* | Unrestricted web scrape, auto-bid, dual-write, two-way Status | Project guardrails | **Suggested human order after keys work:** -1. Plug Teams webhook → prove one match card. +1. Plug Slack/Teams → prove one match card. 2. Enable Actions schedule with small `INGEST_MAX_CREATE`. 3. Weekly: review Ranked + Status triage. -4. Eng: tune keywords/objectives from real false positives. +4. Eng: tune keywords/objectives from real false positives. +5. Only then: discuss §7 second source / geography pack. --- @@ -327,11 +360,11 @@ Full detail: [`BACKLOG.md`](BACKLOG.md). Summary for humans: --- -## 10. One-page “who does what” +## 10. Roles and responsibilities | Role | Does | |------|------| -| **Human (you)** | Keys, sheet share, Teams Workflow, Status triage, bid/no-bid | +| **Operators** | Keys, sheet share, Teams Workflow, Status triage, bid/no-bid | | **Engineering** | keywords.yaml, objectives.yaml, code, tests, Actions YAML | | **System (already built)** | Download, filter, score, store, export, Sheets replace, Grok rank, Teams cards | diff --git a/docs/PROCESSOR_OVERVIEW.md b/docs/PROCESSOR_OVERVIEW.md new file mode 100644 index 0000000..62c6965 --- /dev/null +++ b/docs/PROCESSOR_OVERVIEW.md @@ -0,0 +1,220 @@ +# Opportunity Ingest — one key processor in the system + +**Audience:** operators, capture leads, stakeholders (non-engineers OK). +**As-of:** 2026-08-11 +**Package:** `opportunity_ingest` (this repo) + +This page explains **what this app is**, **where it sits** in the wider contract-bidding / capture setup, **what triggers it**, and **what comes out** — in plain language. + +| Want… | Go to | +|-------|--------| +| Keys & how to turn features on | [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) | +| Technical architecture | [`AS_BUILT.md`](AS_BUILT.md) | +| Day-to-day ops | [`../scripts/ops_runbook.md`](../scripts/ops_runbook.md) | +| What is / isn’t built | [`STATUS.md`](STATUS.md) · [`BACKLOG.md`](BACKLOG.md) | + +--- + +## 1. Plain-language description + +**In one sentence:** +This application is a **scheduled monitor for public tenders**. It downloads open notices, retains those that match configured capabilities, stores them, optionally ranks them in plain English with Grok, and can **notify Slack or Teams** when a score meets the threshold. + +**What it is not:** + +- Not the company website (e.g. Atlas Flow Group marketing site) +- Not a bot that submits bids +- Not a full CRM or proposal writer +- Not an automated agent that sets Status + +**What a human still does:** +Open the notice link, set Status (New → Reviewing → Bidding / Discarded), and choose pursue / pass. + +--- + +## 2. Where this processor sits (end-to-end) + +Capture is a sequence of stages. This repository implements **one stage**: find and alert on new public opportunities. + +```text +┌─────────────────────────────────────────────────────────────────────────┐ +│ Wider capture / bidding system (conceptual) │ +│ │ +│ [1] Public market data │ +│ CanadaBuys open tender notices (CSV open data) │ +│ │ │ +│ ▼ │ +│ [2] ★ THIS APP — Opportunity Ingest processor ★ │ +│ Filter → score → store → optional AI rank → notify │ +│ │ │ +│ ▼ │ +│ [3] Human capture workspace │ +│ SQLite / Google Sheets (Ingest + Ranked) / CSV export │ +│ Slack (or Teams) channel cards with “Open notice” │ +│ │ │ +│ ▼ │ +│ [4] Human decision (outside this app’s automation) │ +│ Status, notes, bid / no-bid, proposals, submission │ +│ │ │ +│ ▼ │ +│ [5] Company positioning (related, not the same code) │ +│ Company website → informs keywords.yaml and objectives.yaml │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +| Step | Role | This repo? | +|------|------|------------| +| **1. Market feed** | Public open-tender list | Download only | +| **2. Ingest processor** | Filter, score, store, rank, alert | **Yes — primary job** | +| **3. Workspace views** | Sheets / Slack / CSV for people | Writes **views** / messages | +| **4. Decision & bid** | Humans pursue/pass | **No auto-bid** | +| **5. Brand / capabilities site** | Company positioning | Config only (`keywords` / `objectives`) | + +--- + +## 3. What this processor does (steps inside the app) + +| Internal step | Plain language | Technical name | +|---------------|----------------|----------------| +| A. Download | Get today’s open tender CSV from CanadaBuys | `download` / `run` | +| B. Parse | Read bilingual columns; prefer English | `parse` | +| C. Keyword filter | Keep rows that match our capability words | `filter_keywords` + `config/keywords.yaml` | +| D. Rule score | Give a 0–100 “relevance” number (rules, not AI) | `score` → `RelevanceScore` | +| E. Dedupe + store | Save **new** opportunities only; never overwrite human Status | SQLite create-only | +| F. Optional Sheets Ingest | Full-replace a spreadsheet tab for browsing | `sync-sheets` | +| G. Optional Grok rank | Rewrite in plain English; score fit to company objectives | `interpret-rank` → `fit_score` | +| H. Optional Sheets Ranked | Full-replace ranked tab | `sync-rank-sheets` / auto with rank | +| I. Optional Slack / Teams | Ping channel when score ≥ threshold (default **40**) | `notify` match alerts | + +--- + +## 4. Trigger points (when this processor runs) + +Nothing runs by itself unless **you or a scheduler** starts it. + +| Trigger | How | Typical use | +|---------|-----|-------------| +| **Manual CLI** | Operator runs a command in the project folder | Smoke tests, ad-hoc refresh | +| **Windows Task Scheduler** | `scripts/daily_sync.ps1` (or similar) on a PC | Local daily job | +| **GitHub Actions schedule** | Workflow cron (e.g. ~14:00 UTC) | Unattended cloud daily | +| **Manual Actions** | `workflow_dispatch` on GitHub | One-off remote run | +| **Follow-on commands** | After ingest, human/script runs rank or sheets | Rank + Slack after store is warm | + +### Command-level triggers + +| You run… | Trigger effect | +|----------|----------------| +| `python -m opportunity_ingest run` | **Dry-run** filter/score (no DB write by default) | +| `python -m opportunity_ingest run --write` | **Processor write path:** create new DB rows; if score ≥ 40 → Slack/Teams match card | +| `python -m opportunity_ingest sync-sheets` | Push DB → Sheets **Ingest** tab | +| `python -m opportunity_ingest interpret-rank` | Grok ranks stored rows; Sheets **Ranked**; if fit ≥ 40 → Slack/Teams | +| `python -m opportunity_ingest sync-rank-sheets` | Re-push last ranking JSON to Sheets (no Grok call) | +| `python -m opportunity_ingest export-csv` | Snapshot file for Excel | +| `python -m opportunity_ingest check-store` | Health only | + +### Score thresholds that trigger alerts + +| Signal | Meaning | Default alert bar | +|--------|---------|-------------------| +| **RelevanceScore** | Rule-based match after keywords | ≥ **40** on **new creates** (`run --write`) | +| **Grok fit_score** | Fit vs `config/objectives.yaml` | ≥ **40** after `interpret-rank` | + +Configured in `config/notify.yaml` / `TEAMS_MATCH_SCORE_THRESHOLD` (shared for Slack + Teams). + +--- + +## 5. Inputs and outputs (black box) + +### Inputs + +| Input | Source | +|-------|--------| +| CanadaBuys open tender CSV | Public download or `--csv` file | +| Keyword list | `config/keywords.yaml` (eng-owned) | +| Company objectives (for Grok) | `config/objectives.yaml` | +| Secrets | Root `.env` (never committed) | +| Existing DB keys | SQLite (dedupe so we don’t double-add) | + +### Outputs + +| Output | What a human sees | +|--------|-------------------| +| SQLite DB | Master list of opportunities (`data/contract_opportunities.db`) | +| CSV export | Spreadsheet file for offline review | +| Google Sheets **Ingest** | Grid of opportunities (overwritten each sync) | +| Google Sheets **Ranked** | Grok-sorted plain-English brief | +| Local ranking files | `data/rankings/interpret-*.md` | +| **Slack channel** | Block Kit card: score, summary, **Open notice** button | +| Teams channel | Adaptive Card (ops + optional match) | +| Run logs | `logs/run-*.json` metrics | + +--- + +## 6. Slack as a notification channel + +When the processor finds a high-enough match, Slack is a **notification channel**, not a second database. + +```text +SQLite (or Grok rank result) + │ score ≥ 40 + ▼ + notify module (designed card) + │ + ▼ + Slack channel (configured via SLACK_CHANNEL_ID) + │ human clicks "Open notice" + ▼ + Tender portal page → human decides +``` + +**Setup:** bot token + channel — [`../scripts/slack_cli_setup.md`](../scripts/slack_cli_setup.md). + +--- + +## 7. Guarantees + +| Promise | Why it matters | +|---------|----------------| +| **Create-only store** | Re-running ingest does not wipe Status or Notes | +| **Dedupe by ID + link** | Same tender is not added twice | +| **Dry-run by default** | `run` without `--write` does not change the DB | +| **Sheets are views** | Spreadsheet is not the system of record | +| **Grok does not write Status** | AI ranks; humans still own triage | +| **Alerts are CTAs** | Slack/Teams open the notice; no auto-bid | + +--- + +## 8. Who owns what + +| Role | Owns | +|------|------| +| **This processor (code)** | Download, filter, score, store, rank, notify | +| **Engineering** | Keywords, objectives, notify thresholds, app code | +| **Operators** | Secrets, schedules, Status triage, bid decisions | +| **Capture lead** | Act on Slack/Sheets; pursue/pass | + +--- + +## 9. Related pages + +| Doc | Purpose | +|-----|---------| +| [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) | Keys, enable each output | +| [`AS_BUILT.md`](AS_BUILT.md) | Modules and pipeline detail | +| [`DATA_UPDATE_DIRECTIVES.md`](DATA_UPDATE_DIRECTIVES.md) | Hard rules for data | +| [`BACKLOG.md`](BACKLOG.md) §7 | **Future:** multi-geography source mesh, self-sustaining loop (lawful) | +| [`../README.md`](../README.md) | Quick start | +| [`../AGENTS.md`](../AGENTS.md) | Rules for AI coding agents | + +--- + +## 10. Future direction (not built) + +Long-term product intent is **not** “scrape the entire internet.” +It is a **lawful multi-source opportunity mesh** (any geography we enable), with human Status feeding **suggested** filter improvements — still create-only, still no auto-bid. + +Full register, architecture sketch, and decision principles: **[`BACKLOG.md`](BACKLOG.md) §7**. + +--- + +*This document is the operator overview for the ingest processor. Update it when triggers, thresholds, or outputs change.* diff --git a/docs/STATUS.md b/docs/STATUS.md index 781c432..888734a 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -8,9 +8,9 @@ Living snapshot of implementation and ops readiness. | **Last updated** | 2026-08-11 | | **Phase** | Phase 1 ingest + optional Grok interpret-rank (post-ingest) | | **Overall status** | **Ingest implemented & locally validated; Grok/Ranked coded + unit-tested; live Grok E2E pending** | -| **GitHub** | https://github.com/vikiscoding/contractbiddingsystem (**private**) | +| **GitHub** | Private repository (see `git remote -v`) | | **Default branch** | `main` | -| **Recommended next** | **Human plug-in:** [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) — Teams webhook, Actions secrets, confirm daily path. Eng roadmap: [`BACKLOG.md`](BACKLOG.md) | +| **Recommended next** | **Human plug-in:** [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md). Eng roadmap: [`BACKLOG.md`](BACKLOG.md). Long-horizon multi-geo vision: BACKLOG **§7**. | --- @@ -29,7 +29,8 @@ Living snapshot of implementation and ops readiness. | Re-push rankings without Grok | **Implemented** | `sync-rank-sheets` from latest `data/rankings/interpret-*.json` | | SharePoint Graph adapter | **Implemented, not activated** | Needs Entra + Sites.Selected grant | | Teams Workflows ops notify | **Code ready** | hard fail / partial / streak → `TEAMS_WEBHOOK_URL` | -| Teams high-match capture ping | **Implemented** | score ≥40 (config); Adaptive Card + OpenUrl CTA; `config/notify.yaml` | +| Teams high-match capture ping | **Implemented** | score ≥40; Adaptive Card + OpenUrl CTA; `config/notify.yaml` | +| Slack high-match capture ping | **Implemented** | ≥40; Slack CLI `SLACK_BOT_TOKEN` + channel via `chat.postMessage`; webhook legacy fallback | | GitHub Actions CI (`ci.yml`) | **In repo** | ruff + pytest | | GitHub Actions daily ingest | **In repo, not yet scheduled live** | Needs vars/secrets; enable after calibration | | Source/website-type keyword packs | **Not built** | Tracked: [`BACKLOG.md`](BACKLOG.md) B-01, B-02 | @@ -115,7 +116,7 @@ Create-only ingest; never silent-truncate Link; Sheets is not SoR; Grok never re ## 6. Go-live checklist (GitHub Actions) -- [x] Code on default branch (`main`) of GitHub remote (`vikiscoding/contractbiddingsystem`) +- [x] Code on default branch (`main`) of the GitHub remote - [ ] CI green on push (watch **Actions** tab after first push) - [ ] Repo variable `INGEST_MAX_CREATE=50` (or desired) - [ ] Secret `TEAMS_WEBHOOK_URL` (recommended) @@ -133,6 +134,7 @@ Create-only ingest; never silent-truncate Link; Sheets is not SoR; Grok never re | Need | Doc | |------|-----| | **Human keys + plug-and-play + function list + roadmap** | [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) | +| **Processor role, triggers, outputs** | [`PROCESSOR_OVERVIEW.md`](PROCESSOR_OVERVIEW.md) | | LLM / agent context | [`../AGENTS.md`](../AGENTS.md) | | Architecture (what exists) | [`AS_BUILT.md`](AS_BUILT.md) | | **Discussed / not built** | [`BACKLOG.md`](BACKLOG.md) | diff --git a/pyproject.toml b/pyproject.toml index 083575c..136c701 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,9 @@ sheets = [ ai = [ "openai>=1.40", ] +slack = [ + "slack-sdk>=3.27", +] [project.scripts] opportunity-ingest = "opportunity_ingest.cli:main" diff --git a/scripts/daily_sync.ps1 b/scripts/daily_sync.ps1 index 127e817..d0f372a 100644 --- a/scripts/daily_sync.ps1 +++ b/scripts/daily_sync.ps1 @@ -5,7 +5,8 @@ # powershell.exe -NoProfile -ExecutionPolicy Bypass -File "...\scripts\daily_sync.ps1" $ErrorActionPreference = "Stop" -Set-Location "C:\Users\Vikrant\Documents\Agentic AI Learning\contractbiddingsystem" +$RepoRoot = Split-Path -Parent $PSScriptRoot +Set-Location $RepoRoot if (Test-Path .\.venv\Scripts\Activate.ps1) { & .\.venv\Scripts\Activate.ps1 diff --git a/scripts/google_sheets_setup.md b/scripts/google_sheets_setup.md index 754037d..8e49cfe 100644 --- a/scripts/google_sheets_setup.md +++ b/scripts/google_sheets_setup.md @@ -26,7 +26,7 @@ Use a separate tab (e.g. **Review**) for manual notes so they are not wiped. - Python package extras for Sheets: ```powershell -cd "C:\Users\Vikrant\Documents\Agentic AI Learning\contractbiddingsystem" +cd .\.venv\Scripts\Activate.ps1 pip install -e ".[sheets]" ``` @@ -199,25 +199,16 @@ python -m opportunity_ingest sync-rank-sheets ## Part E — Daily automatic update -### Option 1 — Windows Task Scheduler (your PC) +### Option 1 — Windows Task Scheduler (operator workstation) -1. Create `scripts\daily_sync.ps1`: - -```powershell -Set-Location "C:\Users\Vikrant\Documents\Agentic AI Learning\contractbiddingsystem" -& .\.venv\Scripts\Activate.ps1 -python -m opportunity_ingest run --write --max-create 50 -if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } -python -m opportunity_ingest sync-sheets -exit $LASTEXITCODE -``` +1. Use the checked-in script `scripts\daily_sync.ps1` (it changes to the repository root from its own location). 2. Task Scheduler → **Create Basic Task** → Daily after ~9:30 AM Eastern (CanadaBuys refreshes earlier; 10:00 AM local is a safe start). 3. Action: start program `powershell.exe` Arguments: - `-NoProfile -ExecutionPolicy Bypass -File "C:\Users\Vikrant\Documents\Agentic AI Learning\contractbiddingsystem\scripts\daily_sync.ps1"` + `-NoProfile -ExecutionPolicy Bypass -File "\scripts\daily_sync.ps1"` PC must be on at that time. diff --git a/scripts/ops_runbook.md b/scripts/ops_runbook.md index 3aa28b9..b5f1ebd 100644 --- a/scripts/ops_runbook.md +++ b/scripts/ops_runbook.md @@ -5,6 +5,7 @@ Operator guide for day-1 SQLite storage, Status triage, calibration, create caps **Audience:** operators who review opportunities and manage the daily schedule. **Engineering owns:** `config/keywords.yaml`, pipeline code, GitHub workflow YAML. **First-time plug-in (keys, Grok, Teams, function list, roadmap):** **[PLUG_AND_PLAY.md](../docs/PLUG_AND_PLAY.md)**. +**What this app is (processor role, triggers, flow):** **[PROCESSOR_OVERVIEW.md](../docs/PROCESSOR_OVERVIEW.md)**. **LLM / data rules:** [AGENTS.md](../AGENTS.md), [DATA_UPDATE_DIRECTIVES](../docs/DATA_UPDATE_DIRECTIVES.md), [AS_BUILT](../docs/AS_BUILT.md). **Related:** [README.md](../README.md), [google_sheets_setup.md](google_sheets_setup.md), [provision_sharepoint_list.md](provision_sharepoint_list.md), [daily_sync.ps1](daily_sync.ps1), [BACKLOG.md](../docs/BACKLOG.md). @@ -450,8 +451,13 @@ Only **`--write`** on `run` persists opportunities. Default (and `--dry-run`) ne | `PARTIAL_ERROR_EXIT_THRESHOLD` | `5` | Exit 1 + notify | | `TEAMS_WEBHOOK_URL` | — | Ops alerts (fail/streak); also match fallback | | `TEAMS_MATCH_WEBHOOK_URL` | — | Optional dedicated capture channel | -| `TEAMS_MATCH_SCORE_THRESHOLD` | `40` | Ping when RelevanceScore or Grok fit ≥ N | -| `TEAMS_MATCH_NOTIFY_ENABLED` | `true` | Master switch for match pings | +| `TEAMS_MATCH_SCORE_THRESHOLD` | `40` | Ping when RelevanceScore or Grok fit ≥ N (Teams + Slack) | +| `TEAMS_MATCH_NOTIFY_ENABLED` | `true` | Master switch for Teams match pings | +| `SLACK_BOT_TOKEN` | — | CLI/Bolt `xoxb-` bot token (preferred) | +| `SLACK_CHANNEL_ID` | — | Channel for `chat.postMessage` | +| `SLACK_APP_TOKEN` | — | Optional `xapp-` (Socket Mode only) | +| `SLACK_WEBHOOK_URL` | — | Legacy Incoming Webhook fallback | +| `SLACK_MATCH_NOTIFY_ENABLED` | `true` | Master switch for Slack match pings | | `KEYWORDS_PATH` | `config/keywords.yaml` | Eng-owned ingest filter | | `OBJECTIVES_PATH` | `config/objectives.yaml` | Eng-owned Grok frame | | `XAI_API_KEY` | — | Required for interpret-rank | diff --git a/scripts/slack_cli_setup.md b/scripts/slack_cli_setup.md new file mode 100644 index 0000000..99f4f33 --- /dev/null +++ b/scripts/slack_cli_setup.md @@ -0,0 +1,189 @@ +# Slack setup via Slack CLI (match notifications) + +This project posts high-match opportunity alerts to Slack using the **same credential model as Slack CLI / Bolt apps**, not Incoming Webhooks (webhooks remain a legacy fallback only). + +**Docs:** [Slack CLI](https://docs.slack.dev/tools/slack-cli/) · [Authorize CLI](https://docs.slack.dev/tools/slack-cli/guides/authorizing-the-slack-cli) · [Bolt Python tokens](https://docs.slack.dev/tools/bolt-python/getting-started) + +--- + +## What you paste into this repo + +| Variable | Example | Required? | Role | +|----------|---------|-----------|------| +| `SLACK_BOT_TOKEN` | `xoxb-...` | **Yes** (preferred path) | Bot User OAuth token → `chat.postMessage` | +| `SLACK_CHANNEL_ID` | `C0123ABCD` or `#bid-alerts-test` | **Yes** (with bot token) | Where to post | +| `SLACK_APP_TOKEN` | `xapp-...` | Optional | Socket Mode / `slack run` only; **not** used for match posts | +| `SLACK_MATCH_NOTIFY_ENABLED` | `true` | Default true | Master switch | +| `TEAMS_MATCH_SCORE_THRESHOLD` | `40` | Shared with Teams | Min score 0–100 | +| `SLACK_WEBHOOK_URL` | `https://hooks.slack.com/...` | Legacy only | Fallback if bot token unset | + +Install SDK: + +```powershell +pip install -e ".[slack]" +``` + +--- + +## Path A — Slack CLI (recommended) + +### 1. Install & login + +Windows: follow [Install Slack CLI for Windows](https://docs.slack.dev/tools/slack-cli/guides/installing-the-slack-cli-for-windows). + +```powershell +slack version +slack login +# Complete /slackauthticket + challenge in Slack +slack auth list +``` + +Auth data for the **CLI itself** lives in `~/.slack/credentials.json` (do not commit). +**This Python app does not read that file** — you still paste **app** tokens into repo `.env` (below). + +### 2. Create or open a Bolt/CLI app + +Either: + +```powershell +# Optional: separate Bolt sample app to mint tokens quickly +slack create bid-alerts-app --template slack-samples/bolt-python-getting-started-app +cd bid-alerts-app +slack run +# Select create/install app when prompted +``` + +Or create an app from [api.slack.com/apps](https://api.slack.com/apps) / `slack app settings` for an existing CLI project. + +### 3. Scopes & install + +In app settings → **OAuth & Permissions** → Bot Token Scopes, ensure at least: + +- `chat:write` (required) +- `chat:write.public` (optional; post to public channels without join) + +**Reinstall** the app to the **test workspace** after adding scopes. + +Invite the bot to the target channel (if private, or if missing `chat:write.public`): + +```text +/invite @YourBotName +``` + +### 4. Copy tokens (CLI / Bolt style) + +| Token | Where in Slack app UI | +|-------|------------------------| +| **Bot User OAuth Token** `xoxb-...` | OAuth & Permissions → *Bot User OAuth Token* → env `SLACK_BOT_TOKEN` | +| **App-Level Token** `xapp-...` | Basic Information → App-Level Tokens (`connections:write`) → env `SLACK_APP_TOKEN` (only if you also run Bolt Socket Mode) | + +### 5. Channel ID + +In Slack (desktop): channel name → **View channel details** → bottom **Channel ID** `C...` +Or use `#channel-name` if the bot can resolve it. + +### 6. Paste into **repo-root** `.env` + +```env +# Slack CLI / Bolt config (preferred) +SLACK_BOT_TOKEN=xoxb-your-bot-token +SLACK_CHANNEL_ID=C0123456789 +# Optional (not used by match notify): +# SLACK_APP_TOKEN=xapp-your-app-token + +SLACK_MATCH_NOTIFY_ENABLED=true +TEAMS_MATCH_SCORE_THRESHOLD=40 + +# Quiet Teams while testing Slack: +TEAMS_MATCH_NOTIFY_ENABLED=false +``` + +### 7. Verify settings load + +```powershell +cd +.\.venv\Scripts\Activate.ps1 +pip install -e ".[slack]" + +python -c "from opportunity_ingest.config import get_settings; s=get_settings(); print('web_api', s.slack_web_api_configured()); print('token', bool(s.resolved_slack_bot_token())); print('channel', s.resolved_slack_channel_id()); print('threshold', s.teams_match_score_threshold)" +``` + +Expect: `web_api True`, token True, channel set. + +### 8. Smoke-test Web API (no full pipeline) + +```powershell +python -c @" +from opportunity_ingest.config import get_settings +from opportunity_ingest.notify import post_slack_chat_message +s = get_settings() +post_slack_chat_message( + bot_token=s.resolved_slack_bot_token(), + channel=s.resolved_slack_channel_id(), + text='contractbiddingsystem Slack CLI config OK', + blocks=None, +) +print('posted ok') +"@ +``` + +Message should appear in the configured channel. + +### 9. End-to-end match notify (lower threshold once) + +```env +TEAMS_MATCH_SCORE_THRESHOLD=1 +``` + +```powershell +python -m opportunity_ingest interpret-rank --status New --limit 5 --no-sync-sheets +``` + +Console should show `slack=True`. Then set threshold back to **40**. + +--- + +## Path B — Tokens without full `slack create` + +If you already have a test app: + +1. Install app to test workspace with `chat:write`. +2. Copy `xoxb-` bot token → `SLACK_BOT_TOKEN`. +3. Set `SLACK_CHANNEL_ID`. +4. Same verify steps §7–9. + +This is still the **CLI/Bolt env contract** (`SLACK_BOT_TOKEN` + channel), even if you did not use `slack create`. + +--- + +## Legacy Incoming Webhook (not preferred) + +If only a `hooks.slack.com` URL exists, set: + +```env +SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... +``` + +The app logs a **warning** and uses the webhook. Migrate to bot token when possible. + +--- + +## Troubleshooting + +| Error / symptom | Fix | +|-----------------|-----| +| `slack-sdk not installed` | `pip install -e ".[slack]"` | +| `not_authed` / `invalid_auth` | Wrong/revoked bot token; reinstall app | +| `channel_not_found` | Bad `SLACK_CHANNEL_ID`; use `C...` ID | +| `not_in_channel` | `/invite @bot` into channel | +| `missing_scope` | Add `chat:write`, reinstall app | +| `web_api False` | Both token **and** channel required | +| No post, threshold | Lower `TEAMS_MATCH_SCORE_THRESHOLD` for test | + +--- + +## Security + +- Never commit `.env`, tokens, or `~/.slack/credentials.json`. +- Prefer a **test workspace** / [developer sandbox](https://docs.slack.dev/tools/developer-sandboxes). +- Rotate bot tokens if leaked. diff --git a/src/opportunity_ingest/cli.py b/src/opportunity_ingest/cli.py index 93ea1ef..d0a8c75 100644 --- a/src/opportunity_ingest/cli.py +++ b/src/opportunity_ingest/cli.py @@ -33,7 +33,10 @@ run_interpret_rank, ) from opportunity_ingest.logging_setup import setup_logging -from opportunity_ingest.notify import match_items_from_ranked, notify_match_alerts +from opportunity_ingest.notify import ( + dispatch_match_notifications, + match_items_from_ranked, +) from opportunity_ingest.pipeline import resolve_write_mode, run_pipeline from opportunity_ingest.sheets_sync import ( DEFAULT_RANK_TAB, @@ -113,6 +116,7 @@ def cmd_run( f"streak={metrics.consecutive_zero_new_days} " f"exit={metrics.exit_code} notified={metrics.notified} " f"match_notified={metrics.match_notified} " + f"slack_match_notified={metrics.slack_match_notified} " f"match_count={metrics.match_notify_count}" ) if metrics.hard_fail and metrics.hard_fail_reason: @@ -418,23 +422,35 @@ def cmd_interpret_rank( return EXIT_FAILURE print(f"Synced {count} ranking rows to sheet {sid} tab {tab!r}") - # Teams capture ping for Grok fits at/above threshold (default 40). - skip_teams = bool(getattr(args, "no_teams", False)) - if settings.teams_match_notify_enabled and not skip_teams: + # Teams + Slack capture pings for Grok fits at/above threshold (default 40). + skip_chat = bool(getattr(args, "no_teams", False)) + if ( + not skip_chat + and ( + settings.teams_match_notify_enabled or settings.slack_match_notify_enabled + ) + ): thr = int(settings.teams_match_score_threshold) match_items = match_items_from_ranked(result.ranked, threshold=thr) - posted = notify_match_alerts( - settings.resolved_match_webhook_url(), + dispatched = dispatch_match_notifications( match_items, threshold=thr, source="interpret-rank", run_url=settings.github_run_url, + sheets_url=settings.resolved_google_sheet_url(), + sheets_tab=settings.google_sheet_rank_tab or "Ranked", max_items=int(settings.teams_match_max_items), - enabled=True, + teams_webhook_url=settings.resolved_match_webhook_url(), + teams_enabled=bool(settings.teams_match_notify_enabled), + slack_bot_token=settings.resolved_slack_bot_token(), + slack_channel_id=settings.resolved_slack_channel_id(), + slack_webhook_url=settings.resolved_slack_match_webhook_url(), + slack_enabled=bool(settings.slack_match_notify_enabled), ) print( - f"teams match_notify posted={posted} " - f"above_threshold={len(match_items)} threshold={thr}" + f"match_notify teams={dispatched.teams_posted} " + f"slack={dispatched.slack_posted} " + f"above_threshold={dispatched.match_count} threshold={thr}" ) return 0 diff --git a/src/opportunity_ingest/config.py b/src/opportunity_ingest/config.py index c8cb92c..1717f8d 100644 --- a/src/opportunity_ingest/config.py +++ b/src/opportunity_ingest/config.py @@ -85,11 +85,41 @@ class Settings(BaseSettings): ) teams_match_max_items: int = Field( default=8, - description="Max opportunities listed on one match Adaptive Card", + description="Max opportunities listed on one match Adaptive Card / Slack msg", + ) + # Slack CLI / Bolt credentials (preferred for match pings) + # https://docs.slack.dev/tools/slack-cli/ — paste tokens from CLI-created app + slack_bot_token: str | None = Field( + default=None, + description="Bot User OAuth Token xoxb-... (Slack CLI / Bolt SLACK_BOT_TOKEN)", + ) + slack_app_token: str | None = Field( + default=None, + description=( + "App-level token xapp-... (SLACK_APP_TOKEN; Socket Mode / slack run). " + "Not required for outbound chat.postMessage match alerts." + ), + ) + slack_channel_id: str | None = Field( + default=None, + description="Channel ID C... or #name for match posts (chat.postMessage)", + ) + # Legacy Incoming Webhooks (fallback if bot token unset) + slack_webhook_url: str | None = Field( + default=None, + description="Legacy Incoming Webhook URL (prefer SLACK_BOT_TOKEN)", + ) + slack_match_webhook_url: str | None = Field( + default=None, + description="Legacy dedicated match webhook; falls back to SLACK_WEBHOOK_URL", + ) + slack_match_notify_enabled: bool = Field( + default=True, + description="Post Slack message when new matches meet score threshold", ) notify_config_path: Path = Field( default=Path("config/notify.yaml"), - description="Optional YAML for Teams match notify defaults", + description="Optional YAML for Teams/Slack match notify defaults", ) github_run_url: str | None = Field( default=None, @@ -108,6 +138,13 @@ class Settings(BaseSettings): default=None, description="Spreadsheet ID from the sheet URL", ) + google_sheet_url: str | None = Field( + default=None, + description=( + "Optional full spreadsheet share/deep link for Teams/Slack cards; " + "when unset, built from GOOGLE_SHEET_ID" + ), + ) google_sheet_tab: str = Field( default="Ingest", description="Worksheet tab name fully replaced on each opportunity sync", @@ -163,15 +200,53 @@ def resolved_sqlite_path(self) -> Path: return self.data_dir / "contract_opportunities.db" def resolved_match_webhook_url(self) -> str | None: - """Webhook for high-match pings (dedicated URL or ops fallback).""" + """Teams webhook for high-match pings (dedicated URL or ops fallback).""" for raw in (self.teams_match_webhook_url, self.teams_webhook_url): if raw and str(raw).strip(): return str(raw).strip() return None + def resolved_google_sheet_url(self) -> str | None: + """Browser URL for the opportunity spreadsheet (match-card CTA). + + Uses ``GOOGLE_SHEET_URL`` when set; otherwise + ``https://docs.google.com/spreadsheets/d/{GOOGLE_SHEET_ID}/edit``. + """ + if self.google_sheet_url and str(self.google_sheet_url).strip(): + u = str(self.google_sheet_url).strip() + if u.startswith(("http://", "https://")): + return u + sid = (self.google_sheet_id or "").strip() + if not sid: + return None + return f"https://docs.google.com/spreadsheets/d/{sid}/edit" + + def resolved_slack_match_webhook_url(self) -> str | None: + """Legacy Incoming Webhook for match pings (if bot token path unused).""" + for raw in (self.slack_match_webhook_url, self.slack_webhook_url): + if raw and str(raw).strip(): + return str(raw).strip() + return None + + def resolved_slack_bot_token(self) -> str | None: + """Bot token for Web API (Slack CLI / Bolt ``SLACK_BOT_TOKEN``).""" + if self.slack_bot_token and str(self.slack_bot_token).strip(): + return str(self.slack_bot_token).strip() + return None + + def resolved_slack_channel_id(self) -> str | None: + """Target channel for ``chat.postMessage``.""" + if self.slack_channel_id and str(self.slack_channel_id).strip(): + return str(self.slack_channel_id).strip() + return None + + def slack_web_api_configured(self) -> bool: + """True when Slack CLI-style bot token + channel are both set.""" + return bool(self.resolved_slack_bot_token() and self.resolved_slack_channel_id()) + def load_notify_yaml_overrides(path: Path | str | None = None) -> dict[str, object]: - """Load optional config/notify.yaml teams section (soft; empty if missing).""" + """Load optional config/notify.yaml teams/slack sections (soft; empty if missing).""" import yaml p = Path(path) if path is not None else Path("config/notify.yaml") @@ -183,29 +258,37 @@ def load_notify_yaml_overrides(path: Path | str | None = None) -> dict[str, obje return {} if not isinstance(raw, dict): return {} - teams = raw.get("teams") - if not isinstance(teams, dict): - return {} out: dict[str, object] = {} - if "match_notify_enabled" in teams: - out["teams_match_notify_enabled"] = bool(teams["match_notify_enabled"]) - if "match_score_threshold" in teams: + + # Shared threshold / max items (root or teams — teams keeps back-compat) + shared = raw.get("match") if isinstance(raw.get("match"), dict) else {} + teams = raw.get("teams") if isinstance(raw.get("teams"), dict) else {} + slack = raw.get("slack") if isinstance(raw.get("slack"), dict) else {} + + thr = shared.get("score_threshold", teams.get("match_score_threshold")) + if thr is not None: try: - out["teams_match_score_threshold"] = int(teams["match_score_threshold"]) + out["teams_match_score_threshold"] = int(thr) except (TypeError, ValueError): pass - if "max_items_per_message" in teams: + max_items = shared.get("max_items_per_message", teams.get("max_items_per_message")) + if max_items is not None: try: - out["teams_match_max_items"] = int(teams["max_items_per_message"]) + out["teams_match_max_items"] = int(max_items) except (TypeError, ValueError): pass + + if "match_notify_enabled" in teams: + out["teams_match_notify_enabled"] = bool(teams["match_notify_enabled"]) + if "match_notify_enabled" in slack: + out["slack_match_notify_enabled"] = bool(slack["match_notify_enabled"]) return out def get_settings() -> Settings: """Load settings from environment / ``.env``, with soft notify.yaml overrides. - Env vars always win for secrets and when ``TEAMS_MATCH_*`` is set. + Env vars always win for secrets and when ``TEAMS_MATCH_*`` / ``SLACK_*`` is set. ``config/notify.yaml`` fills threshold/enabled/max items when those env keys are unset. """ @@ -221,6 +304,7 @@ def get_settings() -> Settings: "teams_match_notify_enabled": "TEAMS_MATCH_NOTIFY_ENABLED", "teams_match_score_threshold": "TEAMS_MATCH_SCORE_THRESHOLD", "teams_match_max_items": "TEAMS_MATCH_MAX_ITEMS", + "slack_match_notify_enabled": "SLACK_MATCH_NOTIFY_ENABLED", } for field_name, env_name in env_map.items(): if env_name not in os.environ and field_name in yaml_over: diff --git a/src/opportunity_ingest/notify.py b/src/opportunity_ingest/notify.py index 4c7713e..a291b5e 100644 --- a/src/opportunity_ingest/notify.py +++ b/src/opportunity_ingest/notify.py @@ -1,26 +1,30 @@ -"""Teams Workflows webhook notifications (Adaptive Card). +"""Channel notifications: Teams Workflows + Slack Incoming Webhooks. -Two pipelines share this module: +Pipelines: -1. **Ops alerts** — hard fail, partial errors, zero-new streak +1. **Ops alerts (Teams)** — hard fail, partial errors, zero-new streak (``notify_ingest_alert`` / ``TEAMS_WEBHOOK_URL``). 2. **Match alerts** — new opportunities or Grok ranks at/above score threshold - (``notify_match_alerts`` / ``TEAMS_MATCH_WEBHOOK_URL`` or ops fallback). - Call-to-action cards include truncated summaries + OpenUrl actions. - -On successful ops notify POST, sets GitHub Actions step output ``notified=true`` -so the workflow backup step can skip double-notify + (default **40/100**): + - Teams Adaptive Card: ``notify_match_alerts`` / ``TEAMS_MATCH_WEBHOOK_URL`` + - Slack (preferred): **Slack CLI / Bolt config** — ``SLACK_BOT_TOKEN`` (``xoxb-``) + + ``SLACK_CHANNEL_ID`` via Web API ``chat.postMessage`` (Block Kit). + Optional ``SLACK_APP_TOKEN`` (``xapp-``) is documented for CLI/Bolt apps but + not required for outbound posts. + - Slack (legacy fallback): Incoming Webhook URL if bot token unset. + - Multi-channel: ``dispatch_match_notifications`` + - Optional Google Sheets link (``GOOGLE_SHEET_ID`` / ``GOOGLE_SHEET_URL``) + as fact + OpenUrl / button on the card. + +Match cards include truncated summaries + open-notice CTAs (buttons / OpenUrl). + +On successful **Teams ops** notify POST, sets GitHub Actions step output +``notified=true`` so the workflow backup step can skip double-notify (``failure() && steps.ingest.outputs.notified != 'true'``). -**GITHUB_OUTPUT ownership (KD-18):** When ``GITHUB_OUTPUT`` is set, notify +**GITHUB_OUTPUT ownership (KD-18):** When ``GITHUB_OUTPUT`` is set, ops notify ownership is complete only if that file is written after a successful webhook -POST. Write is retried once; persistent failure logs at ERROR and returns -``False`` (does not claim ``notified=true``). In that edge case Actions may -still double-notify on job failure — operators should treat GITHUB_OUTPUT I/O -errors as critical. When ``GITHUB_OUTPUT`` is unset (local runs), POST success -alone is enough. - -Match alerts do **not** set ``notified=true`` (ops ownership handoff only). +POST. Match alerts (Teams or Slack) do **not** set ``notified=true``. """ from __future__ import annotations @@ -31,7 +35,7 @@ import re import urllib.error import urllib.request -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any, Mapping, Sequence logger = logging.getLogger(__name__) @@ -41,15 +45,17 @@ DEFAULT_MATCH_MAX_ITEMS = 8 DEFAULT_TITLE_MAX = 120 DEFAULT_SUMMARY_MAX = 280 +# Slack allows many blocks but keep action buttons modest. +SLACK_MAX_ACTION_BUTTONS = 5 class NotifyError(Exception): - """Teams webhook or GITHUB_OUTPUT handoff failure (non-fatal to exit matrix).""" + """Webhook or GITHUB_OUTPUT handoff failure (non-fatal to exit matrix).""" @dataclass(frozen=True, slots=True) class MatchAlertItem: - """One high-match opportunity for a Teams capture ping.""" + """One high-match opportunity for capture-channel pings (Teams and/or Slack).""" title: str opportunity_id: str @@ -63,6 +69,17 @@ class MatchAlertItem: closing_date: str | None = None +@dataclass(slots=True) +class MatchNotifyResult: + """Outcome of multi-channel match dispatch.""" + + match_count: int = 0 + teams_posted: bool = False + slack_posted: bool = False + any_posted: bool = False + details: dict[str, str] = field(default_factory=dict) + + def set_github_output_notified(notified: bool = True) -> bool: """Append ``notified=true|false`` to ``GITHUB_OUTPUT`` when running in Actions. @@ -173,21 +190,46 @@ def filter_match_items( return filtered +def google_sheet_edit_url( + spreadsheet_id: str | None, + *, + override_url: str | None = None, +) -> str | None: + """Build a browser URL for the workbook (Teams/Slack CTA). + + Prefer ``override_url`` when set (full share/deep link). Otherwise + ``https://docs.google.com/spreadsheets/d/{id}/edit`` from the sheet id. + """ + if override_url and str(override_url).strip(): + u = str(override_url).strip() + if u.startswith(("http://", "https://")): + return u + sid = (spreadsheet_id or "").strip() + if not sid: + return None + return f"https://docs.google.com/spreadsheets/d/{sid}/edit" + + def build_match_alert_payload( items: Sequence[MatchAlertItem], *, threshold: int = DEFAULT_MATCH_THRESHOLD, source: str = "ingest", run_url: str | None = None, + sheets_url: str | None = None, + sheets_tab: str | None = None, max_items: int = DEFAULT_MATCH_MAX_ITEMS, title_max_chars: int = DEFAULT_TITLE_MAX, summary_max_chars: int = DEFAULT_SUMMARY_MAX, card_title: str | None = None, cta_label: str = "Open notice", + sheets_cta_label: str = "Open Google Sheet", ) -> dict[str, Any]: """Adaptive Card: high-match opportunities with summary + OpenUrl CTAs. ``source`` is ``ingest`` (rule score) or ``interpret-rank`` (Grok fit). + When ``sheets_url`` is set, the card includes a spreadsheet fact + action + so channel members can open the full grid (Ingest / Ranked). """ matched = filter_match_items(items, threshold=threshold) if not matched: @@ -212,6 +254,10 @@ def build_match_alert_payload( ) if hidden > 0: intro += f" Showing top {len(shown)}; **{hidden}** more not listed." + sheet_link = (sheets_url or "").strip() + if sheet_link.startswith(("http://", "https://")): + tab_hint = f" (tab **{sheets_tab}**)" if (sheets_tab or "").strip() else "" + intro += f" Full list in Google Sheets{tab_hint}." extra_body: list[dict[str, Any]] = [] actions: list[dict[str, Any]] = [] @@ -261,6 +307,19 @@ def build_match_alert_payload( ] if run_url: facts.append({"title": "Run", "value": run_url}) + if sheet_link.startswith(("http://", "https://")): + facts.append({"title": "Google Sheet", "value": sheet_link}) + if (sheets_tab or "").strip(): + facts.append({"title": "Sheet tab", "value": str(sheets_tab).strip()}) + # Prefer sheet CTA first so it survives action truncation. + actions.insert( + 0, + { + "type": "Action.OpenUrl", + "title": _truncate(sheets_cta_label, 40) or "Open Google Sheet", + "url": sheet_link, + }, + ) # Adaptive Cards: too many actions can be dropped by Teams; keep first 6. return build_adaptive_card_payload( @@ -298,37 +357,121 @@ def build_ingest_alert_payload( return build_adaptive_card_payload(title=title, facts=facts) -def post_teams_webhook( +def post_json_webhook( webhook_url: str, payload: Mapping[str, Any], *, timeout: float = DEFAULT_TIMEOUT_SECONDS, + channel_label: str = "webhook", ) -> None: - """POST JSON payload to Teams Workflows webhook URL. + """POST JSON payload to an Incoming Webhook / Workflows URL. Raises ``NotifyError`` on transport / non-2xx responses. """ if not webhook_url or not webhook_url.strip(): - raise NotifyError("TEAMS_WEBHOOK_URL is empty") + raise NotifyError(f"{channel_label} URL is empty") data = json.dumps(payload).encode("utf-8") req = urllib.request.Request( webhook_url.strip(), data=data, - headers={"Content-Type": "application/json"}, + headers={"Content-Type": "application/json; charset=utf-8"}, method="POST", ) try: with urllib.request.urlopen(req, timeout=timeout) as resp: status = getattr(resp, "status", None) or resp.getcode() if status is not None and int(status) >= 300: - raise NotifyError(f"Teams webhook returned HTTP {status}") + raise NotifyError(f"{channel_label} returned HTTP {status}") except urllib.error.HTTPError as exc: - raise NotifyError(f"Teams webhook HTTP error: {exc.code} {exc.reason}") from exc + raise NotifyError( + f"{channel_label} HTTP error: {exc.code} {exc.reason}" + ) from exc except urllib.error.URLError as exc: - raise NotifyError(f"Teams webhook request failed: {exc.reason}") from exc + raise NotifyError(f"{channel_label} request failed: {exc.reason}") from exc except TimeoutError as exc: - raise NotifyError("Teams webhook request timed out") from exc + raise NotifyError(f"{channel_label} request timed out") from exc + + +def post_teams_webhook( + webhook_url: str, + payload: Mapping[str, Any], + *, + timeout: float = DEFAULT_TIMEOUT_SECONDS, +) -> None: + """POST JSON payload to Teams Workflows webhook URL.""" + post_json_webhook( + webhook_url, payload, timeout=timeout, channel_label="Teams webhook" + ) + + +def post_slack_webhook( + webhook_url: str, + payload: Mapping[str, Any], + *, + timeout: float = DEFAULT_TIMEOUT_SECONDS, +) -> None: + """POST JSON to a legacy Slack Incoming Webhook URL.""" + post_json_webhook( + webhook_url, payload, timeout=timeout, channel_label="Slack webhook" + ) + + +def post_slack_chat_message( + *, + bot_token: str, + channel: str, + text: str, + blocks: Sequence[Mapping[str, Any]] | None = None, + timeout: float = DEFAULT_TIMEOUT_SECONDS, +) -> None: + """Post via Slack Web API ``chat.postMessage`` (Slack CLI / Bolt bot token). + + Requires ``slack-sdk`` (``pip install -e ".[slack]"``) and a bot token + ``xoxb-...`` with ``chat:write``, installed to the workspace. Channel is + an ID (``C...`` / ``G...``) or public channel name (``#alerts``). + + See: https://docs.slack.dev/tools/slack-cli/ and Bolt Python env vars + ``SLACK_BOT_TOKEN`` / ``SLACK_APP_TOKEN``. + """ + token = (bot_token or "").strip() + ch = (channel or "").strip() + if not token: + raise NotifyError("SLACK_BOT_TOKEN is empty") + if not token.startswith("xoxb-"): + logger.warning( + "SLACK_BOT_TOKEN does not start with xoxb-; expected Slack CLI / " + "Bolt bot user OAuth token" + ) + if not ch: + raise NotifyError("SLACK_CHANNEL_ID is empty") + + try: + from slack_sdk import WebClient + from slack_sdk.errors import SlackApiError + except ImportError as exc: + raise NotifyError( + 'slack-sdk not installed; run: pip install -e ".[slack]"' + ) from exc + + client = WebClient(token=token, timeout=timeout) + try: + resp = client.chat_postMessage( + channel=ch, + text=text, + blocks=list(blocks) if blocks else None, + ) + except SlackApiError as exc: + err = "" + if exc.response is not None: + err = str(exc.response.get("error") or exc.response) + raise NotifyError(f"Slack chat.postMessage failed: {err or exc}") from exc + except Exception as exc: # noqa: BLE001 — network / SDK + raise NotifyError(f"Slack Web API request failed: {exc}") from exc + + if not getattr(resp, "get", lambda _k, _d=None: None)("ok", True): + # slack_sdk returns SlackResponse mapping-like + raise NotifyError(f"Slack chat.postMessage not ok: {resp}") def notify_ingest_alert( @@ -386,6 +529,124 @@ def notify_ingest_alert( return True +def build_slack_match_payload( + items: Sequence[MatchAlertItem], + *, + threshold: int = DEFAULT_MATCH_THRESHOLD, + source: str = "ingest", + run_url: str | None = None, + sheets_url: str | None = None, + sheets_tab: str | None = None, + max_items: int = DEFAULT_MATCH_MAX_ITEMS, + title_max_chars: int = DEFAULT_TITLE_MAX, + summary_max_chars: int = DEFAULT_SUMMARY_MAX, + card_title: str | None = None, + cta_label: str = "Open notice", + sheets_cta_label: str = "Open Google Sheet", +) -> dict[str, Any]: + """Slack Block Kit payload for high-match opportunities (Incoming Webhook).""" + matched = filter_match_items(items, threshold=threshold) + if not matched: + raise NotifyError("no match items at or above threshold") + + cap = max(1, int(max_items)) + shown = matched[:cap] + hidden = len(matched) - len(shown) + + if card_title: + header = card_title + elif source == "interpret-rank": + header = f"Grok fit matches (≥{threshold})" + else: + header = f"New opportunity matches (≥{threshold})" + + # Slack header plain_text max ~150 chars + header = _truncate(header, 140) + + intro = ( + f"*{len(matched)}* high-match opportunit" + f"{'y' if len(matched) == 1 else 'ies'} " + f"(threshold *{threshold}/100*, source `{source}`). " + "Review links and act — store Status stays human-owned." + ) + if hidden > 0: + intro += f" Showing top {len(shown)}; *{hidden}* more not listed." + if run_url: + intro += f"\nRun: <{run_url}|workflow>" + sheet_link = (sheets_url or "").strip() + if sheet_link.startswith(("http://", "https://")): + tab_hint = f" (tab *{sheets_tab}*)" if (sheets_tab or "").strip() else "" + intro += f"\nFull list: <{sheet_link}|Google Sheet>{tab_hint}" + + blocks: list[dict[str, Any]] = [ + { + "type": "header", + "text": {"type": "plain_text", "text": header, "emoji": True}, + }, + {"type": "section", "text": {"type": "mrkdwn", "text": intro}}, + {"type": "divider"}, + ] + + for idx, item in enumerate(shown, start=1): + t = _truncate(item.title, title_max_chars) or item.opportunity_id + summary = _truncate(item.summary, summary_max_chars) + buyer = _truncate(item.buyer, 80) or "—" + kws = _truncate(item.keywords, 100) or "—" + rec = (item.recommendation or "").strip() + link = (item.link or "").strip() + title_line = f"*<{link}|{t}>*" if link.startswith("http") else f"*{t}*" + rec_bit = f" · `{rec}`" if rec else "" + body = ( + f"*#{idx} · {item.score}/100* ({item.score_kind}){rec_bit}\n" + f"{title_line}\n" + f"Buyer: {buyer} · ID: `{item.opportunity_id}`\n" + f"Keywords: {kws}" + ) + if item.closing_date: + body += f"\nClosing: {item.closing_date}" + if summary: + body += f"\n>{summary}" + blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": body}}) + + # Action buttons (Slack limit: keep few). Sheet button first when present. + elements: list[dict[str, Any]] = [] + if sheet_link.startswith(("http://", "https://")): + elements.append( + { + "type": "button", + "text": { + "type": "plain_text", + "text": _truncate(sheets_cta_label, 75) or "Open Google Sheet", + "emoji": True, + }, + "url": sheet_link, + "action_id": "open_google_sheet", + } + ) + notice_budget = max(0, SLACK_MAX_ACTION_BUTTONS - len(elements)) + for idx, item in enumerate(shown[:notice_budget], start=1): + link = (item.link or "").strip() + if not link.startswith(("http://", "https://")): + continue + elements.append( + { + "type": "button", + "text": { + "type": "plain_text", + "text": _truncate(f"{cta_label} #{idx}", 75), + "emoji": True, + }, + "url": link, + "action_id": f"open_notice_{idx}", + } + ) + if elements: + blocks.append({"type": "actions", "elements": elements}) + + fallback = f"{header}: {len(matched)} match(es) ≥ {threshold}" + return {"text": fallback, "blocks": blocks} + + def notify_match_alerts( webhook_url: str | None, items: Sequence[MatchAlertItem], @@ -393,6 +654,8 @@ def notify_match_alerts( threshold: int = DEFAULT_MATCH_THRESHOLD, source: str = "ingest", run_url: str | None = None, + sheets_url: str | None = None, + sheets_tab: str | None = None, max_items: int = DEFAULT_MATCH_MAX_ITEMS, enabled: bool = True, timeout: float = DEFAULT_TIMEOUT_SECONDS, @@ -401,18 +664,18 @@ def notify_match_alerts( card_title: str | None = None, cta_label: str = "Open notice", ) -> bool: - """Post a capture-channel Adaptive Card for high-match opportunities. + """Post a Teams capture-channel Adaptive Card for high-match opportunities. Returns True when a card was POSTed successfully. Returns False when disabled, no webhook, no items above threshold, or POST fails. Does **not** write GITHUB_OUTPUT (ops double-notify handoff is separate). """ if not enabled: - logger.info("Match notify disabled; skipping (source=%s)", source) + logger.info("Teams match notify disabled; skipping (source=%s)", source) return False if not webhook_url or not str(webhook_url).strip(): logger.warning( - "Match notify skipped (source=%s): no TEAMS_MATCH_WEBHOOK_URL / " + "Teams match notify skipped (source=%s): no TEAMS_MATCH_WEBHOOK_URL / " "TEAMS_WEBHOOK_URL", source, ) @@ -421,7 +684,7 @@ def notify_match_alerts( matched = filter_match_items(items, threshold=threshold) if not matched: logger.info( - "Match notify: no items with score>=%s (source=%s, candidates=%s)", + "Teams match notify: no items with score>=%s (source=%s, candidates=%s)", threshold, source, len(items), @@ -434,6 +697,8 @@ def notify_match_alerts( threshold=threshold, source=source, run_url=run_url, + sheets_url=sheets_url, + sheets_tab=sheets_tab, max_items=max_items, title_max_chars=title_max_chars, summary_max_chars=summary_max_chars, @@ -446,14 +711,186 @@ def notify_match_alerts( return False logger.info( - "Teams match notify sent (source=%s, matches=%s, threshold=%s)", + "Teams match notify sent (source=%s, matches=%s, threshold=%s, sheets=%s)", + source, + len(matched), + threshold, + bool(sheets_url), + ) + return True + + +def notify_slack_match_alerts( + items: Sequence[MatchAlertItem], + *, + bot_token: str | None = None, + channel_id: str | None = None, + webhook_url: str | None = None, + threshold: int = DEFAULT_MATCH_THRESHOLD, + source: str = "ingest", + run_url: str | None = None, + sheets_url: str | None = None, + sheets_tab: str | None = None, + max_items: int = DEFAULT_MATCH_MAX_ITEMS, + enabled: bool = True, + timeout: float = DEFAULT_TIMEOUT_SECONDS, + title_max_chars: int = DEFAULT_TITLE_MAX, + summary_max_chars: int = DEFAULT_SUMMARY_MAX, + card_title: str | None = None, + cta_label: str = "Open notice", +) -> bool: + """Post a Slack Block Kit message for high-match opportunities. + + **Preferred (Slack CLI / Bolt):** ``bot_token`` (``SLACK_BOT_TOKEN``) + + ``channel_id`` (``SLACK_CHANNEL_ID``) → Web API ``chat.postMessage``. + + **Legacy fallback:** Incoming Webhook ``webhook_url`` if bot token unset. + + Returns True on successful post. Does not set GITHUB_OUTPUT. + """ + if not enabled: + logger.info("Slack match notify disabled; skipping (source=%s)", source) + return False + + matched = filter_match_items(items, threshold=threshold) + if not matched: + logger.info( + "Slack match notify: no items with score>=%s (source=%s, candidates=%s)", + threshold, + source, + len(items), + ) + return False + + try: + payload = build_slack_match_payload( + matched, + threshold=threshold, + source=source, + run_url=run_url, + sheets_url=sheets_url, + sheets_tab=sheets_tab, + max_items=max_items, + title_max_chars=title_max_chars, + summary_max_chars=summary_max_chars, + card_title=card_title, + cta_label=cta_label, + ) + except NotifyError as exc: + logger.error("Slack match payload failed (source=%s): %s", source, exc) + return False + + token = (bot_token or "").strip() + channel = (channel_id or "").strip() + hook = (webhook_url or "").strip() + + try: + if token and channel: + post_slack_chat_message( + bot_token=token, + channel=channel, + text=str(payload.get("text") or "Opportunity matches"), + blocks=payload.get("blocks"), # type: ignore[arg-type] + timeout=timeout, + ) + mode = "web_api" + elif hook: + logger.warning( + "Slack match using legacy Incoming Webhook; prefer Slack CLI " + "SLACK_BOT_TOKEN + SLACK_CHANNEL_ID (see scripts/slack_cli_setup.md)" + ) + post_slack_webhook(hook, payload, timeout=timeout) + mode = "webhook" + else: + logger.warning( + "Slack match notify skipped (source=%s): set SLACK_BOT_TOKEN + " + "SLACK_CHANNEL_ID (Slack CLI/Bolt) or legacy SLACK_WEBHOOK_URL", + source, + ) + return False + except NotifyError as exc: + logger.error("Slack match notify failed (source=%s): %s", source, exc) + return False + + logger.info( + "Slack match notify sent (source=%s, mode=%s, matches=%s, threshold=%s, sheets=%s)", source, + mode, len(matched), threshold, + bool(sheets_url), ) return True +def dispatch_match_notifications( + items: Sequence[MatchAlertItem], + *, + threshold: int = DEFAULT_MATCH_THRESHOLD, + source: str = "ingest", + run_url: str | None = None, + sheets_url: str | None = None, + sheets_tab: str | None = None, + max_items: int = DEFAULT_MATCH_MAX_ITEMS, + teams_webhook_url: str | None = None, + teams_enabled: bool = True, + slack_bot_token: str | None = None, + slack_channel_id: str | None = None, + slack_webhook_url: str | None = None, + slack_enabled: bool = True, + timeout: float = DEFAULT_TIMEOUT_SECONDS, +) -> MatchNotifyResult: + """Send high-match alerts to Teams and/or Slack (same score criteria). + + Slack prefers bot token + channel (CLI/Bolt); webhook is legacy fallback. + When ``sheets_url`` is set, cards include an Open Google Sheet CTA. + """ + matched = filter_match_items(items, threshold=threshold) + result = MatchNotifyResult(match_count=len(matched)) + if not matched: + result.details["reason"] = "no_items_above_threshold" + return result + + if teams_enabled: + result.teams_posted = notify_match_alerts( + teams_webhook_url, + matched, + threshold=threshold, + source=source, + run_url=run_url, + sheets_url=sheets_url, + sheets_tab=sheets_tab, + max_items=max_items, + enabled=True, + timeout=timeout, + ) + result.details["teams"] = "posted" if result.teams_posted else "failed_or_skipped" + else: + result.details["teams"] = "disabled" + + if slack_enabled: + result.slack_posted = notify_slack_match_alerts( + matched, + bot_token=slack_bot_token, + channel_id=slack_channel_id, + webhook_url=slack_webhook_url, + threshold=threshold, + source=source, + run_url=run_url, + sheets_url=sheets_url, + sheets_tab=sheets_tab, + max_items=max_items, + enabled=True, + timeout=timeout, + ) + result.details["slack"] = "posted" if result.slack_posted else "failed_or_skipped" + else: + result.details["slack"] = "disabled" + + result.any_posted = bool(result.teams_posted or result.slack_posted) + return result + + def match_items_from_opportunity_fields( fields_list: Sequence[Any], *, diff --git a/src/opportunity_ingest/pipeline.py b/src/opportunity_ingest/pipeline.py index a81c0c3..2c393df 100644 --- a/src/opportunity_ingest/pipeline.py +++ b/src/opportunity_ingest/pipeline.py @@ -28,9 +28,9 @@ from opportunity_ingest.map_fields import MapError, map_to_opportunity_fields from opportunity_ingest.models import ExistingKeys, OpportunityFields from opportunity_ingest.notify import ( + dispatch_match_notifications, match_items_from_opportunity_fields, notify_ingest_alert, - notify_match_alerts, ) from opportunity_ingest.parse import ParseError, parse_csv_file, parse_csv_text from opportunity_ingest.score import compute_score @@ -83,6 +83,7 @@ class RunMetrics: notify_reason: str | None = None match_notified: bool = False match_notify_count: int = 0 + slack_match_notified: bool = False csv_source: str | None = None extra: dict[str, Any] = field(default_factory=dict) @@ -465,28 +466,38 @@ def run_pipeline( extra_facts=extra, ) - # High-match capture pings (new creates only; dry-run never posts). - if write and created_fields and settings.teams_match_notify_enabled: + # High-match capture pings → Teams and/or Slack (new creates only). + if write and created_fields and ( + settings.teams_match_notify_enabled or settings.slack_match_notify_enabled + ): thr = int(settings.teams_match_score_threshold) match_items = match_items_from_opportunity_fields( created_fields, threshold=thr ) metrics.match_notify_count = len(match_items) if match_items: - metrics.match_notified = notify_match_alerts( - settings.resolved_match_webhook_url(), + dispatched = dispatch_match_notifications( match_items, threshold=thr, source="ingest", run_url=settings.github_run_url, + sheets_url=settings.resolved_google_sheet_url(), + sheets_tab=settings.google_sheet_tab or "Ingest", max_items=int(settings.teams_match_max_items), - enabled=True, + teams_webhook_url=settings.resolved_match_webhook_url(), + teams_enabled=bool(settings.teams_match_notify_enabled), + slack_bot_token=settings.resolved_slack_bot_token(), + slack_channel_id=settings.resolved_slack_channel_id(), + slack_webhook_url=settings.resolved_slack_match_webhook_url(), + slack_enabled=bool(settings.slack_match_notify_enabled), ) + metrics.match_notified = dispatched.teams_posted + metrics.slack_match_notified = dispatched.slack_posted logger.info( "Run complete: write=%s added=%s errors=%s would_create=%s " "skipped_dup=%s skipped_max=%s exit=%s notified=%s " - "match_notified=%s match_count=%s", + "match_notified=%s slack_match_notified=%s match_count=%s", write, metrics.added_count, metrics.error_count, @@ -496,6 +507,7 @@ def run_pipeline( metrics.exit_code, metrics.notified, metrics.match_notified, + metrics.slack_match_notified, metrics.match_notify_count, ) diff --git a/tests/test_notify.py b/tests/test_notify.py index 52bf448..7adea24 100644 --- a/tests/test_notify.py +++ b/tests/test_notify.py @@ -14,10 +14,14 @@ build_adaptive_card_payload, build_ingest_alert_payload, build_match_alert_payload, + build_slack_match_payload, + dispatch_match_notifications, filter_match_items, + google_sheet_edit_url, match_items_from_ranked, notify_ingest_alert, notify_match_alerts, + notify_slack_match_alerts, post_teams_webhook, set_github_output_notified, ) @@ -191,6 +195,22 @@ def test_filter_match_items_threshold(): assert [i.opportunity_id for i in out] == ["HI", "ID-1"] +def test_google_sheet_edit_url(): + assert google_sheet_edit_url(None) is None + assert google_sheet_edit_url(" ") is None + assert ( + google_sheet_edit_url("abc123XYZ") + == "https://docs.google.com/spreadsheets/d/abc123XYZ/edit" + ) + assert ( + google_sheet_edit_url( + "abc", + override_url="https://docs.google.com/spreadsheets/d/abc/edit#gid=0", + ) + == "https://docs.google.com/spreadsheets/d/abc/edit#gid=0" + ) + + def test_build_match_alert_payload_has_cta_and_truncation(): items = [_sample_match(52, "WS-AI"), _sample_match(41, "WS-IAM")] payload = build_match_alert_payload(items, threshold=40, source="ingest") @@ -205,6 +225,30 @@ def test_build_match_alert_payload_has_cta_and_truncation(): assert "52/100" in body_text or "52" in body_text +def test_build_match_alert_payload_includes_sheets_link(): + sheet = "https://docs.google.com/spreadsheets/d/SHEETID/edit" + payload = build_match_alert_payload( + [_sample_match(60, "TOP")], + threshold=40, + source="ingest", + sheets_url=sheet, + sheets_tab="Ingest", + ) + card = payload["attachments"][0]["content"] + assert card["actions"][0]["title"] == "Open Google Sheet" + assert card["actions"][0]["url"] == sheet + facts = next( + b["facts"] for b in card["body"] if isinstance(b, dict) and "facts" in b + ) + fact_map = {f["title"]: f["value"] for f in facts} + assert fact_map["Google Sheet"] == sheet + assert fact_map["Sheet tab"] == "Ingest" + body_text = " ".join( + str(b.get("text", "")) for b in card["body"] if isinstance(b, dict) + ) + assert "Google Sheets" in body_text + + def test_notify_match_alerts_success(): with patch("opportunity_ingest.notify.post_teams_webhook") as post: ok = notify_match_alerts( @@ -262,3 +306,114 @@ def test_match_items_from_ranked_dicts(): assert len(items) == 1 assert items[0].score == 52 assert items[0].score_kind == "GrokFit" + + +def test_build_slack_match_payload_blocks_and_buttons(): + items = [_sample_match(52, "WS-AI"), _sample_match(41, "WS-IAM")] + payload = build_slack_match_payload(items, threshold=40, source="ingest") + assert "blocks" in payload + assert payload["text"] + types = [b.get("type") for b in payload["blocks"]] + assert "header" in types + assert "section" in types + assert "actions" in types + actions = next(b for b in payload["blocks"] if b.get("type") == "actions") + assert actions["elements"][0]["type"] == "button" + assert "https://example.com/notice/WS-AI" in actions["elements"][0]["url"] + + +def test_build_slack_match_payload_includes_sheets_button(): + sheet = "https://docs.google.com/spreadsheets/d/SHEETID/edit" + payload = build_slack_match_payload( + [_sample_match(55)], + threshold=40, + sheets_url=sheet, + sheets_tab="Ranked", + ) + intro = payload["blocks"][1]["text"]["text"] + assert "Google Sheet" in intro + assert "SHEETID" in intro + actions = next(b for b in payload["blocks"] if b.get("type") == "actions") + assert actions["elements"][0]["url"] == sheet + assert "Sheet" in actions["elements"][0]["text"]["text"] + + +def test_notify_slack_match_alerts_web_api_success(): + with patch("opportunity_ingest.notify.post_slack_chat_message") as post: + ok = notify_slack_match_alerts( + [_sample_match(60)], + bot_token="xoxb-test-token", + channel_id="C123", + threshold=40, + source="ingest", + ) + assert ok is True + post.assert_called_once() + kwargs = post.call_args.kwargs + assert kwargs["bot_token"] == "xoxb-test-token" + assert kwargs["channel"] == "C123" + assert kwargs["blocks"] + + +def test_notify_slack_match_alerts_legacy_webhook(): + with patch("opportunity_ingest.notify.post_slack_webhook") as post: + ok = notify_slack_match_alerts( + [_sample_match(60)], + webhook_url="https://hooks.slack.com/services/T/B/x", + threshold=40, + ) + assert ok is True + post.assert_called_once() + + +def test_notify_slack_match_alerts_no_credentials(): + with ( + patch("opportunity_ingest.notify.post_slack_chat_message") as cpost, + patch("opportunity_ingest.notify.post_slack_webhook") as wpost, + ): + ok = notify_slack_match_alerts([_sample_match(90)], threshold=40) + assert ok is False + cpost.assert_not_called() + wpost.assert_not_called() + + +def test_dispatch_match_notifications_both_channels(): + items = [_sample_match(55)] + with ( + patch("opportunity_ingest.notify.post_teams_webhook") as tpost, + patch("opportunity_ingest.notify.post_slack_chat_message") as spost, + ): + result = dispatch_match_notifications( + items, + threshold=40, + teams_webhook_url="https://example.com/teams", + teams_enabled=True, + slack_bot_token="xoxb-test", + slack_channel_id="C99", + slack_enabled=True, + ) + assert result.match_count == 1 + assert result.teams_posted is True + assert result.slack_posted is True + assert result.any_posted is True + tpost.assert_called_once() + spost.assert_called_once() + + +def test_dispatch_slack_only_web_api(): + with ( + patch("opportunity_ingest.notify.post_teams_webhook") as tpost, + patch("opportunity_ingest.notify.post_slack_chat_message") as spost, + ): + result = dispatch_match_notifications( + [_sample_match(80)], + threshold=40, + teams_enabled=False, + slack_bot_token="xoxb-test", + slack_channel_id="#alerts", + slack_enabled=True, + ) + assert result.teams_posted is False + assert result.slack_posted is True + tpost.assert_not_called() + spost.assert_called_once()