diff --git a/.env.example b/.env.example index 356b2e4..9214424 100644 --- a/.env.example +++ b/.env.example @@ -27,7 +27,35 @@ LOG_LEVEL=INFO # Optional Google Sheets sync (free service account + Sheets API): # pip install -e ".[sheets]" # GOOGLE_SHEET_ID= # from docs.google.com/spreadsheets/d//edit +# GOOGLE_SHEET_URL= # optional full share/deep link for Teams/Slack cards (else built from ID) # GOOGLE_SHEET_TAB=Ingest +# GOOGLE_SHEET_RANK_TAB=Ranked # Grok interpret-rank full-replace tab (never Ingest) # GOOGLE_SERVICE_ACCOUNT_FILE=secrets/google-service-account.json # # or inline JSON (e.g. CI secret): # GOOGLE_SERVICE_ACCOUNT_JSON= + +# Optional Grok interpret-rank (xAI API; does not mutate ingest rows): +# pip install -e ".[ai]" +# XAI_API_KEY= +# XAI_BASE_URL=https://api.x.ai/v1 +# XAI_MODEL=grok-4.5 +# OBJECTIVES_PATH=config/objectives.yaml + +# Teams Workflows webhooks (ops alerts + high-match capture pings): +# TEAMS_WEBHOOK_URL= # hard fail / streak / partial (ops) +# TEAMS_MATCH_WEBHOOK_URL= # optional dedicated capture channel; else ops URL +# TEAMS_MATCH_NOTIFY_ENABLED=true +# TEAMS_MATCH_SCORE_THRESHOLD=40 # 0–100; RelevanceScore (ingest) or Grok fit +# TEAMS_MATCH_MAX_ITEMS=8 +# NOTIFY_CONFIG_PATH=config/notify.yaml + +# Slack match pings — prefer Slack CLI / Bolt tokens (https://docs.slack.dev/tools/slack-cli/): +# slack login → create/install app with chat:write → paste: +# SLACK_BOT_TOKEN=xoxb-... # Bot User OAuth Token (required for Web API) +# SLACK_CHANNEL_ID=C... # or #channel-name +# SLACK_APP_TOKEN=xapp-... # optional; Socket Mode / slack run only +# SLACK_MATCH_NOTIFY_ENABLED=true +# Setup guide: scripts/slack_cli_setup.md +# Legacy Incoming Webhook fallback (discouraged): +# SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... +# SLACK_MATCH_WEBHOOK_URL= diff --git a/.gitignore b/.gitignore index 4658a48..815932b 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,8 @@ build/ # Local data / state / logs (keep directory placeholders) data/*.db data/*.csv +data/rankings/ +data/atlas_crawl/ !data/.gitkeep logs/* !logs/.gitkeep diff --git a/AGENTS.md b/AGENTS.md index 03528a8..8cf9572 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ |-------|--------| | **Project** | Contract Bidding System — Phase 1 opportunity ingest | | **Package** | `opportunity_ingest` (`src/opportunity_ingest/`) | -| **As-of** | 2026-08-10 (as-built; Google Sheets sync included) | +| **As-of** | 2026-08-11 (as-built; Sheets sync + optional Grok interpret-rank) | | **Primary language** | Python 3.11+ | | **Default store** | SQLite (`STORAGE_BACKEND=sqlite`) | @@ -24,8 +24,9 @@ Daily (or on-demand) pipeline: 6. Dedupe by `OpportunityID` + normalized `Link`. 7. **Create-only** writes to SQLite (default) or SharePoint (optional). 8. Optional: export CSV, **full-replace sync to Google Sheets `Ingest` tab**, Teams alerts. +9. Optional post-ingest: **Grok interpret-rank** — plain-English rewrite + fit ranking vs `config/objectives.yaml` (report only). -**SQLite is the system of record for day-1.** Google Sheets is a **view** (overwritten on sync). SharePoint is a **pluggable store**, not required for go-live. +**SQLite is the system of record for day-1.** Google Sheets is a **view** (overwritten on sync). SharePoint is a **pluggable store**, not required for go-live. Grok ranking writes reports under `data/rankings/`; it does **not** mutate store Status/Notes/RelevanceScore. --- @@ -35,14 +36,18 @@ Daily (or on-demand) pipeline: |----------|------|---------| | 1 | **This file (`AGENTS.md`)** | Non-negotiable rules + orientation | | 2 | [`docs/STATUS.md`](docs/STATUS.md) | Living readiness snapshot (what works / next) | -| 3 | [`docs/DATA_UPDATE_DIRECTIVES.md`](docs/DATA_UPDATE_DIRECTIVES.md) | **MUST / MUST NOT** for data writes, schema, sync | -| 4 | [`docs/AS_BUILT.md`](docs/AS_BUILT.md) | Current architecture, modules, CLI, env | -| 5 | [`docs/CHANGE_PLAYBOOK.md`](docs/CHANGE_PLAYBOOK.md) | How to implement common change types | -| 6 | [`docs/INDEX.md`](docs/INDEX.md) | Full documentation index | -| 7 | [`docs/phase1-canadabuys-sharepoint-implementation-schema.md`](docs/phase1-canadabuys-sharepoint-implementation-schema.md) | Original design (rev 4); prefer AS_BUILT if conflict | -| 8 | [`scripts/ops_runbook.md`](scripts/ops_runbook.md) | Human operator procedures | -| 9 | [`scripts/google_sheets_setup.md`](scripts/google_sheets_setup.md) | Sheets service-account setup | -| 10 | [`scripts/provision_sharepoint_list.md`](scripts/provision_sharepoint_list.md) | SharePoint activation (deferred) | +| 3 | [`docs/PLUG_AND_PLAY.md`](docs/PLUG_AND_PLAY.md) | Human keys + plug-and-play steps for built features | +| 3b | [`docs/PROCESSOR_OVERVIEW.md`](docs/PROCESSOR_OVERVIEW.md) | Plain language: this app as one processor — triggers, I/O, system place | +| 4 | [`docs/DATA_UPDATE_DIRECTIVES.md`](docs/DATA_UPDATE_DIRECTIVES.md) | **MUST / MUST NOT** for data writes, schema, sync | +| 5 | [`docs/AS_BUILT.md`](docs/AS_BUILT.md) | Current architecture, modules, CLI, env | +| 6 | [`docs/BACKLOG.md`](docs/BACKLOG.md) | Roadmap / discussed but **not built** | +| 7 | [`docs/CHANGE_PLAYBOOK.md`](docs/CHANGE_PLAYBOOK.md) | How to implement common change types | +| 8 | [`docs/INDEX.md`](docs/INDEX.md) | Full documentation index | +| 9 | [`docs/DECISIONS.md`](docs/DECISIONS.md) | ADR-style as-built decisions | +| 10 | [`docs/phase1-canadabuys-sharepoint-implementation-schema.md`](docs/phase1-canadabuys-sharepoint-implementation-schema.md) | Original design (rev 4); prefer AS_BUILT if conflict | +| 11 | [`scripts/ops_runbook.md`](scripts/ops_runbook.md) | Human operator procedures | +| 12 | [`scripts/google_sheets_setup.md`](scripts/google_sheets_setup.md) | Sheets service-account setup | +| 13 | [`scripts/provision_sharepoint_list.md`](scripts/provision_sharepoint_list.md) | SharePoint activation (deferred) | --- @@ -56,8 +61,9 @@ Daily (or on-demand) pipeline: 6. **CanadaBuys download requires browser-like User-Agent** (CDN returns 403 otherwise). See `download.py` `DEFAULT_HEADERS`. 7. **Google Sheets `Ingest` tab is full-replace.** Manual edits on that tab are wiped next `sync-sheets`. Human work → separate tab (e.g. `Review`). 8. **Keywords:** engineering owns `config/keywords.yaml`. Ops request changes via issues. -9. **Secrets:** never commit `.env`, `secrets/`, service-account JSON, Azure secrets. -10. **Phase 1 non-goals:** multi-source (MERX/etc.), AI ranking, auto-bidding, historical backfill, dual-write SQLite+SharePoint, two-way Sheets Status sync. +9. **Secrets:** never commit `.env`, `secrets/`, service-account JSON, Azure secrets, `XAI_API_KEY`. +10. **Phase 1 non-goals:** multi-source (MERX/etc.), auto-bidding, historical backfill, dual-write SQLite+SharePoint, two-way Sheets Status sync. +11. **Grok interpret-rank is post-ingest only.** Never write AI fit scores into `contract_opportunities` Status/Notes/RelevanceScore. Objectives live in `config/objectives.yaml` (eng-owned). --- @@ -75,15 +81,18 @@ src/opportunity_ingest/ models.py # TenderRecord, OpportunityFields, ExistingKeys config.py # pydantic-settings state.py # zero-new streak JSON - notify.py # Teams Workflows Adaptive Card + notify.py # Teams + Slack match pings (score≥threshold); Teams ops alerts exit_codes.py # exit + notify matrix sheets_sync.py # SQLite → Google Sheets full tab replace + interpret_rank.py # Grok rephrase + fit rank (report only) storage/ base.py # OpportunityStore Protocol, normalize_link factory.py # sqlite | sharepoint sqlite_store.py # default backend + export_csv sharepoint_store.py # Graph adapter (optional) config/keywords.yaml +config/objectives.yaml # company objectives for interpret-rank +config/notify.yaml # Teams match threshold / card limits .github/workflows/ci.yml .github/workflows/daily-canadabuys-ingest.yml ``` @@ -98,9 +107,11 @@ python -m opportunity_ingest download-sample [--out PATH] python -m opportunity_ingest check-store python -m opportunity_ingest export-csv [--out PATH] python -m opportunity_ingest sync-sheets [--sheet-id ID] [--tab NAME] +python -m opportunity_ingest interpret-rank [--status STATUS] [--limit N] [--sync-sheets|--no-sync-sheets] [--rank-tab NAME] +python -m opportunity_ingest sync-rank-sheets [--from-json PATH] [--rank-tab NAME] ``` -Optional install extras: `pip install -e ".[sheets]"` for Google Sheets; SharePoint uses core deps (`msal`, `httpx`). +Optional install extras: `pip install -e ".[sheets]"` for Google Sheets; `pip install -e ".[ai]"` for Grok interpret-rank; SharePoint uses core deps (`msal`, `httpx`). Grok rankings sync to tab **`Ranked`** (never `Ingest`). --- @@ -115,7 +126,9 @@ Optional install extras: `pip install -e ".[sheets]"` for Google Sheets; SharePo | Store schema | `sqlite_store.py` DDL + `models.OpportunityFields` + design schema | | Pipeline order / metrics | `pipeline.py` | | Exit / notify policy | `exit_codes.py`, `notify.py` | -| Sheets sync | `sheets_sync.py`, `cli.py` `sync-sheets` | +| Sheets sync (opportunities) | `sheets_sync.py`, `cli.py` `sync-sheets` → tab `Ingest` | +| Sheets sync (Grok ranks) | `sheets_sync.py` `sync_rankings_to_sheet`, `interpret-rank --sync-sheets`, `sync-rank-sheets` → tab `Ranked` | +| Grok interpret / rank | `interpret_rank.py`, `config/objectives.yaml`, `cli.py` `interpret-rank` | | SharePoint | `storage/sharepoint_store.py`, `scripts/provision_sharepoint_list.md` | | Daily schedule | `.github/workflows/daily-canadabuys-ingest.yml` | | Env vars | `config.py`, `.env.example`, `config/settings.example.env` | @@ -128,6 +141,8 @@ Optional install extras: `pip install -e ".[sheets]"` for Google Sheets; SharePo |------|------| | `data/contract_opportunities.db` | SQLite system of record | | `data/sample-openTenderNotice.csv` | Downloaded open-tender sample | +| `data/rankings/interpret-*.md` | Grok ranked brief (human) | +| `data/rankings/interpret-*.json` | Grok ranked brief (machine) | | `data/export-opportunities.csv` | Human CSV export | | `state/zero_new_streak.json` | Zero-new calendar streak | | `logs/run-*.json` | Per-run metrics | diff --git a/README.md b/README.md index fde5cea..4061739 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ Phase 1 pipeline: download CanadaBuys **Open Tender Notices** CSV, filter by key | **Optional store** | SharePoint via Microsoft Graph (`STORAGE_BACKEND=sharepoint`) | | **Optional view** | Google Sheets tab full-replace (`sync-sheets`) | | **LLM / agent entry** | **[`AGENTS.md`](AGENTS.md)** ← start here for AI-assisted development | +| **Human plug-in** | **[`docs/PLUG_AND_PLAY.md`](docs/PLUG_AND_PLAY.md)** ← keys, steps, roadmap to go live | +| **Processor overview** | **[`docs/PROCESSOR_OVERVIEW.md`](docs/PROCESSOR_OVERVIEW.md)** ← role, triggers, I/O in the wider system | --- @@ -15,10 +17,13 @@ Phase 1 pipeline: download CanadaBuys **Open Tender Notices** CSV, filter by key | Doc | Audience | Purpose | |-----|----------|---------| +| [`docs/PROCESSOR_OVERVIEW.md`](docs/PROCESSOR_OVERVIEW.md) | **Everyone** | This app as **one key processor** — role, triggers, outputs | +| [`docs/PLUG_AND_PLAY.md`](docs/PLUG_AND_PLAY.md) | **Operators / humans** | **Keys to plug, function list, go-live sequence, roadmap** | | [`AGENTS.md`](AGENTS.md) | LLMs + devs | Hard rules, module map, task routing | | [`docs/STATUS.md`](docs/STATUS.md) | Everyone | Living readiness / what works now | | [`docs/INDEX.md`](docs/INDEX.md) | Everyone | Full doc index | | [`docs/AS_BUILT.md`](docs/AS_BUILT.md) | Devs / LLMs | Implemented architecture | +| [`docs/BACKLOG.md`](docs/BACKLOG.md) | Everyone | Roadmap / not built yet | | [`docs/DATA_UPDATE_DIRECTIVES.md`](docs/DATA_UPDATE_DIRECTIVES.md) | Devs / LLMs | **MUST / MUST NOT** for data writes & sync | | [`docs/CHANGE_PLAYBOOK.md`](docs/CHANGE_PLAYBOOK.md) | Devs / LLMs | How to make common changes | | [`docs/DECISIONS.md`](docs/DECISIONS.md) | Devs / LLMs | Key ADRs | @@ -42,8 +47,11 @@ If design doc and as-built conflict → **prefer code + AS_BUILT + DATA_UPDATE_D ## Quick start (Windows) +**Full human checklist (keys, Sheets, Grok, Teams, roadmap):** +→ **[`docs/PLUG_AND_PLAY.md`](docs/PLUG_AND_PLAY.md)** + ```powershell -cd "C:\Users\Vikrant\Documents\Agentic AI Learning\contractbiddingsystem" +cd python -m venv .venv .\.venv\Scripts\Activate.ps1 @@ -71,7 +79,23 @@ pip install -e ".[sheets]" python -m opportunity_ingest sync-sheets ``` -**Notes learned in setup:** +### Grok interpret-rank (optional AI brief) + +Plain-English rewrite + rank against `config/objectives.yaml` (does **not** change SQLite rows). +When `GOOGLE_SHEET_ID` is set (your live sheet), rankings are **full-replaced** into a **`Ranked`** tab (separate from `Ingest`): + +```powershell +pip install -e ".[ai]" +pip install -e ".[sheets]" # same service account as sync-sheets +# Set XAI_API_KEY + existing GOOGLE_* in root .env +python -m opportunity_ingest interpret-rank --status New --limit 20 +# Local: data/rankings/interpret-*.md +# Sheet: tab Ranked (created if missing) +# Skip sheet push: --no-sync-sheets +# Re-push last report without Grok: python -m opportunity_ingest sync-rank-sheets +``` + +**Setup notes:** - `pip install -e .` needs the trailing `.` - CanadaBuys may return **403** without a browser User-Agent (handled in `download.py`) @@ -88,14 +112,18 @@ python -m opportunity_ingest download-sample [--out PATH] python -m opportunity_ingest check-store python -m opportunity_ingest export-csv [--out PATH] python -m opportunity_ingest sync-sheets [--sheet-id ID] [--tab NAME] +python -m opportunity_ingest interpret-rank [--status STATUS] [--limit N] [--sync-sheets|--no-sync-sheets] +python -m opportunity_ingest sync-rank-sheets [--from-json PATH] [--rank-tab NAME] ``` | Command | Persist store? | Notes | |---------|----------------|--------| | `run` | Only with `--write` | Default dry-run | | `export-csv` | No | Snapshot for Excel | -| `sync-sheets` | No SQLite write | **Full-replace** Sheet tab | +| `sync-sheets` | No SQLite write | **Full-replace** Sheet tab `Ingest` | | `check-store` | No | Health + key count | +| `interpret-rank` | No | Grok rewrite + rank → `data/rankings/` + optional Sheets **Ranked** | +| `sync-rank-sheets` | No | Re-push ranking JSON → Sheets **Ranked** (no Grok call) | --- @@ -107,15 +135,22 @@ python -m opportunity_ingest sync-sheets [--sheet-id ID] [--tab NAME] | `DATA_DIR` | `data` | DB + exports | | `MAX_CREATE` | `50` | Create-**attempt** budget; `0`=unlimited | | `GOOGLE_SHEET_ID` | — | Sheets sync | -| `GOOGLE_SHEET_TAB` | `Ingest` | Tab overwritten on sync | +| `GOOGLE_SHEET_TAB` | `Ingest` | Opportunity tab overwritten on sync | +| `GOOGLE_SHEET_RANK_TAB` | `Ranked` | Grok ranking tab overwritten on interpret-rank | | `GOOGLE_SERVICE_ACCOUNT_FILE` | — | Path to JSON key | -| `TEAMS_WEBHOOK_URL` | — | Failure / streak alerts | +| `TEAMS_WEBHOOK_URL` | — | Ops alerts + match fallback | +| `TEAMS_MATCH_WEBHOOK_URL` | — | Optional capture-channel webhook | +| `TEAMS_MATCH_SCORE_THRESHOLD` | `40` | Ping when score ≥ N | +| `XAI_API_KEY` | — | Grok interpret-rank | +| `OBJECTIVES_PATH` | `config/objectives.yaml` | AI ranking frame of reference | -Full list: `.env.example`, `config/settings.example.env`, `docs/AS_BUILT.md`. +Full list: `.env.example`, `config/settings.example.env`, [`docs/PLUG_AND_PLAY.md`](docs/PLUG_AND_PLAY.md), `docs/AS_BUILT.md`. **Data rules (critical):** create-only; never silent-truncate Link; Sheets is a view. → [`docs/DATA_UPDATE_DIRECTIVES.md`](docs/DATA_UPDATE_DIRECTIVES.md) +**What exists vs not built:** [`docs/STATUS.md`](docs/STATUS.md) · [`docs/AS_BUILT.md`](docs/AS_BUILT.md) · [`docs/BACKLOG.md`](docs/BACKLOG.md) + --- ## Development diff --git a/config/keywords.yaml b/config/keywords.yaml index 5104654..7f06662 100644 --- a/config/keywords.yaml +++ b/config/keywords.yaml @@ -1,106 +1,138 @@ # Engineering-owned keyword config for CanadaBuys open-tender filtering. -# Ops requests changes via GitHub issues; eng reviews/merges. -# Do not add bare "teams" or "strategy" — use multi-word / scoped terms only. +# Tuned for Atlas Flow Group capabilities (public site review 2026-08-11): +# https://www.atlasflowgroup.com/ +# Ops request changes via GitHub issues; eng reviews/merges. +# +# Atlas Flow Group is NOT a classic MSP / ServiceNow SI / construction vendor. +# Prefer: operating-model advisory, human-governed AI, Microsoft-native enablement +# (M365, Copilot, Power Platform), process/workflow under authority design. +# Avoid bare high-noise terms (teams, strategy, service desk alone). # # Matching notes: # - Short terms (len <= short_term_max_len) use word-boundary regex. -# - Unique-by-term weights; nested overlaps are span-aware: a shorter term is -# dropped only when every hit is covered by a longer term's span (e.g. -# "managed service" inside "managed services"). Independent shorter hits -# still count (e.g. "workflow automation and general automation"). -# - Terms dual-listed across groups (e.g. "power automate") contribute one -# weight but attribute all declaring groups for diversity scoring. +# - Unique-by-term weights; nested overlaps are span-aware. +# - Terms dual-listed across groups contribute one weight but all groups for scoring. version: 1 match: case_sensitive: false - fields: ["title", "description"] # honored by loader (title | description) + fields: ["title", "description"] search_category: true short_term_max_len: 4 groups: + # Critical — site: "Microsoft-native", M365, Copilot, Power Platform microsoft_cloud: - label: "Microsoft / M365 / Power Platform / Azure / Copilot" + label: "Microsoft-native (M365 / Copilot / Power Platform / Azure)" keywords: - - { term: "microsoft 365", weight: 20 } - - { term: "microsoft365", weight: 20 } - - { term: "m365", weight: 18 } + - { term: "microsoft 365", weight: 22 } + - { term: "microsoft365", weight: 22 } + - { term: "m365", weight: 20 } - { term: "office 365", weight: 18 } - { term: "o365", weight: 15 } + - { term: "microsoft-native", weight: 20 } + - { term: "microsoft native", weight: 20 } - { term: "sharepoint", weight: 16 } - - { term: "power platform", weight: 20 } - - { term: "power apps", weight: 18 } - - { term: "power automate", weight: 18 } - - { term: "power bi", weight: 14 } - - { term: "dynamics 365", weight: 16 } - - { term: "copilot", weight: 20 } - - { term: "azure", weight: 14 } - - { term: "entra id", weight: 12 } + - { term: "power platform", weight: 22 } + - { term: "power apps", weight: 20 } + - { term: "power automate", weight: 20 } + - { term: "power bi", weight: 12 } + - { term: "copilot", weight: 22 } + - { term: "microsoft copilot", weight: 22 } + - { term: "azure", weight: 12 } + - { term: "entra id", weight: 14 } - { term: "active directory", weight: 10 } - { term: "microsoft teams", weight: 12 } + - { term: "dynamics 365", weight: 10 } - managed_services: - label: "Managed services" - keywords: - - { term: "managed service", weight: 16 } - - { term: "managed services", weight: 16 } - - { term: "msp", weight: 10 } - - { term: "managed it", weight: 14 } - - { term: "outsourcing", weight: 10 } - - { term: "service desk", weight: 14 } - - { term: "end user computing", weight: 12 } - - { term: "desktop support", weight: 12 } - - { term: "device management", weight: 12 } - - { term: "intune", weight: 14 } - - itsm_servicenow: - label: "ITSM / ServiceNow" + # Critical — site: Authority, operating model, source of truth, runbooks, registries + operating_authority: + label: "Operating model / authority / governance design" keywords: - - { term: "servicenow", weight: 20 } - - { term: "service now", weight: 18 } - - { term: "itsm", weight: 16 } - - { term: "itil", weight: 12 } - - { term: "incident management", weight: 12 } - - { term: "service management", weight: 14 } - - { term: "cmdb", weight: 12 } + - { term: "operating model", weight: 22 } + - { term: "operating system", weight: 12 } + - { term: "source of truth", weight: 18 } + - { term: "source of authority", weight: 20 } + - { term: "decision rights", weight: 16 } + - { term: "operating contract", weight: 18 } + - { term: "operating contracts", weight: 18 } + - { term: "runbook", weight: 14 } + - { term: "runbooks", weight: 14 } + - { term: "governance model", weight: 16 } + - { term: "governance framework", weight: 16 } + - { term: "corporate governance", weight: 10 } + - { term: "operational governance", weight: 18 } + - { term: "process governance", weight: 16 } + - { term: "business process", weight: 12 } + - { term: "process design", weight: 14 } + - { term: "process discipline", weight: 14 } + - { term: "traceability", weight: 12 } + - { term: "accountability", weight: 10 } + # High — lead offering on /services advisory_consulting: - label: "Advisory / Consulting (ops)" + label: "Advisory / operating-model consulting" keywords: + - { term: "advisory services", weight: 18 } + - { term: "advisory service", weight: 16 } - { term: "management consulting", weight: 12 } - - { term: "advisory services", weight: 12 } - { term: "professional services", weight: 10 } - - { term: "digital transformation", weight: 14 } - - { term: "operating model", weight: 12 } - - { term: "change management", weight: 10 } + - { term: "digital transformation", weight: 12 } + - { term: "change management", weight: 12 } - { term: "business analysis", weight: 10 } + - { term: "operations assessment", weight: 16 } + - { term: "organisational design", weight: 12 } + - { term: "organizational design", weight: 12 } + - { term: "business transformation", weight: 12 } + # High — site: automation & agents only under human authority automation_process: - label: "Automation / Process improvement" + label: "Workflow automation / agents under governance" keywords: - - { term: "automation", weight: 14 } - - { term: "rpa", weight: 14 } - - { term: "robotic process", weight: 16 } + - { term: "workflow automation", weight: 18 } + - { term: "workflow design", weight: 16 } + - { term: "process automation", weight: 16 } - { term: "process improvement", weight: 14 } - - { term: "workflow automation", weight: 12 } + - { term: "business process automation", weight: 16 } + - { term: "robotic process", weight: 12 } + - { term: "rpa", weight: 12 } + - { term: "low code", weight: 14 } + - { term: "no code", weight: 12 } + - { term: "power automate", weight: 20 } - { term: "orchestration", weight: 10 } - - { term: "low code", weight: 12 } - - { term: "no code", weight: 10 } - - { term: "power automate", weight: 18 } + - { term: "intelligent automation", weight: 14 } + # Bare "automation" kept moderate — many false friends in construction notices + - { term: "automation", weight: 10 } + # High — site: human-governed AI, AI assistance without authority (not unowned GenAI product) ai_operations: - label: "AI + operations / AI governance" + label: "Human-governed AI / AI assistance / responsible AI" + keywords: + - { term: "artificial intelligence", weight: 14 } + - { term: "generative ai", weight: 16 } + - { term: "genai", weight: 14 } + - { term: "ai governance", weight: 22 } + - { term: "responsible ai", weight: 20 } + - { term: "ai assistance", weight: 18 } + - { term: "human in the loop", weight: 16 } + - { term: "human-in-the-loop", weight: 16 } + - { term: "ai adoption", weight: 14 } + - { term: "ai enablement", weight: 16 } + - { term: "llm", weight: 12 } + - { term: "machine learning", weight: 10 } + - { term: "data governance", weight: 14 } + - { term: "ai operations", weight: 12 } + + # Low / selective — only when notice is really about knowledge ops / continuous improvement loops + knowledge_flow: + label: "Knowledge / continuous operating flow" keywords: - - { term: "artificial intelligence", weight: 16 } - - { term: "machine learning", weight: 14 } - - { term: "generative ai", weight: 18 } - - { term: "genai", weight: 16 } - - { term: "ai governance", weight: 18 } - - { term: "responsible ai", weight: 16 } - - { term: "llm", weight: 14 } - - { term: "ai operations", weight: 16 } - - { term: "mlops", weight: 14 } - - { term: "data governance", weight: 12 } + - { term: "knowledge management", weight: 12 } + - { term: "knowledge base", weight: 10 } + - { term: "continuous improvement", weight: 12 } + - { term: "operational excellence", weight: 12 } + - { term: "shared services", weight: 10 } +# Prefer services-style procurement categories (not construction). category_boosts: "*SRV": 5 "SRV": 5 diff --git a/config/notify.yaml b/config/notify.yaml new file mode 100644 index 0000000..7f52f64 --- /dev/null +++ b/config/notify.yaml @@ -0,0 +1,35 @@ +# Notification pipeline config (engineering-owned). +# Loaded by opportunity_ingest when present; env vars override (see Settings). +# +# Ops alerts (hard fail / partial / zero-new streak): TEAMS_WEBHOOK_URL only today. +# +# Match alerts (score ≥ threshold): Teams Adaptive Card and/or Slack Block Kit. +# Ingest write path → rule RelevanceScore +# interpret-rank → Grok fit_score +# When GOOGLE_SHEET_ID (or GOOGLE_SHEET_URL) is set, cards include Open Google Sheet. +# +# Slack preferred auth = Slack CLI / Bolt tokens (not Incoming Webhooks): +# https://docs.slack.dev/tools/slack-cli/ +# SLACK_BOT_TOKEN=xoxb-... (required for chat.postMessage) +# SLACK_CHANNEL_ID=C... (or #channel-name) +# SLACK_APP_TOKEN=xapp-... (optional; used by `slack run` Socket Mode only) +version: 1 + +match: + score_threshold: 40 + max_items_per_message: 8 + +teams: + match_notify_enabled: true + match_score_threshold: 40 + max_items_per_message: 8 + title_max_chars: 120 + summary_max_chars: 280 + card_title_ingest: "New opportunity matches (≥ threshold)" + card_title_rank: "Grok fit matches (≥ threshold)" + cta_label: "Open notice" + +slack: + match_notify_enabled: true + # Prefer Web API (bot token). Legacy webhook only if SLACK_BOT_TOKEN unset. + # mode: web_api | webhook # informational; code auto-selects diff --git a/config/objectives.yaml b/config/objectives.yaml new file mode 100644 index 0000000..bbf2b3d --- /dev/null +++ b/config/objectives.yaml @@ -0,0 +1,116 @@ +# Engineering-owned company objectives for Grok-assisted interpret-rank. +# Source of truth for company positioning: Atlas Flow Group public site (reviewed 2026-08-11). +# https://www.atlasflowgroup.com/ +# Pages: /, /services, /approach, /about, /contact +# Align keyword groups in config/keywords.yaml with these objectives. +# +# Used by: python -m opportunity_ingest interpret-rank +# This file does NOT change ingest scoring; it frames AI ranking only. +version: 2 + +company: + name: "Atlas Flow Group" + website: "https://www.atlasflowgroup.com/" + email: "hello@atlasflowgroup.com" + summary: > + SMB-focused advisory and Microsoft-native enablement firm. We design + operating systems where people hold authority, AI assists without taking + control, and every material decision is traceable by design. Lead with + advisory (authority, operating model, governance readiness), then + implement Copilot / Power Platform / M365 workflows under that model — + not tool sprawl, not unowned AI, not large multi-year systems-integrator programmes. + geography: > + Canada-oriented public and private buyers are in scope when the work is + operating-model, process, Microsoft-native AI enablement, or human-governed + automation. Prefer mid-market / SMB-right-sized scopes; large SI-only + footprints are a poor fit. + posture: > + One Authority · Human Governed · Traceable by Design. Authority before + automation. AI assistance without authority. Implementation over presentation-only deliverables. + Not a pure product vendor, not a classic MSP/service desk outsourcer, + not ServiceNow platform SI, not construction/facilities/commodities. + +# Priority: critical | high | medium | low (soft ranking hints for Grok). +objectives: + - id: operating_authority + label: "Operating model / authority / governance design" + priority: critical + description: > + Help leaders define one source of authority for operations: decision + rights, canonical source of truth, operating contracts, registries, + runbooks, and governance readiness. Discovery and authority assessment + engagements. AtlasFlow OS thinking delivered through advisory. + + - id: microsoft_cloud + label: "Microsoft-native enablement (M365 / Copilot / Power Platform)" + priority: critical + description: > + Practical Microsoft-native AI and workflow enablement on the stack SMBs + already own: Microsoft 365, Copilot, Power Platform (Power Apps / Power + Automate), SharePoint/Teams patterns, with security and governance that + fit real organisations — wired into operating contracts, not pilot demos. + + - id: advisory_consulting + label: "Advisory retainer & operations counsel" + priority: high + description: > + Ongoing specialized advisory for operations/IT leaders and owners + (roughly 50–500 employees) who need a partner without hiring a large SI. + Operations and authority assessment, prioritisation of high-leverage + improvements, decision support without generic strategy decks. + + - id: automation_process + label: "Governed workflow automation & agents" + priority: high + description: > + Workflow design and automation/agents only where authority is already + defined. Continuous flow Capture → Review → Curate → Use → Govern → Scale. + Execution under human authority — never above it. Power Automate and + adjacent automation under an operating contract. + + - id: ai_operations + label: "Human-governed AI / AI governance / responsible enablement" + priority: high + description: > + AI adoption and enablement with human governance: AI assists; people + remain accountable. Strong fit for AI governance, responsible AI, + Copilot adoption with decision rights, and anti-patterns of unowned AI. + Weak fit for pure GenAI product platforms, AI that replaces ownership, + or speculative transformation without governance. + + - id: knowledge_flow + label: "Traceable operating flow & continuous improvement" + priority: medium + description: > + Design work so it is reviewable, auditable, and improvable; shared + intelligence, knowledge that is curated and reused, operational + excellence loops — always attached to authority/governance design. + +# Soft filters for the interpreter (not hard ingest rules). +guidance: + prefer: + - "Operating model, authority, governance, runbooks, decision rights" + - "Microsoft 365 / Copilot / Power Platform enablement with process ownership" + - "Human-governed AI, AI governance, responsible AI adoption" + - "Workflow automation under clear operating contracts" + - "Advisory / professional services scopes (not pure product resale)" + - "SMB-right-sized or modular public-sector work (not multi-year mega-SI only)" + - "Services categories (*SRV) with real ops/IT/process scope" + deprioritize: + - "Construction, facilities, elevator, snow/ice, food/beverage design, pure advertising creative" + - "Classic MSP / service desk / desktop outsourcing as the core SOW (outside published services)" + - "ServiceNow / ITSM platform implementation as primary scope (not a published capability)" + - "Specialized product platforms (e.g. third-party risk intelligence SaaS) unless a partnership is in scope" + - "Military payroll / niche HRIS sustainment without ops-model angle" + - "Keyword false positives with no operating-model or Microsoft-native scope" + - "Buyers who want AI to replace human ownership" + - "Speculative AI transformation without governance" + output_tone: > + Plain language for an Atlas Flow Group capture lead. Lead with whether the + buyer needs authority/governance design and/or Microsoft-native enablement + under human control. Be candid: high keyword scores do not equal fit when + the SOW is product resale, construction, or classic outsourcing. + ranking_order_hint: > + Score highest when operating-model/advisory + Microsoft-native or + human-governed AI clearly align. Medium when only one pillar fits. + Low/pass for out-of-scope domains even if a single keyword matched. diff --git a/config/settings.example.env b/config/settings.example.env index ec8c998..36e0bf8 100644 --- a/config/settings.example.env +++ b/config/settings.example.env @@ -50,7 +50,34 @@ LOG_LEVEL=INFO # pip install -e ".[sheets]" # .env must be at REPO ROOT (not scripts/) # GOOGLE_SHEET_ID= +# GOOGLE_SHEET_URL= # optional; Teams/Slack "Open Google Sheet" CTA (else from ID) # GOOGLE_SHEET_TAB=Ingest +# GOOGLE_SHEET_RANK_TAB=Ranked # GOOGLE_SERVICE_ACCOUNT_FILE=secrets/google-service-account.json # # CI alternative: # GOOGLE_SERVICE_ACCOUNT_JSON= + +# --- Grok interpret-rank (optional; does not mutate SQLite rows) --- +# pip install -e ".[ai]" +# XAI_API_KEY= +# XAI_BASE_URL=https://api.x.ai/v1 +# XAI_MODEL=grok-4.5 +# OBJECTIVES_PATH=config/objectives.yaml + +# --- Teams notify --- +# TEAMS_WEBHOOK_URL= +# TEAMS_MATCH_WEBHOOK_URL= +# TEAMS_MATCH_NOTIFY_ENABLED=true +# TEAMS_MATCH_SCORE_THRESHOLD=40 +# TEAMS_MATCH_MAX_ITEMS=8 +# NOTIFY_CONFIG_PATH=config/notify.yaml + +# --- Slack notify (Slack CLI / Bolt tokens; same match threshold as Teams) --- +# https://docs.slack.dev/tools/slack-cli/ — see scripts/slack_cli_setup.md +# SLACK_BOT_TOKEN=xoxb-... +# SLACK_CHANNEL_ID=C... +# SLACK_APP_TOKEN=xapp-... +# SLACK_MATCH_NOTIFY_ENABLED=true +# Legacy webhook fallback: +# SLACK_WEBHOOK_URL= +# SLACK_MATCH_WEBHOOK_URL= diff --git a/docs/AS_BUILT.md b/docs/AS_BUILT.md index 1fcb9df..34de10e 100644 --- a/docs/AS_BUILT.md +++ b/docs/AS_BUILT.md @@ -1,10 +1,11 @@ -# As-built architecture (Phase 1) +# As-built architecture (Phase 1 + post-ingest Grok) -**Source of truth for the implemented system.** Supersedes design-doc details when they differ. +**Source of truth for the implemented system.** Supersedes design-doc details when they differ. +**Unbuilt / discussed ideas:** [`BACKLOG.md`](BACKLOG.md) — do not invent features from backlog into this file. | Field | Value | |-------|--------| -| Status | Implemented and manually exercised (Windows, 2026-08-10) | +| Status | Ingest exercised (Windows, 2026-08-10); Grok/Ranked in code + unit tests (2026-08-11) | | Package | `opportunity_ingest` | | Python | 3.11+ (verified on 3.14 in local venv) | @@ -12,69 +13,105 @@ ## 1. Pipeline +### 1.1 Ingest (system of record) + ```text CanadaBuys CSV → download (UA headers, 1 retry) → parse (bilingual headers, EN/FR coalesce) → filter_keywords (any-match; short terms word-boundary; span-aware nested suppress) - → score (0–100, UTC) + → score (0–100, UTC) # rule-based RelevanceScore only → map_fields (OpportunityFields) → load_existing_keys (store) → for each new candidate while attempts < MAX_CREATE: - store.create() + store.create() # create-only; never updates Status/Notes → streak state (write runs) → optional Teams notify → exit code matrix - → optional: export-csv | sync-sheets + → optional: export-csv | sync-sheets (Ingest tab) +``` + +### 1.2 Post-ingest Grok rank (derived views only) + +```text +SQLite contract_opportunities + → filter optional Status / limit / min rule score + → load config/objectives.yaml + → Grok (xAI OpenAI-compatible API) batch interpret + fit_score + → write data/rankings/interpret-*.json + .md + → optional full-replace Google Sheet tab Ranked + (default ON when GOOGLE_SHEET_ID set; --no-sync-sheets to skip) ``` +**Not in pipeline today:** website-type classifier, per-portal keyword packs, notice-page crawl, AI rewriting `keywords.yaml` (see BACKLOG). + --- ## 2. Components | Module | Path | Responsibility | |--------|------|----------------| -| CLI | `cli.py` | `run`, `download-sample`, `check-store`, `export-csv`, `sync-sheets` | +| CLI | `cli.py` | `run`, `download-sample`, `check-store`, `export-csv`, `sync-sheets`, `interpret-rank`, `sync-rank-sheets` | | Pipeline | `pipeline.py` | Orchestration, metrics, run logs | | Download | `download.py` | HTTP GET + DEFAULT_HEADERS (anti-403) | | Parse | `parse.py` | HEADER_CANDIDATES, TenderRecord | -| Filter | `filter_keywords.py` | keywords.yaml load + match | +| Filter | `filter_keywords.py` | keywords.yaml load + match (global groups only) | | Score | `score.py` | Rule-based RelevanceScore | | Map | `map_fields.py` | Schema mapping, Link required | +| Interpret / rank | `interpret_rank.py` | Grok rephrase + fit rank vs objectives; local reports | | Config | `config.py` | pydantic-settings / `.env` | | State | `state.py` | zero-new streak | | Notify | `notify.py` | Teams Workflows Adaptive Card + GITHUB_OUTPUT | | Exit | `exit_codes.py` | Exit + notify decisions | -| Sheets | `sheets_sync.py` | Full tab replace via gspread | +| Sheets | `sheets_sync.py` | Full tab replace: Ingest (SQLite) + Ranked (Grok JSON) | | Store protocol | `storage/base.py` | `OpportunityStore` | | Factory | `storage/factory.py` | `sqlite` \| `sharepoint` | | SQLite | `storage/sqlite_store.py` | Default backend, export_csv | | SharePoint | `storage/sharepoint_store.py` | Graph client credentials | +### Config files (repo) + +| Path | Role | +|------|------| +| `config/keywords.yaml` | Eng-owned ingest filter terms + weights | +| `config/objectives.yaml` | Eng-owned company objectives for Grok ranking (not used by ingest filter) | + --- -## 3. Storage backends +## 3. Storage backends and views -### 3.1 SQLite (default) +### 3.1 SQLite (default system of record) - Path: `{DATA_DIR}/contract_opportunities.db` (default `data/…`) - Unique indexes on OpportunityID, Link - `list_rows` / `export_csv` for review - No cloud secrets +- **Grok never UPDATEs these rows** ### 3.2 SharePoint (optional) - MSAL client credentials → Graph - Paginated key load; create list items - Requires app registration + Sites.Selected grant (see provision runbook) -- Export-csv / sync-sheets: **sqlite-oriented** today +- Export-csv / sync-sheets / interpret-rank: **sqlite-oriented** today + +### 3.3 Google Sheets (derived views only) -### 3.3 Google Sheets (derived view) +| Tab | Command | Source | Mode | +|-----|---------|--------|------| +| `Ingest` (default) | `sync-sheets` | SQLite opportunities | Full clear + rewrite | +| `Ranked` (default) | `interpret-rank` / `sync-rank-sheets` | Grok ranking report | Full clear + rewrite | +| `Review` (optional) | human | operator notes | Not written by pipeline | - Not a `STORAGE_BACKEND` -- Command: `sync-sheets` - Extra deps: `pip install -e ".[sheets]"` (`gspread`, `google-auth`) -- Full replace of tab (default `Ingest`) +- Rankings **refused** if target tab name is `Ingest` +- Ranked columns: see `RANKING_EXPORT_COLUMNS` in `sheets_sync.py` + +### 3.4 Local ranking reports + +- Directory: `{DATA_DIR}/rankings/` (default `data/rankings/`, gitignored) +- Files: `interpret-{run_id}.json`, `interpret-{run_id}.md` --- @@ -94,9 +131,14 @@ Loaded from environment and optional **repo-root** `.env` (`config.py`). | `PARTIAL_ERROR_EXIT_THRESHOLD` | `5` | | | `TEAMS_WEBHOOK_URL` | unset | optional | | `GOOGLE_SHEET_ID` | unset | for sync-sheets | -| `GOOGLE_SHEET_TAB` | `Ingest` | | +| `GOOGLE_SHEET_TAB` | `Ingest` | opportunity rows | +| `GOOGLE_SHEET_RANK_TAB` | `Ranked` | Grok rankings | | `GOOGLE_SERVICE_ACCOUNT_FILE` | unset | path to JSON key | | `GOOGLE_SERVICE_ACCOUNT_JSON` | unset | inline JSON (CI) | +| `XAI_API_KEY` | unset | Grok interpret-rank | +| `XAI_BASE_URL` | `https://api.x.ai/v1` | OpenAI-compatible | +| `XAI_MODEL` | `grok-4.5` | | +| `OBJECTIVES_PATH` | `config/objectives.yaml` | company objectives for AI rank | | Azure / SharePoint IDs | unset | only for SP backend | --- @@ -115,6 +157,13 @@ python -m opportunity_ingest run --csv data/sample-openTenderNotice.csv python -m opportunity_ingest run --write --csv data/sample-openTenderNotice.csv --max-create 10 python -m opportunity_ingest export-csv python -m opportunity_ingest sync-sheets + +# Optional Grok ranking (post-ingest; does not mutate SQLite) +pip install -e ".[ai]" +# set XAI_API_KEY in root .env (and keep GOOGLE_* if Ranked push wanted) +python -m opportunity_ingest interpret-rank --status New --limit 20 +# Skip Ranked sheet: --no-sync-sheets +# Re-push last JSON: python -m opportunity_ingest sync-rank-sheets ``` **Observed sample dry-run (live CSV ~6MB):** parsed≈893, filtered≈83, mapped≈27, many map skips for empty Link (expected). @@ -128,8 +177,9 @@ python -m opportunity_ingest sync-sheets | CI | `.github/workflows/ci.yml` — ruff + pytest on 3.11/3.12 | | Daily ingest | `.github/workflows/daily-canadabuys-ingest.yml` — cron `0 14 * * *` UTC, sqlite default, cache `data/`+`state/`, MAX_CREATE soft cap | | Local daily + Sheets | Operator: Task Scheduler → `run --write` then `sync-sheets` (see google_sheets_setup.md) | +| Local + Grok Ranked | After write: `interpret-rank` (needs `XAI_API_KEY`; auto Ranked if sheet id set) | -Sheets sync is **not** yet wired into the GitHub Actions YAML by default (add optional step when secrets exist). +Sheets **Ingest** sync and **interpret-rank** are **not** yet wired into the GitHub Actions YAML by default (BACKLOG B-06). --- @@ -141,7 +191,8 @@ ruff check src tests ``` Fixtures: `tests/fixtures/open_tender_*.csv` -Sheets unit tests mock JSON load only (no live Google). +Sheets unit tests: SA JSON load + ranking grid/helpers (no live Google). +Grok unit tests: fake client (no live xAI). SharePoint tests mock HTTP (no live Graph). --- @@ -154,12 +205,17 @@ SharePoint tests mock HTTP (no live Graph). 4. **Windows double extension** `file.json.json` breaks service account path 5. **Map skips** when keyword hits but notice URL empty — by design 6. Quotes in `.env` values usually unnecessary +7. **Grok fit_score ≠ RelevanceScore** — do not treat Ranked fit as store score +8. **Do not put Grok output on Ingest** — use Ranked (code enforces) --- ## 9. Related docs +- **Human plug-and-play (keys, functions, roadmap):** [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) - Data rules: [`DATA_UPDATE_DIRECTIVES.md`](DATA_UPDATE_DIRECTIVES.md) - Change recipes: [`CHANGE_PLAYBOOK.md`](CHANGE_PLAYBOOK.md) - Decisions: [`DECISIONS.md`](DECISIONS.md) +- Backlog (not built): [`BACKLOG.md`](BACKLOG.md) +- Status snapshot: [`STATUS.md`](STATUS.md) - Original design: [`phase1-canadabuys-sharepoint-implementation-schema.md`](phase1-canadabuys-sharepoint-implementation-schema.md) diff --git a/docs/BACKLOG.md b/docs/BACKLOG.md new file mode 100644 index 0000000..fef955d --- /dev/null +++ b/docs/BACKLOG.md @@ -0,0 +1,310 @@ +# Backlog — discussed, deferred, or not yet built + +**Purpose:** Track ideas and designs that were **discussed or proposed** but are **not implemented** (or only partially documented). +**Do not treat this as as-built.** For what exists in code, use [`AS_BUILT.md`](AS_BUILT.md) and [`STATUS.md`](STATUS.md). + +| Field | Value | +|-------|--------| +| **Last updated** | 2026-08-11 | +| **Owner** | Engineering (product decisions via issues / review) | + +--- + +## How to use this file + +| Status | Meaning | +|--------|---------| +| **Proposed** | Discussed; no design decision locked | +| **Designed (not built)** | Approach agreed in conversation/docs; no production code | +| **Deferred** | Explicitly out of current phase | +| **Blocked** | Waiting on secrets, product answer, or dependency | + +When an item ships: move a one-line note to `STATUS.md` / `AS_BUILT.md` / `DECISIONS.md`, then mark the backlog row **Done** or remove it. + +--- + +## 1. Built recently (do not re-open as “missing”) + +These **are implemented** in code as of 2026-08-11. Listed here only so reviewers do not confuse them with backlog. + +| ID | Item | Where | +|----|------|--------| +| B-DONE-1 | Grok post-ingest interpret + fit rank | `interpret_rank.py`, CLI `interpret-rank` | +| B-DONE-2 | Company objectives frame of reference | `config/objectives.yaml` | +| B-DONE-3 | Local ranking reports | `data/rankings/interpret-*.{json,md}` (gitignored) | +| B-DONE-4 | Sheets **Ranked** tab full-replace | `sheets_sync.sync_rankings_to_sheet`, CLI `sync-rank-sheets` | +| B-DONE-5 | Auto sheet push when `GOOGLE_SHEET_ID` set | `interpret-rank` default; `--no-sync-sheets` to skip | +| B-DONE-6 | Hard refuse rankings → `Ingest` tab | `sync_rankings_to_sheet` | +| B-DONE-7 | Optional extras | `pip install -e ".[ai]"`, `".[sheets]"` | +| B-DONE-8 | Atlas Flow Group site-tuned keywords + objectives | `config/keywords.yaml`, `config/objectives.yaml` (crawl 2026-08-11) | +| B-DONE-9 | Teams high-match capture pipeline (≥40 score) | `config/notify.yaml`, `notify_match_alerts`, ingest write + interpret-rank | +| B-DONE-10 | Slack high-match engine (CLI/Bolt tokens + legacy webhook) | `SLACK_BOT_TOKEN`/`SLACK_CHANNEL_ID`, `slack-sdk`, `scripts/slack_cli_setup.md` | + +Normative rules: [`DATA_UPDATE_DIRECTIVES.md`](DATA_UPDATE_DIRECTIVES.md) §2.8, §3.2. + +**Note (2026-08-11):** Keyword and objective packs were derived from a review of the +Atlas Flow Group public site (home, services, approach, about, contact). +Automated site crawl → YAML (B-04) remains unbuilt; this was a one-off engineering tuning pass. + +--- + +## 2. Active backlog (discussed, not built) + +### B-01 — Source / website-type classifier +| | | +|--|--| +| **Status** | **Designed (not built)** | +| **Discussed** | 2026-08-11 — model keywords by interpreting “website type” / notice context | +| **Intent** | Classify each notice by portal (CanadaBuys / MERX / Ariba host), instrument (RFI/RFP/RFSA/ITQ), and category family (services vs construction) **before** keyword matching | +| **Proposed approach** | Cheap rules on URL + title/category cues → `SourceType` labels; no LLM required for daily path | +| **Depends on** | Product answers: which “type” axes matter first (portal vs instrument vs company site) | +| **Out of scope for now** | Full multi-source crawlers | + +### B-02 — Keyword packs by type +| | | +|--|--| +| **Status** | **Designed (not built)** | +| **Discussed** | 2026-08-11 — “most appropriate keywords from website type” | +| **Intent** | Split `keywords.yaml` into `global` + packs (`by_portal`, `by_category`, `by_instrument`); active terms = union of packs for classified type | +| **Proposed approach** | Extend `filter_keywords.py` loader; keep rule-based match engine | +| **Constraint** | Keywords remain **eng-owned** (D-10); no silent AI rewrite of production YAML | + +### B-03 — Grok `suggest-keywords` (human-in-the-loop) +| | | +|--|--| +| **Status** | **Proposed** | +| **Discussed** | 2026-08-11 — AI for discovery, not silent filter mutation | +| **Intent** | Offline/on-demand CLI that reads recent opportunities + false positives + `objectives.yaml` and **proposes** a keywords.yaml patch (PR), does not auto-merge | +| **Depends on** | B-01/B-02 optional; `XAI_API_KEY` already used by interpret-rank | +| **Must not** | Auto-edit `config/keywords.yaml` in the daily pipeline | + +### B-04 — Company website → capability vocabulary +| | | +|--|--| +| **Status** | **Proposed** | +| **Discussed** | 2026-08-11 — alternate “compare against” design: crawl *our* site, not only tender portals | +| **Intent** | Extract services language from company website sections → seed objectives + keyword suggestions | +| **Depends on** | Product: real company URL + which pages are authoritative | +| **Note** | Distinct from tender-portal type packs (B-01/B-02) | + +### B-05 — Notice `Link` page crawl / enrichment +| | | +|--|--| +| **Status** | **Proposed** | +| **Discussed** | 2026-08-11 — CSV description often thin; some rows already point at MERX/Ariba | +| **Intent** | Optional fetch of notice abstract HTML to improve filter/rank input | +| **Risks** | ToS, rate limits, auth walls, non-determinism; create-only store must not overwrite existing rows with “refresh” without new design | +| **Constraint** | Never invent Links; never silent-truncate Links | + +### B-06 — Wire Grok rank + Ranked Sheets into GitHub Actions +| | | +|--|--| +| **Status** | **Deferred** (ops) | +| **Intent** | After `run --write` (+ optional `sync-sheets`), run `interpret-rank` in Actions with `XAI_API_KEY` + sheet secrets | +| **Depends on** | Live Grok validation, cost/budget policy, secrets on repo | + +### B-07 — Live E2E validation of interpret-rank + Ranked tab +| | | +|--|--| +| **Status** | **Blocked** (operator secrets / credits) | +| **Intent** | Document a successful live run: `XAI_API_KEY` + real Grok call + Ranked tab visible on shared sheet | +| **Current** | Unit tests mock Grok; Sheets ranking helpers tested without network; live path not recorded as validated in STATUS §2 | + +--- + +## 3. Explicit phase non-goals (still deferred) + +Carried from Phase 1 design / AGENTS hard rules. These items are **intentionally deferred**. + +| ID | Item | Notes | +|----|------|--------| +| N-1 | Multi-source first-class adapters (MERX/municipal as ingest SoR) | CanadaBuys open-data CSV remains primary | +| N-2 | Learning-to-rank / AI scoring from human Status feedback | Separate from post-ingest Grok fit report | +| N-3 | Two-way Sheets ↔ SQLite Status sync | Sheets remain views | +| N-4 | Automatic amendment / closing-date row refresh | Operator opens `Link` | +| N-5 | Dual-write SQLite + SharePoint | Single `STORAGE_BACKEND` | +| N-6 | Auto-bidding / proposal generation | Out of scope | +| N-7 | Historical backfill campaigns | Day-forward ingest | +| N-8 | AI auto-mutates production `keywords.yaml` on schedule | Conflicts with D-10 / eng ownership | + +--- + +## 4. Open product questions (need answers before B-01–B-04) + +Recorded from 2026-08-11 design discussion. Update when answered. + +1. **Website type definition:** portal host vs notice instrument vs category family vs company website — which axis first? +2. **AI role on keywords:** suggest-only (PR) vs any automatic weight tweak? (Default recommendation: **suggest-only**.) +3. **Primary pain:** fewer false positives vs more true positives vs multi-source readiness? +4. **Ingest surface:** stay on CanadaBuys CSV only, or also crawl notice `Link` pages? +5. **Company identity:** keep template `objectives.yaml` or lock real firm name/services for ranking quality? + +--- + +## 5. Roadmap summary (humans) + +**Already built — needs only keys:** see [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) (Sheets, Grok, Teams/Slack, SQLite ingest). + +| Horizon | Focus | +|---------|--------| +| **This week (ops)** | Slack/Teams match cards stable; Actions secrets + one scheduled dry run | +| **This month (eng+ops)** | B-06 optional: Actions step for interpret-rank + Ranked; tune objectives from real Ranked outcomes | +| **Next** | B-01/B-02 type-aware keywords if false positives remain high | +| **Later** | B-03 suggest-keywords; B-04/B-05 crawls only with product OK | +| **Long horizon** | §7 multi-geography **lawful** source mesh + self-improving rank loop (not unrestricted web scrape) | +| **Not planned now** | N-* auto-bid, dual-write, two-way Status (see §3) | + +### Suggested implementation order (when prioritized) + +```text +0) Human: PLUG_AND_PLAY — channel webhooks + Actions secrets (no code) +1) B-07 finish remaining live proofs (Teams ops if needed) +2) B-06 Actions wire-up (if daily AI cost OK) +3) B-01 source type classifier (rules only) +4) B-02 keyword packs by type +5) B-03 suggest-keywords (Grok → PR diff) +6) B-04 / B-05 only after product answers +7) Long horizon: V-* items in §7 (adapter mesh, geography packs, feedback loop) +``` + +--- + +## 6. Related docs + +| Doc | Role | +|-----|------| +| [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) | Human keys, function list, go-live sequence | +| [`PROCESSOR_OVERVIEW.md`](PROCESSOR_OVERVIEW.md) | Processor role | +| [`STATUS.md`](STATUS.md) | What works / validated | +| [`AS_BUILT.md`](AS_BUILT.md) | Architecture that exists | +| [`DECISIONS.md`](DECISIONS.md) | Locked ADRs | +| [`DATA_UPDATE_DIRECTIVES.md`](DATA_UPDATE_DIRECTIVES.md) | MUST / MUST NOT | +| [`CHANGE_PLAYBOOK.md`](CHANGE_PLAYBOOK.md) | How to change keywords / Grok objectives | +| [`../AGENTS.md`](../AGENTS.md) | Hard rules for agents | + +--- + +## 7. Long-horizon vision — multi-geography opportunity intelligence + +**Status:** Future goals only (not scheduled for build). +**Recorded:** 2026-08-11 (product discussion: global data lakes / USMCA / self-sustaining system). +**Guardrails:** Only the constraints of **this project** ([`AGENTS.md`](../AGENTS.md), DATA_UPDATE, Phase 1 non-goals N-*). +**Processor context:** [`PROCESSOR_OVERVIEW.md`](PROCESSOR_OVERVIEW.md). + +### 7.1 Intent (restated from first principles) + +**Common informal request:** scrape every public data source for bids, any geography. + +**Goal this project will track (lawful and useful):** + +> Continuously discover **high-signal procurement / opportunity records** from **declared, permitted sources**, normalize them into **one opportunity schema**, rank them against **our objectives**, and improve filters from **human Status feedback** — for **any geography we choose to enable** — without auto-bidding and without violating create-only / eng-owned control planes. + +**Explicitly out of bounds (even as “future”):** + +- Unrestricted scraping of the whole public web +- Bypassing ToS, paywalls, auth, or robots where prohibited +- Inventing Links or silent-truncating URLs +- Auto-mutate production `keywords.yaml` without eng review (N-8) +- Auto-bid / proposal auto-submit (N-6) +- Treating Sheets as system of record + +Opaque translation of signals is allowed **only** as: structured extraction from a **registered source adapter** + optional LLM **interpretation** into the same schema — not unrestricted harvesting of arbitrary pages. + +### 7.2 Future goal register (V-*) + +| ID | Goal | Status | Notes | +|----|------|--------|--------| +| **V-1** | **Source registry** — catalog of lakes/portals per geography (open data APIs, official CSVs, licensed aggregators) | Proposed | Each source: license, rate limit, identity keys, health | +| **V-2** | **Adapter interface** — pluggable fetch→normalize→`TenderRecord`/`OpportunityFields` | Designed (not built) | Generalizes CanadaBuys download/parse; multi-source N-1 becomes real | +| **V-3** | **Geography packs** — keywords + objectives + legal flags per region (CA / US / other) | Proposed | USMCA: mark set-aside / domestic-preference when known | +| **V-4** | **Signal fusion** — same real-world tender from two portals → one OpportunityID/Link family | Proposed | Extends current dedupe; never invent IDs | +| **V-5** | **Enrichment lane** — optional fetch of notice `Link` body when CSV is thin | Proposed | Same as B-05; per-source ToS gate | +| **V-6** | **Opaque → structured** — LLM maps messy HTML/PDF fields into schema (post-fetch) | Proposed | Post-ingest / side table; do not overwrite Status | +| **V-7** | **Human feedback loop** — Status/Discarded/Bidding trains **suggested** keyword/objective diffs | Proposed | B-03 + N-2 hybrid; eng merge only | +| **V-8** | **Self-sustaining ops** — schedule, budgets (MAX_CREATE), cost caps (Grok), source health, Slack/Teams heartbeats | Partial | Daily Actions + notify exist; multi-source health TBD | +| **V-9** | **Advantage layer** — rank not only “keyword hit” but **winnability** (set-aside, clearance, geography, partner-needed) | Proposed | Grok objectives expand; still report-only | +| **V-10** | **Commercial + public** — optional private RFP feeds / partner drops as first-class sources | Proposed | Not only government open data | + +### 7.3 Evolutionary architecture (self-sustaining, under project constraints) + +```text + ┌──────────────────────────┐ + │ Source Registry (V-1) │ + │ CA · US · … (enabled) │ + └────────────┬─────────────┘ + │ lawful adapters only (V-2) + ▼ +┌──────────────┐ normalize ┌────────────────────────────┐ +│ CanadaBuys │─────────────────►│ Opportunity mesh (SQLite) │ +│ + future │ dedupe V-4 │ create-only SoR │ +└──────────────┘ └────────────┬───────────────┘ + │ + optional enrich V-5/V-6 │ + ▼ + ┌────────────────────────────┐ + │ Rank / interpret (Grok) │ + │ geography packs V-3 / V-9 │ + └────────────┬───────────────┘ + │ + views + alerts │ + Sheets · Slack · Teams │ + ▼ + ┌────────────────────────────┐ + │ Human Status (truth) │ + │ pursue / discard / bid │ + └────────────┬───────────────┘ + │ suggest-only V-7 + ▼ + ┌────────────────────────────┐ + │ Eng PR: keywords/objectives│ + │ (never silent AI overwrite)│ + └────────────────────────────┘ +``` + +**Self-sustaining means:** scheduled runs + budget caps + source health + human-in-the-loop learning **suggestions** — not unsupervised self-rewriting of production rules. + +### 7.4 Decision principles + +**Question:** How should this project pursue multi-geography coverage while staying inside project guardrails and remaining operable? + +**Review lenses (advisory):** + +| Seat | Lens | Core claim | +|------|------|------------| +| **Systems engineer** | Interfaces | One schema plus adapters is preferable to a single unrestricted crawler | +| **Procurement** | Markets | Geography is a **policy surface** (set-asides, TAA), not only additional HTTP sources | +| **Security / counsel** | Lawful use | Only sources that can be defended; license and terms of use are product features | +| **Capture lead** | ROI | Alert quality over coverage volume; Status feedback is the training signal | +| **Company positioning** | Fit | Canada public-sector plus partner/commercial sources before large US federal scope | + +**Atomic unit of value:** +A **deduplicated opportunity with a real Link**, a **justified fit score**, and a **human action** (open / pursue / discard) — not additional pages retrieved. + +**Adopted principles:** + +1. **Operating constraint:** Wins come from **timely, relevant, actionable** notices — not from total bytes retrieved. +2. **Exclude:** Unbounded crawl, dual system of record, auto-bid, silent keyword mutation. +3. **Keep / build:** Adapter mesh (V-2), geography packs (V-3), fusion dedupe (V-4), human feedback → engineering PR (V-7), cost-capped rank + alerts (already started). +4. **Expansion rule:** Add geography only when the **Canada loop is stable** and a **named source** has a clear license and identity model. +5. **Iteration:** Each new source is a **module with a disable switch**, not a rewrite of the core processor. + +**Decision:** +Pursue broader coverage via a **lawful, extensible source mesh and a human-feedback rank loop**, not unrestricted scraping. That is the only path that stays inside AGENTS hard rules and can remain operable. + +### 7.5 Phased enablement (when product prioritizes) + +| Phase | Deliver | Exit criteria | +|-------|---------|---------------| +| **G0** (now) | Single source CanadaBuys + rank + Slack | Operators use Status weekly | +| **G1** | V-1 registry + second **official** adapter (e.g. one more CA or licensed feed) | Create-only + dedupe still green | +| **G2** | V-3 geography pack US (flags only) + optional partner feed | Clear “skip set-aside” behavior | +| **G3** | V-5/V-6 enrichment + V-7 suggest-keywords from Status | Eng merges PRs; no silent YAML write | +| **G4** | Multi-geo schedule + cost/health automation V-8 | Budget alarms; dead sources auto-disable | + +### 7.6 Product questions for long horizon + +1. Which **second source** is first (MERX, provincial, SAM unrestricted only, commercial feed)? +2. Who owns **source license** approval? +3. Is **US** a geography pack for Atlas, or only partner-sub signals? +4. What Status values feed **positive** vs **negative** training for V-7? +5. Hard **monthly $ cap** on Grok + fetch? diff --git a/docs/CHANGE_PLAYBOOK.md b/docs/CHANGE_PLAYBOOK.md index 64dcf51..c19a637 100644 --- a/docs/CHANGE_PLAYBOOK.md +++ b/docs/CHANGE_PLAYBOOK.md @@ -54,6 +54,21 @@ How to implement common changes safely. Follow [`DATA_UPDATE_DIRECTIVES.md`](DAT --- +## E2. Tune Grok interpret-rank / company objectives + +1. Edit `config/objectives.yaml` (eng-owned; align with keyword groups when possible). +2. Prompt / schema / merge logic: `interpret_rank.py` + `tests/test_interpret_rank.py`. +3. **Never** write fit scores into SQLite opportunity rows. +4. Local run: `pip install -e ".[ai]"`, set `XAI_API_KEY`, then + `python -m opportunity_ingest interpret-rank --status New --limit 20`. +5. Review `data/rankings/interpret-*.md` and Sheets **Ranked** tab; adjust objectives, not Status automation. +6. Re-push without Grok: `python -m opportunity_ingest sync-rank-sheets`. +7. Skip sheet: `--no-sync-sheets`. + +**Not yet implemented** (do not invent code paths): website-type keyword packs, `suggest-keywords` CLI — track in [`BACKLOG.md`](BACKLOG.md) B-01–B-03. + +--- + ## F. Google Sheets behavior change 1. Default remains **full replace** of one tab unless product explicitly requests upsert. diff --git a/docs/DATA_UPDATE_DIRECTIVES.md b/docs/DATA_UPDATE_DIRECTIVES.md index 9fb08b2..a949a8c 100644 --- a/docs/DATA_UPDATE_DIRECTIVES.md +++ b/docs/DATA_UPDATE_DIRECTIVES.md @@ -3,7 +3,7 @@ **Normative rules for any agent or developer that writes, migrates, or syncs opportunity data.** Violating these breaks dedupe, human review, or external views. -Last updated: 2026-08-10. +Last updated: 2026-08-11. --- @@ -14,6 +14,8 @@ Last updated: 2026-08-10. | **SQLite** `data/contract_opportunities.db` | **System of record (day-1)** | `STORAGE_BACKEND=sqlite` | | **SharePoint list** | Alternate store when activated | `STORAGE_BACKEND=sharepoint` | | **Google Sheets `Ingest` tab** | **Derived view only** (full replace) | `sync-sheets` after writes | +| **Google Sheets `Ranked` tab** | **Derived Grok ranking view** (full replace) | `interpret-rank` / `sync-rank-sheets` | +| **`data/rankings/*`** | Local Grok reports (not SoR) | `interpret-rank` | | **CSV export** | Snapshot for Excel / import | `export-csv` | **MUST:** Treat SQLite (or active SharePoint backend) as authoritative for create/dedupe. @@ -34,6 +36,8 @@ Last updated: 2026-08-10. | `export-csv` | No store write; writes CSV file | | `sync-sheets` | No SQLite write; **replaces** Sheet tab | | `download-sample` | Writes sample CSV file only | +| `interpret-rank` | **No SQLite write**; writes `data/rankings/*`; may full-replace Sheets **Ranked** tab | +| `sync-rank-sheets` | **No SQLite write**; full-replace Sheets **Ranked** tab from ranking JSON | **MUST:** Use `--write` for persistence. **MUST NOT:** Rely on `DRY_RUN` env to enable writes (it never does). @@ -94,10 +98,22 @@ normalize_link(url) = strip → lower → rstrip('/') **MUST NOT:** Overwrite Status on later ingest. **Human Status values:** `New` | `Reviewing` | `Relevant` | `Bidding` | `Discarded`. +### 2.8 Grok interpret-rank (post-ingest) + +**MUST:** Treat AI ranking as a **derived report**, not the system of record. +**MUST NOT:** UPDATE `contract_opportunities` Status, Notes, RelevanceScore, Title, Description, or Link from Grok output. +**MUST:** Persist local interpret-rank files under `data/rankings/` (JSON + Markdown) or an explicit `--out-dir`. +**MUST:** Keep company objectives in engineering-owned `config/objectives.yaml` (not free-text secrets). +**MUST NOT:** Commit `XAI_API_KEY` or ranking report dumps that contain secrets. +**MUST:** Google Sheets ranking push targets the **Ranked** tab only (or another non-`Ingest` name via `GOOGLE_SHEET_RANK_TAB`). +**MUST NOT:** Write Grok rankings into the opportunity **`Ingest`** tab (code refuses this). + --- ## 3. Google Sheets sync directives +### 3.1 Opportunity tab (`sync-sheets`) + Command: `python -m opportunity_ingest sync-sheets` | Rule | Detail | @@ -110,10 +126,20 @@ Command: `python -m opportunity_ingest sync-sheets` | **MUST** | File name exactly `*.json` (Windows double-extension `.json.json` is a known failure mode) | | **MUST** | Load `.env` from **repo root** (not `scripts/.env`) | +### 3.2 Ranked tab (`interpret-rank` / `sync-rank-sheets`) + +| Rule | Detail | +|------|--------| +| **MUST** | Full **clear + rewrite** of target tab (default `Ranked` / `GOOGLE_SHEET_RANK_TAB`) | +| **MUST** | Same spreadsheet + service account as opportunity sync (`GOOGLE_SHEET_ID`) | +| **MUST NOT** | Target the `Ingest` tab for rankings | +| **Default** | When `GOOGLE_SHEET_ID` is set, `interpret-rank` syncs Ranked unless `--no-sync-sheets` | +| **Re-push** | `sync-rank-sheets` uses latest `data/rankings/interpret-*.json` (no Grok call) | + **After ingest recommended sequence:** ```text -run --write → sync-sheets → (optional) export-csv +run --write → sync-sheets → (optional) interpret-rank → (optional) export-csv ``` **MUST NOT:** Two-way sync Status from Sheets back to SQLite in Phase 1 (out of scope). diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index 9c9ab4f..a6f41a2 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -22,13 +22,28 @@ Condensed ADRs for Phase 1. See also design-doc Key Decisions (rev 4). | D-16 | `sync-sheets` optional extra (`[sheets]`) | Keep core install light | | D-17 | EN preferred, FR fallback on bilingual CSV | Maximize non-empty Title/Description/Link | | D-18 | Closing naive times as fixed UTC−05:00 | Deterministic vs DST-dependent ZoneInfo | +| D-19 | Grok ranking is **post-ingest only** (optional) | Ingest stays explainable rule-based; AI never blocks create path | +| D-20 | Grok never UPDATEs SQLite Status/Notes/RelevanceScore | Protects human triage; AI is a report, not SoR | +| D-21 | Company objectives in `config/objectives.yaml` (eng-owned) | Same review model as keywords; frame for fit_score | +| D-22 | Optional `[ai]` extra + `XAI_API_KEY` (xAI / Grok) | Keep core install light; OpenAI-compatible `https://api.x.ai/v1` | +| D-23 | Google Sheets **Ranked** tab separate from **Ingest** | Full-replace both; refuse writing rankings into Ingest | +| D-24 | Auto Ranked sync when `GOOGLE_SHEET_ID` set (`--no-sync-sheets` opt-out) | Live sheet path without extra flags; explicit skip for offline | +| D-25 | `sync-rank-sheets` re-push from local JSON | Avoid re-calling Grok to refresh the sheet | --- ## Explicit non-decisions (deferred) -- Multi-source adapters (MERX, municipal) -- Learning-to-rank / AI scoring from Status feedback +Tracked in detail: [`BACKLOG.md`](BACKLOG.md). + +- Multi-source adapters (MERX, municipal) as first-class ingest +- Learning-to-rank / AI scoring from Status feedback into store - Two-way Sheets ↔ SQLite Status sync - Automatic amendment/closing-date refresh - Dual-write SQLite + SharePoint +- Website/source-type classifier + keyword packs by portal (B-01, B-02) +- Grok auto-suggest keywords into production YAML without human PR (forbidden; B-03 is suggest-only if built) +- Company-website crawl for capability vocabulary (B-04) +- Notice `Link` HTML crawl enrichment (B-05) +- Auto-bidding / proposal generation +- Historical backfill campaigns diff --git a/docs/INDEX.md b/docs/INDEX.md index 86ecdcd..3110ce0 100644 --- a/docs/INDEX.md +++ b/docs/INDEX.md @@ -10,7 +10,10 @@ Central map for humans and LLMs. **Start at [`../AGENTS.md`](../AGENTS.md).** |-----|-------------| | [`../AGENTS.md`](../AGENTS.md) | Entry point: hard rules, module map, task routing | | [`STATUS.md`](STATUS.md) | Living implementation / ops readiness snapshot | +| [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) | **Human keys + next steps** to turn on built features | +| [`PROCESSOR_OVERVIEW.md`](PROCESSOR_OVERVIEW.md) | **Plain-language view:** this app as one processor — triggers, I/O, place in system | | [`AS_BUILT.md`](AS_BUILT.md) | Implemented architecture (source of truth for “what exists now”) | +| [`BACKLOG.md`](BACKLOG.md) | Roadmap / not built — includes **§7 long-horizon multi-geo vision** | | [`DATA_UPDATE_DIRECTIVES.md`](DATA_UPDATE_DIRECTIVES.md) | Normative data write/sync rules (MUST / MUST NOT) | | [`CHANGE_PLAYBOOK.md`](CHANGE_PLAYBOOK.md) | Step recipes for common development changes | | [`DECISIONS.md`](DECISIONS.md) | Key architectural decisions (ADR-style, as-built) | @@ -31,6 +34,7 @@ Central map for humans and LLMs. **Start at [`../AGENTS.md`](../AGENTS.md).** |-----|-------------| | [`../scripts/ops_runbook.md`](../scripts/ops_runbook.md) | Daily triage, MAX_CREATE policy, calibration, rollback | | [`../scripts/google_sheets_setup.md`](../scripts/google_sheets_setup.md) | Free Sheets service-account setup + `sync-sheets` | +| [`../scripts/slack_cli_setup.md`](../scripts/slack_cli_setup.md) | Slack CLI / Bolt tokens for match notify | | [`../scripts/provision_sharepoint_list.md`](../scripts/provision_sharepoint_list.md) | SharePoint list + Sites.Selected activation | --- @@ -41,7 +45,9 @@ Central map for humans and LLMs. **Start at [`../AGENTS.md`](../AGENTS.md).** |------|-------------| | [`../.env.example`](../.env.example) | Root env template | | [`../config/settings.example.env`](../config/settings.example.env) | Settings mirror | -| [`../config/keywords.yaml`](../config/keywords.yaml) | Keyword groups (eng-owned) | +| [`../config/keywords.yaml`](../config/keywords.yaml) | Keyword groups for ingest filter (eng-owned) | +| [`../config/objectives.yaml`](../config/objectives.yaml) | Company objectives for Grok interpret-rank (eng-owned) | +| [`../config/notify.yaml`](../config/notify.yaml) | Teams match threshold / card limits | --- @@ -50,3 +56,5 @@ Central map for humans and LLMs. **Start at [`../AGENTS.md`](../AGENTS.md).** | Doc | Description | |-----|-------------| | [`../README.md`](../README.md) | Quick start, CLI, secrets table, links | +| [`PROCESSOR_OVERVIEW.md`](PROCESSOR_OVERVIEW.md) | Plain-language processor role, triggers, outputs | +| [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) | Operator plug-in checklist (keys → working stack) | diff --git a/docs/PLUG_AND_PLAY.md b/docs/PLUG_AND_PLAY.md new file mode 100644 index 0000000..99f9655 --- /dev/null +++ b/docs/PLUG_AND_PLAY.md @@ -0,0 +1,371 @@ +# Plug-and-play guide — human keys & next steps + +**Audience:** operators and humans who will turn on what engineering already built. +**As-of:** 2026-08-11 +**Code status:** features below are **implemented**; they stay idle until secrets and configuration are provided. + +| Related docs | | +|--------------|--| +| **What this processor is** | [`PROCESSOR_OVERVIEW.md`](PROCESSOR_OVERVIEW.md) | +| What works now | [`STATUS.md`](STATUS.md) | +| Architecture | [`AS_BUILT.md`](AS_BUILT.md) | +| Future / not built | [`BACKLOG.md`](BACKLOG.md) · roadmap § below | +| Data rules | [`DATA_UPDATE_DIRECTIVES.md`](DATA_UPDATE_DIRECTIVES.md) | +| Daily ops | [`../scripts/ops_runbook.md`](../scripts/ops_runbook.md) | + +--- + +## 1. Big picture (what the product does) + +```text +CanadaBuys open tenders (CSV) + │ + ▼ + keyword filter + rule score (0–100) + │ + ▼ + SQLite create-only ◄── system of record + │ + ├──► export-csv + ├──► Google Sheets tab Ingest (view) + ├──► Grok interpret-rank (plain English + fit 0–100) + │ ├──► data/rankings/*.md + │ └──► Google Sheets tab Ranked + └──► Teams + ├── ops alerts (fail / streak) + └── capture pings when score ≥ 40 +``` + +**Not auto-bidding.** Humans still set Status and decide pursue/pass. + +--- + +## 2. Function list (what you can run) + +| CLI command | What it does | Needs human key? | +|-------------|--------------|------------------| +| `download-sample` | Download CanadaBuys open-tender CSV sample | No (public; User-Agent in code) | +| `run` | Dry-run ingest (default) | No | +| `run --write` | Create **new** opportunities in SQLite | No for local SQLite | +| `check-store` | Health + key counts | No (sqlite) | +| `export-csv` | Dump SQLite → CSV | No | +| `sync-sheets` | Full-replace Sheets **Ingest** | **Yes** — Google SA + sheet ID | +| `interpret-rank` | Grok rephrase + rank; optional Ranked tab + Teams | **Yes** — `XAI_API_KEY` (+ Sheets/Teams optional) | +| `sync-rank-sheets` | Re-push last ranking JSON → **Ranked** | **Yes** — Google SA + sheet ID | +| *(internal)* Teams match notify | Card when RelevanceScore or Grok fit ≥ threshold | **Yes** — Teams webhook | + +**Install extras (one-time on the machine):** + +```powershell +pip install -e . +pip install -e ".[sheets]" # Google Sheets +pip install -e ".[ai]" # Grok / xAI +``` + +--- + +## 3. Keys & secrets checklist (you provide) + +Copy `.env.example` → **repo-root** `.env` (never `scripts/.env`). Never commit `.env` or `secrets/`. + +| # | Secret / config | Where to get it | Unlocks | +|---|-----------------|-----------------|---------| +| 1 | *(none)* | — | Local ingest + SQLite + export-csv | +| 2 | `GOOGLE_SHEET_ID` | Spreadsheet URL `…/d//edit` | Sheets sync | +| 3 | `GOOGLE_SERVICE_ACCOUNT_FILE` or `GOOGLE_SERVICE_ACCOUNT_JSON` | Google Cloud SA key JSON | Sheets API | +| 4 | Share sheet **Editor** with SA email | Sheets UI Share | Sheets write works | +| 5 | `XAI_API_KEY` | [console.x.ai](https://console.x.ai) API keys | `interpret-rank` | +| 6 | `TEAMS_WEBHOOK_URL` | Teams channel → Workflows → “Post when webhook received” | Ops alerts + match fallback | +| 7 | `TEAMS_MATCH_WEBHOOK_URL` *(optional)* | Second Workflows webhook | Dedicated capture channel | +| 8 | `TEAMS_MATCH_SCORE_THRESHOLD` | Default **40** via env or `config/notify.yaml` | When to ping (Teams **and** Slack) | +| 8b | `SLACK_BOT_TOKEN` + `SLACK_CHANNEL_ID` | Slack CLI / Bolt app (`xoxb-` + channel) | High-match Slack posts (preferred) | +| 8c | `SLACK_APP_TOKEN` | Optional `xapp-` for Socket Mode only | Not required for match posts | +| 9 | Azure / SharePoint secrets | Only if `STORAGE_BACKEND=sharepoint` | Alternate store (not required day-1) | +| 10 | GitHub Actions secrets/vars | Repo Settings → Secrets | Unattended daily schedule | + +### Minimal `.env` templates + +**A — Local SQLite only (works immediately after `pip install -e .`)** + +```env +STORAGE_BACKEND=sqlite +DATA_DIR=data +MAX_CREATE=50 +LOG_LEVEL=INFO +``` + +**B — + Google Sheets (Ingest + Ranked)** + +```env +GOOGLE_SHEET_ID=1yourSpreadsheetId +GOOGLE_SHEET_TAB=Ingest +GOOGLE_SHEET_RANK_TAB=Ranked +GOOGLE_SERVICE_ACCOUNT_FILE=secrets/google-service-account.json +``` + +**C — + Grok ranking** + +```env +XAI_API_KEY=xai-... +XAI_MODEL=grok-4.5 +OBJECTIVES_PATH=config/objectives.yaml +``` + +**D — + Teams** + +```env +TEAMS_WEBHOOK_URL=https://...powerautomate.../invoke +# optional separate capture channel: +# TEAMS_MATCH_WEBHOOK_URL=https://... +TEAMS_MATCH_NOTIFY_ENABLED=true +TEAMS_MATCH_SCORE_THRESHOLD=40 +``` + +**E — + Slack via Slack CLI / Bolt tokens (same match criteria as Teams)** + +```env +# https://docs.slack.dev/tools/slack-cli/ — full steps: scripts/slack_cli_setup.md +SLACK_BOT_TOKEN=xoxb-... +SLACK_CHANNEL_ID=C... +# SLACK_APP_TOKEN=xapp-... # optional; slack run Socket Mode only +SLACK_MATCH_NOTIFY_ENABLED=true +TEAMS_MATCH_SCORE_THRESHOLD=40 +``` + +```powershell +pip install -e ".[slack]" +``` + +Detailed Sheets steps: [`../scripts/google_sheets_setup.md`](../scripts/google_sheets_setup.md). +Slack CLI paste-config: [`../scripts/slack_cli_setup.md`](../scripts/slack_cli_setup.md). + +--- + +## 4. Human plug-and-play sequence (recommended order) + +Do these in order. Skip optional blocks you do not need yet. + +### Step 0 — One-time machine setup (no keys) + +```powershell +cd "" +python -m venv .venv +.\.venv\Scripts\Activate.ps1 +pip install -r requirements.txt -r requirements-dev.txt +pip install -e . +copy .env.example .env +# edit .env with STORAGE_BACKEND=sqlite at minimum +``` + +**Verify:** + +```powershell +python -m opportunity_ingest --help +python -m opportunity_ingest check-store +``` + +### Step 1 — Ingest works offline + +```powershell +python -m opportunity_ingest download-sample +python -m opportunity_ingest run --csv data/sample-openTenderNotice.csv +python -m opportunity_ingest run --write --csv data/sample-openTenderNotice.csv --max-create 25 +python -m opportunity_ingest export-csv +python -m opportunity_ingest check-store +``` + +**You should see:** SQLite file under `data/`, CSV export, non-zero opportunity count. + +### Step 2 — Google Sheets (human actions) + +1. Create a Google Cloud **service account** → download JSON key. +2. Save as `secrets/google-service-account.json` (**not** `.json.json`). +3. Create a Google Sheet; rename first tab **`Ingest`** (optional tab **`Ranked`** / **`Review`**). +4. **Share** the sheet with the service account email as **Editor**. +5. Put `GOOGLE_SHEET_ID` + `GOOGLE_SERVICE_ACCOUNT_FILE` in root `.env`. +6. `pip install -e ".[sheets]"` +7. Run: + +```powershell +python -m opportunity_ingest sync-sheets +``` + +**You should see:** Ingest tab filled with opportunity rows (full replace each run). + +### Step 3 — Grok ranking (human actions) + +1. Create key at [console.x.ai](https://console.x.ai); add credits. +2. Set `XAI_API_KEY` in root `.env`. +3. Review/edit **`config/objectives.yaml`** (company frame — currently Atlas Flow Group). +4. `pip install -e ".[ai]"` +5. Run: + +```powershell +python -m opportunity_ingest interpret-rank --status New --limit 27 +``` + +**You should see:** + +- `data/rankings/interpret-*.md` +- Sheets tab **`Ranked`** updated (if Sheets configured) +- Console: `top #1 fit=…` + +Skip sheet: `--no-sync-sheets`. Skip Teams: `--no-teams`. + +### Step 4 — Teams and/or Slack capture pings (human actions) + +**Teams** + +1. **Workflows** → *Post to a channel when a webhook request is received* → copy URL. +2. Set `TEAMS_WEBHOOK_URL` (and optionally `TEAMS_MATCH_WEBHOOK_URL`). + +**Slack (CLI / Bolt config)** + +1. Install [Slack CLI](https://docs.slack.dev/tools/slack-cli/), run `slack login`, create/install an app with **`chat:write`**. +2. Paste **Bot User OAuth Token** → `SLACK_BOT_TOKEN=xoxb-...` +3. Set **channel** → `SLACK_CHANNEL_ID=C...` (invite bot to channel). +4. `pip install -e ".[slack]"` — see [`../scripts/slack_cli_setup.md`](../scripts/slack_cli_setup.md). + +**Shared criteria** + +3. Threshold **40** (`TEAMS_MATCH_SCORE_THRESHOLD` or `config/notify.yaml` → `match.score_threshold`). +4. Enable flags: `TEAMS_MATCH_NOTIFY_ENABLED` / `SLACK_MATCH_NOTIFY_ENABLED` (default true). +5. Trigger: + +```powershell +# New high rule-score creates: +python -m opportunity_ingest run --write --max-create 10 + +# Or Grok fits ≥ 40: +python -m opportunity_ingest interpret-rank --limit 27 +``` + +**You should see:** Teams Adaptive Card and/or Slack Block Kit message with summaries + open-notice buttons. +Ops alerts (hard fail / streak) remain **Teams-only** today (`TEAMS_WEBHOOK_URL`). + +### Step 5 — Daily automation (pick one) + +| Path | Human work | +|------|------------| +| **Windows Task Scheduler** | Point at `scripts/daily_sync.ps1` (or extend it with `interpret-rank`) | PC must be on | +| **GitHub Actions** | Push `dev`/`main`; set secrets below; run `workflow_dispatch` once | Unattended | + +**GitHub secrets / vars to plug:** + +| Name | Required for | +|------|----------------| +| `TEAMS_WEBHOOK_URL` | Alerts (recommended) | +| `INGEST_MAX_CREATE` (variable) | Soft create budget (default 50) | +| `GOOGLE_SHEET_ID` + `GOOGLE_SERVICE_ACCOUNT_JSON` | Sheets in CI (optional; add step) | +| `XAI_API_KEY` | interpret-rank in CI (optional; not in workflow by default yet — BACKLOG B-06) | + +Workflow file: [`.github/workflows/daily-canadabuys-ingest.yml`](../.github/workflows/daily-canadabuys-ingest.yml). + +### Step 6 — Human triage (ongoing, no more keys) + +1. Open Sheets **Ingest** or **Ranked**, or `export-csv`. +2. Set **Status** / Notes on rows you care about (SQLite or Review tab — **not** relying on Ingest if it is auto-replaced). +3. Act on Teams CTAs (open Link, decide pursue/pass). +4. Request keyword/objective changes via eng (`config/keywords.yaml`, `config/objectives.yaml`). + +--- + +## 5. Config files you may edit (no rebuild) + +| File | Who | Purpose | +|------|-----|---------| +| `.env` | Ops | Secrets + paths (gitignored) | +| `config/keywords.yaml` | Eng | Ingest keyword groups / weights | +| `config/objectives.yaml` | Eng | Grok ranking company frame | +| `config/notify.yaml` | Eng/Ops | Match threshold, max items, enable flag | +| `secrets/google-service-account.json` | Ops | Sheets key (gitignored) | + +--- + +## 6. End-to-end verification (all integrations enabled) + +After Steps 0–4: + +```powershell +.\.venv\Scripts\Activate.ps1 +python -m opportunity_ingest run --write --max-create 50 +python -m opportunity_ingest sync-sheets +python -m opportunity_ingest interpret-rank --status New --limit 50 +``` + +| Output | Where | +|--------|--------| +| New rows | SQLite `data/contract_opportunities.db` | +| Opportunity grid | Sheets **Ingest** | +| Grok brief | `data/rankings/interpret-*.md` + Sheets **Ranked** | +| Capture ping | Teams channel if any score ≥ 40 | + +--- + +## 7. Feature readiness vs human plug + +| Capability | Code | Human must plug | Live-validated? | +|------------|------|-----------------|-----------------| +| CanadaBuys download + filter + score | Yes | — | Yes (local) | +| SQLite create-only store | Yes | — | Yes | +| export-csv | Yes | — | Yes | +| Sheets **Ingest** | Yes | SA + sheet share + ID | Yes (local) | +| Grok **interpret-rank** | Yes | `XAI_API_KEY` | Yes (local) | +| Sheets **Ranked** | Yes | same Sheets as Ingest | Yes (local) | +| Teams ops alerts | Yes | `TEAMS_WEBHOOK_URL` | Code ready; requires a configured webhook | +| Teams match ≥40 | Yes | webhook + threshold | Code ready; requires a configured webhook | +| Slack match ≥40 | Yes | `SLACK_WEBHOOK_URL` + same threshold | Code ready; requires a configured webhook | +| GitHub daily schedule | Yes | repo secrets + enable schedule | Not fully go-live | +| SharePoint SoR | Yes | Entra + Sites.Selected | Not activated | +| Type-based keyword packs / auto crawl | No | — | See roadmap | + +--- + +## 8. Roadmap (what’s next — not plug-and-play yet) + +Full detail: [`BACKLOG.md`](BACKLOG.md). Summary for humans: + +| Priority | ID | Item | Blocked on | +|----------|-----|------|------------| +| Now (ops) | B-07 | Confirm Teams webhook live in channel | Human Workflows URL | +| Now (ops) | — | GitHub Actions secrets + first scheduled run | Human repo settings | +| Near | B-06 | Wire interpret-rank + Ranked + Teams into Actions YAML | Secrets + cost policy | +| Near | B-01 / B-02 | Source-type classifier + keyword packs | Product priority | +| Later | B-03 | Grok suggest-keywords (PR only) | Design sign-off | +| Later | B-04 / B-05 | Company-site crawl / notice page crawl | Product + ToS | +| Long horizon | V-1…V-10 | Multi-geography **lawful** source mesh + self-sustaining loop | [`BACKLOG.md`](BACKLOG.md) **§7** | +| Deferred | N-* | Unrestricted web scrape, auto-bid, dual-write, two-way Status | Project guardrails | + +**Suggested human order after keys work:** + +1. Plug Slack/Teams → prove one match card. +2. Enable Actions schedule with small `INGEST_MAX_CREATE`. +3. Weekly: review Ranked + Status triage. +4. Eng: tune keywords/objectives from real false positives. +5. Only then: discuss §7 second source / geography pack. + +--- + +## 9. Quick troubleshooting + +| Symptom | Check | +|---------|--------| +| CanadaBuys **403** | Code has User-Agent; retry; network block | +| `set XAI_API_KEY` | Root `.env`; restart shell; `pip install -e ".[ai]"` | +| Sheets permission error | Share sheet with **SA email** as Editor | +| SA file not found | Exact path; not `.json.json` | +| Teams silent | Webhook URL set; Workflows enabled; threshold too high vs scores | +| No new SQLite rows | Create-only + dedupe; already ingested IDs skip | +| Ranked empty | Run `interpret-rank` with key; not the same as `sync-sheets` | + +--- + +## 10. Roles and responsibilities + +| Role | Does | +|------|------| +| **Operators** | Keys, sheet share, Teams Workflow, Status triage, bid/no-bid | +| **Engineering** | keywords.yaml, objectives.yaml, code, tests, Actions YAML | +| **System (already built)** | Download, filter, score, store, export, Sheets replace, Grok rank, Teams cards | + +When something is built but idle, the fix is almost always **this guide §3–4**, not new code. diff --git a/docs/PROCESSOR_OVERVIEW.md b/docs/PROCESSOR_OVERVIEW.md new file mode 100644 index 0000000..62c6965 --- /dev/null +++ b/docs/PROCESSOR_OVERVIEW.md @@ -0,0 +1,220 @@ +# Opportunity Ingest — one key processor in the system + +**Audience:** operators, capture leads, stakeholders (non-engineers OK). +**As-of:** 2026-08-11 +**Package:** `opportunity_ingest` (this repo) + +This page explains **what this app is**, **where it sits** in the wider contract-bidding / capture setup, **what triggers it**, and **what comes out** — in plain language. + +| Want… | Go to | +|-------|--------| +| Keys & how to turn features on | [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) | +| Technical architecture | [`AS_BUILT.md`](AS_BUILT.md) | +| Day-to-day ops | [`../scripts/ops_runbook.md`](../scripts/ops_runbook.md) | +| What is / isn’t built | [`STATUS.md`](STATUS.md) · [`BACKLOG.md`](BACKLOG.md) | + +--- + +## 1. Plain-language description + +**In one sentence:** +This application is a **scheduled monitor for public tenders**. It downloads open notices, retains those that match configured capabilities, stores them, optionally ranks them in plain English with Grok, and can **notify Slack or Teams** when a score meets the threshold. + +**What it is not:** + +- Not the company website (e.g. Atlas Flow Group marketing site) +- Not a bot that submits bids +- Not a full CRM or proposal writer +- Not an automated agent that sets Status + +**What a human still does:** +Open the notice link, set Status (New → Reviewing → Bidding / Discarded), and choose pursue / pass. + +--- + +## 2. Where this processor sits (end-to-end) + +Capture is a sequence of stages. This repository implements **one stage**: find and alert on new public opportunities. + +```text +┌─────────────────────────────────────────────────────────────────────────┐ +│ Wider capture / bidding system (conceptual) │ +│ │ +│ [1] Public market data │ +│ CanadaBuys open tender notices (CSV open data) │ +│ │ │ +│ ▼ │ +│ [2] ★ THIS APP — Opportunity Ingest processor ★ │ +│ Filter → score → store → optional AI rank → notify │ +│ │ │ +│ ▼ │ +│ [3] Human capture workspace │ +│ SQLite / Google Sheets (Ingest + Ranked) / CSV export │ +│ Slack (or Teams) channel cards with “Open notice” │ +│ │ │ +│ ▼ │ +│ [4] Human decision (outside this app’s automation) │ +│ Status, notes, bid / no-bid, proposals, submission │ +│ │ │ +│ ▼ │ +│ [5] Company positioning (related, not the same code) │ +│ Company website → informs keywords.yaml and objectives.yaml │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +| Step | Role | This repo? | +|------|------|------------| +| **1. Market feed** | Public open-tender list | Download only | +| **2. Ingest processor** | Filter, score, store, rank, alert | **Yes — primary job** | +| **3. Workspace views** | Sheets / Slack / CSV for people | Writes **views** / messages | +| **4. Decision & bid** | Humans pursue/pass | **No auto-bid** | +| **5. Brand / capabilities site** | Company positioning | Config only (`keywords` / `objectives`) | + +--- + +## 3. What this processor does (steps inside the app) + +| Internal step | Plain language | Technical name | +|---------------|----------------|----------------| +| A. Download | Get today’s open tender CSV from CanadaBuys | `download` / `run` | +| B. Parse | Read bilingual columns; prefer English | `parse` | +| C. Keyword filter | Keep rows that match our capability words | `filter_keywords` + `config/keywords.yaml` | +| D. Rule score | Give a 0–100 “relevance” number (rules, not AI) | `score` → `RelevanceScore` | +| E. Dedupe + store | Save **new** opportunities only; never overwrite human Status | SQLite create-only | +| F. Optional Sheets Ingest | Full-replace a spreadsheet tab for browsing | `sync-sheets` | +| G. Optional Grok rank | Rewrite in plain English; score fit to company objectives | `interpret-rank` → `fit_score` | +| H. Optional Sheets Ranked | Full-replace ranked tab | `sync-rank-sheets` / auto with rank | +| I. Optional Slack / Teams | Ping channel when score ≥ threshold (default **40**) | `notify` match alerts | + +--- + +## 4. Trigger points (when this processor runs) + +Nothing runs by itself unless **you or a scheduler** starts it. + +| Trigger | How | Typical use | +|---------|-----|-------------| +| **Manual CLI** | Operator runs a command in the project folder | Smoke tests, ad-hoc refresh | +| **Windows Task Scheduler** | `scripts/daily_sync.ps1` (or similar) on a PC | Local daily job | +| **GitHub Actions schedule** | Workflow cron (e.g. ~14:00 UTC) | Unattended cloud daily | +| **Manual Actions** | `workflow_dispatch` on GitHub | One-off remote run | +| **Follow-on commands** | After ingest, human/script runs rank or sheets | Rank + Slack after store is warm | + +### Command-level triggers + +| You run… | Trigger effect | +|----------|----------------| +| `python -m opportunity_ingest run` | **Dry-run** filter/score (no DB write by default) | +| `python -m opportunity_ingest run --write` | **Processor write path:** create new DB rows; if score ≥ 40 → Slack/Teams match card | +| `python -m opportunity_ingest sync-sheets` | Push DB → Sheets **Ingest** tab | +| `python -m opportunity_ingest interpret-rank` | Grok ranks stored rows; Sheets **Ranked**; if fit ≥ 40 → Slack/Teams | +| `python -m opportunity_ingest sync-rank-sheets` | Re-push last ranking JSON to Sheets (no Grok call) | +| `python -m opportunity_ingest export-csv` | Snapshot file for Excel | +| `python -m opportunity_ingest check-store` | Health only | + +### Score thresholds that trigger alerts + +| Signal | Meaning | Default alert bar | +|--------|---------|-------------------| +| **RelevanceScore** | Rule-based match after keywords | ≥ **40** on **new creates** (`run --write`) | +| **Grok fit_score** | Fit vs `config/objectives.yaml` | ≥ **40** after `interpret-rank` | + +Configured in `config/notify.yaml` / `TEAMS_MATCH_SCORE_THRESHOLD` (shared for Slack + Teams). + +--- + +## 5. Inputs and outputs (black box) + +### Inputs + +| Input | Source | +|-------|--------| +| CanadaBuys open tender CSV | Public download or `--csv` file | +| Keyword list | `config/keywords.yaml` (eng-owned) | +| Company objectives (for Grok) | `config/objectives.yaml` | +| Secrets | Root `.env` (never committed) | +| Existing DB keys | SQLite (dedupe so we don’t double-add) | + +### Outputs + +| Output | What a human sees | +|--------|-------------------| +| SQLite DB | Master list of opportunities (`data/contract_opportunities.db`) | +| CSV export | Spreadsheet file for offline review | +| Google Sheets **Ingest** | Grid of opportunities (overwritten each sync) | +| Google Sheets **Ranked** | Grok-sorted plain-English brief | +| Local ranking files | `data/rankings/interpret-*.md` | +| **Slack channel** | Block Kit card: score, summary, **Open notice** button | +| Teams channel | Adaptive Card (ops + optional match) | +| Run logs | `logs/run-*.json` metrics | + +--- + +## 6. Slack as a notification channel + +When the processor finds a high-enough match, Slack is a **notification channel**, not a second database. + +```text +SQLite (or Grok rank result) + │ score ≥ 40 + ▼ + notify module (designed card) + │ + ▼ + Slack channel (configured via SLACK_CHANNEL_ID) + │ human clicks "Open notice" + ▼ + Tender portal page → human decides +``` + +**Setup:** bot token + channel — [`../scripts/slack_cli_setup.md`](../scripts/slack_cli_setup.md). + +--- + +## 7. Guarantees + +| Promise | Why it matters | +|---------|----------------| +| **Create-only store** | Re-running ingest does not wipe Status or Notes | +| **Dedupe by ID + link** | Same tender is not added twice | +| **Dry-run by default** | `run` without `--write` does not change the DB | +| **Sheets are views** | Spreadsheet is not the system of record | +| **Grok does not write Status** | AI ranks; humans still own triage | +| **Alerts are CTAs** | Slack/Teams open the notice; no auto-bid | + +--- + +## 8. Who owns what + +| Role | Owns | +|------|------| +| **This processor (code)** | Download, filter, score, store, rank, notify | +| **Engineering** | Keywords, objectives, notify thresholds, app code | +| **Operators** | Secrets, schedules, Status triage, bid decisions | +| **Capture lead** | Act on Slack/Sheets; pursue/pass | + +--- + +## 9. Related pages + +| Doc | Purpose | +|-----|---------| +| [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) | Keys, enable each output | +| [`AS_BUILT.md`](AS_BUILT.md) | Modules and pipeline detail | +| [`DATA_UPDATE_DIRECTIVES.md`](DATA_UPDATE_DIRECTIVES.md) | Hard rules for data | +| [`BACKLOG.md`](BACKLOG.md) §7 | **Future:** multi-geography source mesh, self-sustaining loop (lawful) | +| [`../README.md`](../README.md) | Quick start | +| [`../AGENTS.md`](../AGENTS.md) | Rules for AI coding agents | + +--- + +## 10. Future direction (not built) + +Long-term product intent is **not** “scrape the entire internet.” +It is a **lawful multi-source opportunity mesh** (any geography we enable), with human Status feeding **suggested** filter improvements — still create-only, still no auto-bid. + +Full register, architecture sketch, and decision principles: **[`BACKLOG.md`](BACKLOG.md) §7**. + +--- + +*This document is the operator overview for the ingest processor. Update it when triggers, thresholds, or outputs change.* diff --git a/docs/STATUS.md b/docs/STATUS.md index ed6f810..888734a 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -5,12 +5,12 @@ Living snapshot of implementation and ops readiness. | Field | Value | |-------|--------| -| **Last updated** | 2026-08-10 | -| **Phase** | Phase 1 — CanadaBuys open tender ingest | -| **Overall status** | **Implemented & manually validated (local); code on GitHub** | -| **GitHub** | https://github.com/vikiscoding/contractbiddingsystem (**private**) | +| **Last updated** | 2026-08-11 | +| **Phase** | Phase 1 ingest + optional Grok interpret-rank (post-ingest) | +| **Overall status** | **Ingest implemented & locally validated; Grok/Ranked coded + unit-tested; live Grok E2E pending** | +| **GitHub** | Private repository (see `git remote -v`) | | **Default branch** | `main` | -| **Recommended next** | Configure Actions secrets/vars → `workflow_dispatch` smoke → enable schedule | +| **Recommended next** | **Human plug-in:** [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md). Eng roadmap: [`BACKLOG.md`](BACKLOG.md). Long-horizon multi-geo vision: BACKLOG **§7**. | --- @@ -19,30 +19,50 @@ Living snapshot of implementation and ops readiness. | Capability | Status | Notes | |------------|--------|--------| | Download CanadaBuys open tender CSV | **Working** | Browser User-Agent required (403 without) | -| Bilingual parse + keyword filter + score | **Working** | `config/keywords.yaml` eng-owned | +| Bilingual parse + keyword filter + score | **Working** | `config/keywords.yaml` eng-owned; **single global pack** (no per-site packs yet) | | SQLite create-only store | **Working** | System of record (day-1) | | Dedupe OpportunityID + Link | **Working** | Unique indexes + pre-check | | CLI dry-run / `--write` / export-csv | **Working** | Only `--write` persists | -| Google Sheets full-replace sync | **Working (local)** | `sync-sheets`; service account + shared sheet | +| Google Sheets `Ingest` full-replace | **Working (local)** | `sync-sheets`; service account + shared sheet | +| Google Sheets `Ranked` full-replace | **Implemented** | From Grok report; same sheet/SA; **never** writes `Ingest` | +| Grok interpret-rank (post-ingest) | **Implemented (optional)** | `interpret-rank`; needs `XAI_API_KEY` + `.[ai]`; does **not** mutate SQLite | +| Re-push rankings without Grok | **Implemented** | `sync-rank-sheets` from latest `data/rankings/interpret-*.json` | | SharePoint Graph adapter | **Implemented, not activated** | Needs Entra + Sites.Selected grant | -| Teams Workflows notify | **Code ready** | Needs `TEAMS_WEBHOOK_URL` | -| GitHub Actions CI (`ci.yml`) | **In repo** | Runs on push/PR once remote exists | -| GitHub Actions daily ingest | **In repo, not yet scheduled live** | Needs remote + vars/secrets; enable after calibration | -| Multi-source / AI ranking | **Out of scope** | Phase 1 non-goals | +| Teams Workflows ops notify | **Code ready** | hard fail / partial / streak → `TEAMS_WEBHOOK_URL` | +| Teams high-match capture ping | **Implemented** | score ≥40; Adaptive Card + OpenUrl CTA; `config/notify.yaml` | +| Slack high-match capture ping | **Implemented** | ≥40; Slack CLI `SLACK_BOT_TOKEN` + channel via `chat.postMessage`; webhook legacy fallback | +| GitHub Actions CI (`ci.yml`) | **In repo** | ruff + pytest | +| GitHub Actions daily ingest | **In repo, not yet scheduled live** | Needs vars/secrets; enable after calibration | +| Source/website-type keyword packs | **Not built** | Tracked: [`BACKLOG.md`](BACKLOG.md) B-01, B-02 | +| AI auto-suggest keywords CLI | **Not built** | Tracked: B-03 | +| Multi-source / auto-bidding | **Out of scope** | Non-goals; see BACKLOG §3 | --- -## 2. Validated local runs (2026-08-10) +## 2. Validated local runs + +### 2.1 Ingest + Sheets Ingest (2026-08-10) | Step | Result | |------|--------| | `pip install -e .` / `.[sheets]` | OK | | `download-sample` | OK (~6.2 MB open tender CSV) after UA fix | | Dry-run on sample | parsed≈**893**, filtered≈**83**, mapped≈**27** (many map skips: empty Link) | -| `run --write` (capped) | SQLite rows created | +| `run --write` (capped) | SQLite rows created (~27 local) | | `export-csv` | OK → `data/export-opportunities.csv` (local, gitignored) | -| `sync-sheets` | OK → **27 rows** written to Google Sheet tab `Ingest` | -| Unit tests | **pytest** green (package + sheets unit tests) | +| `sync-sheets` | OK → rows written to Google Sheet tab **`Ingest`** | +| Unit tests (ingest era) | **pytest** green | + +### 2.2 Grok interpret-rank + Ranked tab (2026-08-11) + +| Step | Result | +|------|--------| +| Code: `interpret_rank.py`, CLI, `objectives.yaml` | **In tree** | +| Code: `sync_rankings_to_sheet` / `sync-rank-sheets` | **In tree** | +| Docs: DATA_UPDATE, AGENTS, README, playbook | **Updated** | +| Unit tests (mocked Grok, no live xAI) | **pytest green** (210 as of doc update) | +| Live `XAI_API_KEY` → real Grok ranking | **Not recorded as validated** (see BACKLOG B-07) | +| Live push to sheet tab **`Ranked`** | **Not recorded as validated** (B-07) | --- @@ -54,7 +74,10 @@ Living snapshot of implementation and ops readiness. | Root `.env` | Present (gitignored) | **Not** `scripts/.env` | | `STORAGE_BACKEND` | `sqlite` | Day-1 default | | `secrets/google-service-account.json` | Present (gitignored) | Exact `.json` name (not `.json.json`) | -| `GOOGLE_SHEET_ID` / tab | Configured | Sheet shared Editor with SA email | +| `GOOGLE_SHEET_ID` / `Ingest` tab | Configured | Sheet shared Editor with SA email | +| `GOOGLE_SHEET_RANK_TAB` | `Ranked` (default) | Created on first rank sync if missing | +| `XAI_API_KEY` | Optional | Required only for `interpret-rank` | +| `config/objectives.yaml` | Present | Eng-owned frame for Grok (template until customized) | | `TEAMS_WEBHOOK_URL` | Optional | Alerts silent if unset | | SharePoint secrets | Not required | Until SP activation | @@ -65,13 +88,15 @@ Living snapshot of implementation and ops readiness. | Store / view | Role | Live? | |--------------|------|-------| | SQLite `data/contract_opportunities.db` | **System of record** | Local yes | -| Google Sheets `Ingest` | Derived view (full replace) | Yes (user sheet) | +| Google Sheets `Ingest` | Derived opportunity view (full replace) | Yes (user sheet) | +| Google Sheets `Ranked` | Derived Grok ranking view (full replace) | Code ready; live E2E pending | | Google Sheets `Review` (if used) | Human notes | Operator-owned | +| `data/rankings/*` | Local Grok reports | Local only; gitignored | | SharePoint list | Alternate SoR | Not activated | | Teams | Failure / streak notify | Not configured | **Data rules:** see [`DATA_UPDATE_DIRECTIVES.md`](DATA_UPDATE_DIRECTIVES.md). -Create-only ingest; never silent-truncate Link; Sheets is not SoR. +Create-only ingest; never silent-truncate Link; Sheets is not SoR; Grok never rewrites SQLite Status/Notes/scores. --- @@ -81,19 +106,24 @@ Create-only ingest; never silent-truncate Link; Sheets is not SoR. 2. Windows **double extension** on service account file (`.json.json`). 3. Keyword hits with **empty notice URL** → map skip (by design). 4. `MAX_CREATE=0` means **unlimited**, not “block writes”. -5. Daily Actions workflow does **not** run Sheets sync by default (add step + secrets when ready). -6. `export-csv` / `sync-sheets` are **sqlite-oriented**. +5. Daily Actions workflow does **not** run Sheets sync or interpret-rank by default. +6. `export-csv` / `sync-sheets` / `interpret-rank` are **sqlite-oriented**. +7. Rule keywords are **global only** — false positives (e.g. construction “service desk”) still possible; type packs not built (BACKLOG B-01/B-02). +8. `objectives.yaml` still template company name until customized. +9. Without `XAI_API_KEY`, `interpret-rank` exits usage error (expected). --- ## 6. Go-live checklist (GitHub Actions) -- [x] Code on default branch (`main`) of GitHub remote (`vikiscoding/contractbiddingsystem`) +- [x] Code on default branch (`main`) of the GitHub remote - [ ] CI green on push (watch **Actions** tab after first push) - [ ] Repo variable `INGEST_MAX_CREATE=50` (or desired) - [ ] Secret `TEAMS_WEBHOOK_URL` (recommended) - [ ] Optional Sheets: secrets `GOOGLE_SHEET_ID`, `GOOGLE_SERVICE_ACCOUNT_JSON` + workflow step +- [ ] Optional Grok: secret `XAI_API_KEY` + workflow step (BACKLOG B-06) - [ ] Manual `workflow_dispatch` dry-run / capped write successful +- [ ] Live interpret-rank + Ranked tab smoke (BACKLOG B-07) - [ ] Enable schedule confidence after calibration week - [ ] Operators following [`scripts/ops_runbook.md`](../scripts/ops_runbook.md) @@ -103,10 +133,14 @@ Create-only ingest; never silent-truncate Link; Sheets is not SoR. | Need | Doc | |------|-----| +| **Human keys + plug-and-play + function list + roadmap** | [`PLUG_AND_PLAY.md`](PLUG_AND_PLAY.md) | +| **Processor role, triggers, outputs** | [`PROCESSOR_OVERVIEW.md`](PROCESSOR_OVERVIEW.md) | | LLM / agent context | [`../AGENTS.md`](../AGENTS.md) | -| Architecture | [`AS_BUILT.md`](AS_BUILT.md) | +| Architecture (what exists) | [`AS_BUILT.md`](AS_BUILT.md) | +| **Discussed / not built** | [`BACKLOG.md`](BACKLOG.md) | | Data MUST/MUST NOT | [`DATA_UPDATE_DIRECTIVES.md`](DATA_UPDATE_DIRECTIVES.md) | | How to change code | [`CHANGE_PLAYBOOK.md`](CHANGE_PLAYBOOK.md) | +| Decisions (ADR) | [`DECISIONS.md`](DECISIONS.md) | | Doc index | [`INDEX.md`](INDEX.md) | | Sheets setup | [`../scripts/google_sheets_setup.md`](../scripts/google_sheets_setup.md) | | Ops | [`../scripts/ops_runbook.md`](../scripts/ops_runbook.md) | @@ -118,9 +152,11 @@ Create-only ingest; never silent-truncate Link; Sheets is not SoR. When something material changes, edit §1–§6 and bump **Last updated**. Examples: - First successful Actions scheduled run +- Live Grok + Ranked tab validation (close B-07) - SharePoint activation -- Sheets wired into Actions +- Sheets / interpret-rank wired into Actions - Keyword/policy changes that affect typical filtered volume - New known issues +- Backlog item shipped → mark Done in BACKLOG + capability matrix -Keep STATUS factual and short; put deep rules in DATA_UPDATE_DIRECTIVES / AS_BUILT. +Keep STATUS factual and short; put deep rules in DATA_UPDATE_DIRECTIVES / AS_BUILT; put unbuilt designs in BACKLOG. diff --git a/pyproject.toml b/pyproject.toml index ae356e5..136c701 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,12 @@ sheets = [ "gspread>=6.0", "google-auth>=2.0", ] +ai = [ + "openai>=1.40", +] +slack = [ + "slack-sdk>=3.27", +] [project.scripts] opportunity-ingest = "opportunity_ingest.cli:main" diff --git a/scripts/daily_sync.ps1 b/scripts/daily_sync.ps1 index 127e817..d0f372a 100644 --- a/scripts/daily_sync.ps1 +++ b/scripts/daily_sync.ps1 @@ -5,7 +5,8 @@ # powershell.exe -NoProfile -ExecutionPolicy Bypass -File "...\scripts\daily_sync.ps1" $ErrorActionPreference = "Stop" -Set-Location "C:\Users\Vikrant\Documents\Agentic AI Learning\contractbiddingsystem" +$RepoRoot = Split-Path -Parent $PSScriptRoot +Set-Location $RepoRoot if (Test-Path .\.venv\Scripts\Activate.ps1) { & .\.venv\Scripts\Activate.ps1 diff --git a/scripts/google_sheets_setup.md b/scripts/google_sheets_setup.md index d71aa6a..8e49cfe 100644 --- a/scripts/google_sheets_setup.md +++ b/scripts/google_sheets_setup.md @@ -26,7 +26,7 @@ Use a separate tab (e.g. **Review**) for manual notes so they are not wiped. - Python package extras for Sheets: ```powershell -cd "C:\Users\Vikrant\Documents\Agentic AI Learning\contractbiddingsystem" +cd .\.venv\Scripts\Activate.ps1 pip install -e ".[sheets]" ``` @@ -96,7 +96,8 @@ You will share the Sheet with this address. 1. Open [Google Sheets](https://sheets.google.com) → **Blank** spreadsheet. 2. Rename the file, e.g. **Contract Opportunities**. 3. Rename the first tab to **`Ingest`** (or leave default and set `GOOGLE_SHEET_TAB` later). -4. Optional: add a second tab **`Review`** for human notes (job will not touch it if you only sync `Ingest`). +4. Optional: add a **`Ranked`** tab for Grok interpret-rank output (created automatically on first rank sync if missing). Do **not** put rankings on `Ingest`. +5. Optional: add a **`Review`** tab for human notes (pipeline will not touch it if you only sync `Ingest` / `Ranked`). ### B2. Share the sheet with the service account @@ -139,7 +140,9 @@ DATA_DIR=data GOOGLE_SHEET_ID=1abc...your_id...xyz GOOGLE_SHEET_TAB=Ingest +GOOGLE_SHEET_RANK_TAB=Ranked GOOGLE_SERVICE_ACCOUNT_FILE=secrets/google-service-account.json +# Optional Grok (separate extra): XAI_API_KEY=... ``` **Do not** commit `.env` or the JSON key. @@ -165,7 +168,7 @@ Or override flags: python -m opportunity_ingest sync-sheets --sheet-id "1abc..." --tab Ingest ``` -Expected success line: +Expected success line (Ingest): ```text Synced 10 rows to sheet 1abc... tab 'Ingest' @@ -174,29 +177,38 @@ Synced 10 rows to sheet 1abc... tab 'Ingest' Open the Google Sheet in the browser → **Ingest** tab should show headers + rows (Title, OpportunityID, Link, RelevanceScore, Status, …). +### D2. Grok Ranked tab (optional) + +Same spreadsheet and service account. Rankings never write to **Ingest**. + +```powershell +pip install -e ".[ai]" +# XAI_API_KEY in root .env +python -m opportunity_ingest interpret-rank --status New --limit 20 +# or re-push last local report without calling Grok: +python -m opportunity_ingest sync-rank-sheets +``` + +- Creates/replaces tab **`Ranked`** (or `GOOGLE_SHEET_RANK_TAB`). +- Local copies: `data/rankings/interpret-*.md`. +- Skip sheet push: `--no-sync-sheets`. +- Full human checklist (all features): [`docs/PLUG_AND_PLAY.md`](../docs/PLUG_AND_PLAY.md). +- Architecture: [`docs/AS_BUILT.md`](../docs/AS_BUILT.md). Unbuilt ideas: [`docs/BACKLOG.md`](../docs/BACKLOG.md). + --- ## Part E — Daily automatic update -### Option 1 — Windows Task Scheduler (your PC) - -1. Create `scripts\daily_sync.ps1`: +### Option 1 — Windows Task Scheduler (operator workstation) -```powershell -Set-Location "C:\Users\Vikrant\Documents\Agentic AI Learning\contractbiddingsystem" -& .\.venv\Scripts\Activate.ps1 -python -m opportunity_ingest run --write --max-create 50 -if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } -python -m opportunity_ingest sync-sheets -exit $LASTEXITCODE -``` +1. Use the checked-in script `scripts\daily_sync.ps1` (it changes to the repository root from its own location). 2. Task Scheduler → **Create Basic Task** → Daily after ~9:30 AM Eastern (CanadaBuys refreshes earlier; 10:00 AM local is a safe start). 3. Action: start program `powershell.exe` Arguments: - `-NoProfile -ExecutionPolicy Bypass -File "C:\Users\Vikrant\Documents\Agentic AI Learning\contractbiddingsystem\scripts\daily_sync.ps1"` + `-NoProfile -ExecutionPolicy Bypass -File "\scripts\daily_sync.ps1"` PC must be on at that time. diff --git a/scripts/ops_runbook.md b/scripts/ops_runbook.md index c684b92..b5f1ebd 100644 --- a/scripts/ops_runbook.md +++ b/scripts/ops_runbook.md @@ -4,8 +4,10 @@ Operator guide for day-1 SQLite storage, Status triage, calibration, create caps **Audience:** operators who review opportunities and manage the daily schedule. **Engineering owns:** `config/keywords.yaml`, pipeline code, GitHub workflow YAML. +**First-time plug-in (keys, Grok, Teams, function list, roadmap):** **[PLUG_AND_PLAY.md](../docs/PLUG_AND_PLAY.md)**. +**What this app is (processor role, triggers, flow):** **[PROCESSOR_OVERVIEW.md](../docs/PROCESSOR_OVERVIEW.md)**. **LLM / data rules:** [AGENTS.md](../AGENTS.md), [DATA_UPDATE_DIRECTIVES](../docs/DATA_UPDATE_DIRECTIVES.md), [AS_BUILT](../docs/AS_BUILT.md). -**Related:** [README.md](../README.md), [google_sheets_setup.md](google_sheets_setup.md), [provision_sharepoint_list.md](provision_sharepoint_list.md), [daily_sync.ps1](daily_sync.ps1). +**Related:** [README.md](../README.md), [google_sheets_setup.md](google_sheets_setup.md), [provision_sharepoint_list.md](provision_sharepoint_list.md), [daily_sync.ps1](daily_sync.ps1), [BACKLOG.md](../docs/BACKLOG.md). --- @@ -14,11 +16,17 @@ Operator guide for day-1 SQLite storage, Status triage, calibration, create caps ```text 1. run --write → SQLite system of record (create-only) 2. sync-sheets → optional full-replace of Google tab "Ingest" -3. export-csv → optional Excel snapshot -4. Human Status triage → SQLite / export / Sheet Review tab (not Ingest if auto-synced) +3. interpret-rank → optional Grok plain-English + fit rank + (writes data/rankings/*; full-replaces Sheet tab "Ranked" + when GOOGLE_SHEET_ID set — use --no-sync-sheets to skip) +4. export-csv → optional Excel snapshot +5. Human Status triage → SQLite / export / Sheet Review tab (not Ingest if auto-synced) + Use Ranked tab for capture prioritization; do not treat fit_score + as store RelevanceScore ``` -Windows automation sample: [`daily_sync.ps1`](daily_sync.ps1) (Task Scheduler). +Windows automation sample: [`daily_sync.ps1`](daily_sync.ps1) (Task Scheduler). +**Not built yet:** website-type keyword packs, auto keyword suggestions — see [`docs/BACKLOG.md`](../docs/BACKLOG.md). --- @@ -57,7 +65,25 @@ pip install -e ".[sheets]" python -m opportunity_ingest sync-sheets ``` -- Setup: [google_sheets_setup.md](google_sheets_setup.md) +- Setup: [google_sheets_setup.md](google_sheets_setup.md) +- Tab **`Ingest`**: opportunity rows from SQLite (full replace). +- Tab **`Ranked`**: Grok ranking (full replace via `interpret-rank` / `sync-rank-sheets`). +- Tab **`Review`**: human-only if you create it — pipeline does not write it. + +### Grok interpret-rank (optional) + +Requires `pip install -e ".[ai]"`, root `.env` `XAI_API_KEY`, and eng-owned `config/objectives.yaml`. + +```bash +python -m opportunity_ingest interpret-rank --status New --limit 20 +# Offline reports only: +python -m opportunity_ingest interpret-rank --limit 20 --no-sync-sheets +# Re-push last ranking JSON to Ranked tab (no Grok call): +python -m opportunity_ingest sync-rank-sheets +``` + +- Local files: `data/rankings/interpret-*.md` (and `.json`). +- **Does not** change SQLite Status, Notes, or RelevanceScore. - **Ingest** tab is **full-replaced** every sync — put manual notes on a separate **Review** tab - SQLite remains system of record; Sheets is a derived view - Normative rules: [DATA_UPDATE_DIRECTIVES.md](../docs/DATA_UPDATE_DIRECTIVES.md) §3 @@ -406,21 +432,38 @@ python -m opportunity_ingest run [--write | --dry-run] [--csv PATH] [--max-creat python -m opportunity_ingest download-sample [--out PATH] python -m opportunity_ingest check-store python -m opportunity_ingest export-csv [--out PATH] +python -m opportunity_ingest sync-sheets [--sheet-id ID] [--tab NAME] +python -m opportunity_ingest interpret-rank [--status S] [--limit N] [--sync-sheets|--no-sync-sheets] +python -m opportunity_ingest sync-rank-sheets [--from-json PATH] [--rank-tab NAME] ``` -Only **`--write`** persists. Default (and `--dry-run`) never writes. `DRY_RUN` env never enables write alone. +Only **`--write`** on `run` persists opportunities. Default (and `--dry-run`) never writes store rows. `DRY_RUN` env never enables write alone. +`interpret-rank` / sheet commands never UPDATE opportunity rows. ## Quick reference — key env / vars | Name | Default | Notes | |------|---------|--------| | `STORAGE_BACKEND` | `sqlite` | `sharepoint` when activated | -| `DATA_DIR` | `data` | SQLite + exports | +| `DATA_DIR` | `data` | SQLite + exports + rankings | | `MAX_CREATE` / `INGEST_MAX_CREATE` | `50` | Attempt budget; `0` = unlimited | | `ZERO_NEW_STREAK_THRESHOLD` | `3` | Teams after N zero-new UTC days | | `PARTIAL_ERROR_EXIT_THRESHOLD` | `5` | Exit 1 + notify | -| `TEAMS_WEBHOOK_URL` | — | Strongly recommended day-1 | -| `KEYWORDS_PATH` | `config/keywords.yaml` | Eng-owned | +| `TEAMS_WEBHOOK_URL` | — | Ops alerts (fail/streak); also match fallback | +| `TEAMS_MATCH_WEBHOOK_URL` | — | Optional dedicated capture channel | +| `TEAMS_MATCH_SCORE_THRESHOLD` | `40` | Ping when RelevanceScore or Grok fit ≥ N (Teams + Slack) | +| `TEAMS_MATCH_NOTIFY_ENABLED` | `true` | Master switch for Teams match pings | +| `SLACK_BOT_TOKEN` | — | CLI/Bolt `xoxb-` bot token (preferred) | +| `SLACK_CHANNEL_ID` | — | Channel for `chat.postMessage` | +| `SLACK_APP_TOKEN` | — | Optional `xapp-` (Socket Mode only) | +| `SLACK_WEBHOOK_URL` | — | Legacy Incoming Webhook fallback | +| `SLACK_MATCH_NOTIFY_ENABLED` | `true` | Master switch for Slack match pings | +| `KEYWORDS_PATH` | `config/keywords.yaml` | Eng-owned ingest filter | +| `OBJECTIVES_PATH` | `config/objectives.yaml` | Eng-owned Grok frame | +| `XAI_API_KEY` | — | Required for interpret-rank | +| `GOOGLE_SHEET_ID` | — | Ingest + Ranked sheet | +| `GOOGLE_SHEET_TAB` | `Ingest` | Opportunity full-replace | +| `GOOGLE_SHEET_RANK_TAB` | `Ranked` | Grok full-replace | | `STATE_PATH` | `state/zero_new_streak.json` | Streak JSON | --- diff --git a/scripts/slack_cli_setup.md b/scripts/slack_cli_setup.md new file mode 100644 index 0000000..99f4f33 --- /dev/null +++ b/scripts/slack_cli_setup.md @@ -0,0 +1,189 @@ +# Slack setup via Slack CLI (match notifications) + +This project posts high-match opportunity alerts to Slack using the **same credential model as Slack CLI / Bolt apps**, not Incoming Webhooks (webhooks remain a legacy fallback only). + +**Docs:** [Slack CLI](https://docs.slack.dev/tools/slack-cli/) · [Authorize CLI](https://docs.slack.dev/tools/slack-cli/guides/authorizing-the-slack-cli) · [Bolt Python tokens](https://docs.slack.dev/tools/bolt-python/getting-started) + +--- + +## What you paste into this repo + +| Variable | Example | Required? | Role | +|----------|---------|-----------|------| +| `SLACK_BOT_TOKEN` | `xoxb-...` | **Yes** (preferred path) | Bot User OAuth token → `chat.postMessage` | +| `SLACK_CHANNEL_ID` | `C0123ABCD` or `#bid-alerts-test` | **Yes** (with bot token) | Where to post | +| `SLACK_APP_TOKEN` | `xapp-...` | Optional | Socket Mode / `slack run` only; **not** used for match posts | +| `SLACK_MATCH_NOTIFY_ENABLED` | `true` | Default true | Master switch | +| `TEAMS_MATCH_SCORE_THRESHOLD` | `40` | Shared with Teams | Min score 0–100 | +| `SLACK_WEBHOOK_URL` | `https://hooks.slack.com/...` | Legacy only | Fallback if bot token unset | + +Install SDK: + +```powershell +pip install -e ".[slack]" +``` + +--- + +## Path A — Slack CLI (recommended) + +### 1. Install & login + +Windows: follow [Install Slack CLI for Windows](https://docs.slack.dev/tools/slack-cli/guides/installing-the-slack-cli-for-windows). + +```powershell +slack version +slack login +# Complete /slackauthticket + challenge in Slack +slack auth list +``` + +Auth data for the **CLI itself** lives in `~/.slack/credentials.json` (do not commit). +**This Python app does not read that file** — you still paste **app** tokens into repo `.env` (below). + +### 2. Create or open a Bolt/CLI app + +Either: + +```powershell +# Optional: separate Bolt sample app to mint tokens quickly +slack create bid-alerts-app --template slack-samples/bolt-python-getting-started-app +cd bid-alerts-app +slack run +# Select create/install app when prompted +``` + +Or create an app from [api.slack.com/apps](https://api.slack.com/apps) / `slack app settings` for an existing CLI project. + +### 3. Scopes & install + +In app settings → **OAuth & Permissions** → Bot Token Scopes, ensure at least: + +- `chat:write` (required) +- `chat:write.public` (optional; post to public channels without join) + +**Reinstall** the app to the **test workspace** after adding scopes. + +Invite the bot to the target channel (if private, or if missing `chat:write.public`): + +```text +/invite @YourBotName +``` + +### 4. Copy tokens (CLI / Bolt style) + +| Token | Where in Slack app UI | +|-------|------------------------| +| **Bot User OAuth Token** `xoxb-...` | OAuth & Permissions → *Bot User OAuth Token* → env `SLACK_BOT_TOKEN` | +| **App-Level Token** `xapp-...` | Basic Information → App-Level Tokens (`connections:write`) → env `SLACK_APP_TOKEN` (only if you also run Bolt Socket Mode) | + +### 5. Channel ID + +In Slack (desktop): channel name → **View channel details** → bottom **Channel ID** `C...` +Or use `#channel-name` if the bot can resolve it. + +### 6. Paste into **repo-root** `.env` + +```env +# Slack CLI / Bolt config (preferred) +SLACK_BOT_TOKEN=xoxb-your-bot-token +SLACK_CHANNEL_ID=C0123456789 +# Optional (not used by match notify): +# SLACK_APP_TOKEN=xapp-your-app-token + +SLACK_MATCH_NOTIFY_ENABLED=true +TEAMS_MATCH_SCORE_THRESHOLD=40 + +# Quiet Teams while testing Slack: +TEAMS_MATCH_NOTIFY_ENABLED=false +``` + +### 7. Verify settings load + +```powershell +cd +.\.venv\Scripts\Activate.ps1 +pip install -e ".[slack]" + +python -c "from opportunity_ingest.config import get_settings; s=get_settings(); print('web_api', s.slack_web_api_configured()); print('token', bool(s.resolved_slack_bot_token())); print('channel', s.resolved_slack_channel_id()); print('threshold', s.teams_match_score_threshold)" +``` + +Expect: `web_api True`, token True, channel set. + +### 8. Smoke-test Web API (no full pipeline) + +```powershell +python -c @" +from opportunity_ingest.config import get_settings +from opportunity_ingest.notify import post_slack_chat_message +s = get_settings() +post_slack_chat_message( + bot_token=s.resolved_slack_bot_token(), + channel=s.resolved_slack_channel_id(), + text='contractbiddingsystem Slack CLI config OK', + blocks=None, +) +print('posted ok') +"@ +``` + +Message should appear in the configured channel. + +### 9. End-to-end match notify (lower threshold once) + +```env +TEAMS_MATCH_SCORE_THRESHOLD=1 +``` + +```powershell +python -m opportunity_ingest interpret-rank --status New --limit 5 --no-sync-sheets +``` + +Console should show `slack=True`. Then set threshold back to **40**. + +--- + +## Path B — Tokens without full `slack create` + +If you already have a test app: + +1. Install app to test workspace with `chat:write`. +2. Copy `xoxb-` bot token → `SLACK_BOT_TOKEN`. +3. Set `SLACK_CHANNEL_ID`. +4. Same verify steps §7–9. + +This is still the **CLI/Bolt env contract** (`SLACK_BOT_TOKEN` + channel), even if you did not use `slack create`. + +--- + +## Legacy Incoming Webhook (not preferred) + +If only a `hooks.slack.com` URL exists, set: + +```env +SLACK_WEBHOOK_URL=https://hooks.slack.com/services/... +``` + +The app logs a **warning** and uses the webhook. Migrate to bot token when possible. + +--- + +## Troubleshooting + +| Error / symptom | Fix | +|-----------------|-----| +| `slack-sdk not installed` | `pip install -e ".[slack]"` | +| `not_authed` / `invalid_auth` | Wrong/revoked bot token; reinstall app | +| `channel_not_found` | Bad `SLACK_CHANNEL_ID`; use `C...` ID | +| `not_in_channel` | `/invite @bot` into channel | +| `missing_scope` | Add `chat:write`, reinstall app | +| `web_api False` | Both token **and** channel required | +| No post, threshold | Lower `TEAMS_MATCH_SCORE_THRESHOLD` for test | + +--- + +## Security + +- Never commit `.env`, tokens, or `~/.slack/credentials.json`. +- Prefer a **test workspace** / [developer sandbox](https://docs.slack.dev/tools/developer-sandboxes). +- Rotate bot tokens if leaked. diff --git a/src/opportunity_ingest/cli.py b/src/opportunity_ingest/cli.py index 29bf584..d0a8c75 100644 --- a/src/opportunity_ingest/cli.py +++ b/src/opportunity_ingest/cli.py @@ -7,9 +7,15 @@ python -m opportunity_ingest check-store python -m opportunity_ingest export-csv [--out PATH] python -m opportunity_ingest sync-sheets [--sheet-id ID] [--tab NAME] + python -m opportunity_ingest interpret-rank [options] [--sync-sheets] + python -m opportunity_ingest sync-rank-sheets [--from-json PATH] [--tab NAME] Write gating: only ``--write`` persists. ``DRY_RUN`` env never enables write and does not disable ``--write``. Default (neither flag) is dry-run. + +``interpret-rank`` is post-ingest only: it never updates store Status/Notes/scores. +Grok rankings may full-replace a separate Google Sheet tab (default ``Ranked``), +never the opportunity ``Ingest`` tab. """ from __future__ import annotations @@ -21,9 +27,25 @@ from opportunity_ingest.config import Settings, get_settings from opportunity_ingest.download import DEFAULT_SAMPLE_PATH, DownloadError, download_to_path from opportunity_ingest.exit_codes import EXIT_FAILURE, EXIT_USAGE +from opportunity_ingest.interpret_rank import ( + DEFAULT_BATCH_SIZE, + InterpretRankError, + run_interpret_rank, +) from opportunity_ingest.logging_setup import setup_logging +from opportunity_ingest.notify import ( + dispatch_match_notifications, + match_items_from_ranked, +) from opportunity_ingest.pipeline import resolve_write_mode, run_pipeline -from opportunity_ingest.sheets_sync import SheetsSyncError, sync_sqlite_to_sheet +from opportunity_ingest.sheets_sync import ( + DEFAULT_RANK_TAB, + SheetsSyncError, + find_latest_rankings_json, + load_rankings_json, + sync_rankings_to_sheet, + sync_sqlite_to_sheet, +) from opportunity_ingest.storage.base import StoreError from opportunity_ingest.storage.factory import build_store from opportunity_ingest.storage.sqlite_store import SqliteOpportunityStore @@ -92,7 +114,10 @@ def cmd_run( f"errors={metrics.error_count} skipped_dup={metrics.skipped_duplicate_count} " f"skipped_max={metrics.skipped_max_create_count} " f"streak={metrics.consecutive_zero_new_days} " - f"exit={metrics.exit_code} notified={metrics.notified}" + f"exit={metrics.exit_code} notified={metrics.notified} " + f"match_notified={metrics.match_notified} " + f"slack_match_notified={metrics.slack_match_notified} " + f"match_count={metrics.match_notify_count}" ) if metrics.hard_fail and metrics.hard_fail_reason: print(f"error: {metrics.hard_fail_reason}", file=sys.stderr) @@ -242,6 +267,237 @@ def cmd_sync_sheets( return 0 +def _resolve_rank_sheet_sync( + args: argparse.Namespace, + settings: Settings, +) -> bool: + """Whether to push rankings to Google Sheets. + + ``--sync-sheets`` forces on; ``--no-sync-sheets`` forces off. + Default (neither flag): on when GOOGLE_SHEET_ID is configured. + """ + if getattr(args, "no_sync_sheets", False): + return False + if getattr(args, "sync_sheets", False): + return True + return bool((settings.google_sheet_id or "").strip()) + + +def _push_rankings_to_sheet( + *, + ranked_rows: list[dict[str, object]], + run_id: str, + model: str, + settings: Settings, + sheet_id: str | None, + rank_tab: str | None, +) -> tuple[int, str, str]: + """Sync rankings; return (count, spreadsheet_id, tab). Raises SheetsSyncError.""" + resolved_id = (sheet_id or settings.google_sheet_id or "").strip() + if not resolved_id: + raise SheetsSyncError("set --sheet-id or GOOGLE_SHEET_ID") + resolved_tab = ( + (rank_tab or settings.google_sheet_rank_tab or DEFAULT_RANK_TAB).strip() + or DEFAULT_RANK_TAB + ) + count = sync_rankings_to_sheet( + ranked_rows, + run_id=run_id, + model=model, + spreadsheet_id=resolved_id, + worksheet_title=resolved_tab, + service_account_file=settings.google_service_account_file, + service_account_json=settings.google_service_account_json, + ) + return count, resolved_id, resolved_tab + + +def cmd_interpret_rank( + args: argparse.Namespace, + settings: Settings | None = None, +) -> int: + """Grok rephrase + fit-rank stored opportunities (report only; no store mutate).""" + try: + settings = settings or get_settings() + except Exception as exc: + print(f"error: invalid settings: {exc}", file=sys.stderr) + return EXIT_USAGE + setup_logging(settings.log_level) + + backend = (settings.storage_backend or "sqlite").strip().lower() + if backend != "sqlite": + print( + "error: interpret-rank requires STORAGE_BACKEND=sqlite " + f"(current={backend!r})", + file=sys.stderr, + ) + return EXIT_FAILURE + + api_key = (settings.xai_api_key or "").strip() + if not api_key: + print( + "error: set XAI_API_KEY for interpret-rank " + '(and pip install -e ".[ai]")', + file=sys.stderr, + ) + return EXIT_USAGE + + objectives_path = Path(args.objectives) if args.objectives else settings.objectives_path + out_dir = ( + Path(args.out_dir) + if args.out_dir + else settings.data_dir / "rankings" + ) + limit = args.limit + if limit is not None and limit < 0: + return _usage_error("--limit must be >= 0") + min_score = args.min_score + if min_score is not None and min_score < 0: + return _usage_error("--min-score must be >= 0") + batch_size = args.batch_size if args.batch_size is not None else DEFAULT_BATCH_SIZE + if batch_size < 1: + return _usage_error("--batch-size must be >= 1") + + try: + store = build_store(settings) + except (ValueError, NotImplementedError) as exc: + print(f"error: store config: {exc}", file=sys.stderr) + return EXIT_FAILURE + if not isinstance(store, SqliteOpportunityStore): + print("error: interpret-rank requires SqliteOpportunityStore", file=sys.stderr) + return EXIT_FAILURE + + try: + store.health_check() + rows = store.list_rows() + except StoreError as exc: + print(f"error: store failed: {exc}", file=sys.stderr) + return EXIT_FAILURE + + status = (args.status or "").strip() or None + try: + result = run_interpret_rank( + rows, + objectives_path=objectives_path, + api_key=api_key, + base_url=settings.xai_base_url, + model=settings.xai_model, + out_dir=out_dir, + status=status, + limit=limit, + min_rule_score=min_score, + batch_size=batch_size, + ) + except InterpretRankError as exc: + print(f"error: interpret-rank failed: {exc}", file=sys.stderr) + return EXIT_FAILURE + + top = result.ranked[0] if result.ranked else None + print( + f"interpret-rank model={result.model} input={result.input_count} " + f"ranked={len(result.ranked)} run_id={result.run_id}" + ) + if top: + print( + f"top #{top.rank} fit={top.fit_score} rec={top.recommendation} " + f"id={top.opportunity_id} title={top.title[:80]}" + ) + if result.markdown_path: + print(f"Wrote {result.markdown_path}") + if result.json_path: + print(f"Wrote {result.json_path}") + + if _resolve_rank_sheet_sync(args, settings): + try: + count, sid, tab = _push_rankings_to_sheet( + ranked_rows=[r.as_dict() for r in result.ranked], + run_id=result.run_id, + model=result.model, + settings=settings, + sheet_id=getattr(args, "sheet_id", None), + rank_tab=getattr(args, "rank_tab", None), + ) + except SheetsSyncError as exc: + print(f"error: rank sheets sync failed: {exc}", file=sys.stderr) + return EXIT_FAILURE + print(f"Synced {count} ranking rows to sheet {sid} tab {tab!r}") + + # Teams + Slack capture pings for Grok fits at/above threshold (default 40). + skip_chat = bool(getattr(args, "no_teams", False)) + if ( + not skip_chat + and ( + settings.teams_match_notify_enabled or settings.slack_match_notify_enabled + ) + ): + thr = int(settings.teams_match_score_threshold) + match_items = match_items_from_ranked(result.ranked, threshold=thr) + dispatched = dispatch_match_notifications( + match_items, + threshold=thr, + source="interpret-rank", + run_url=settings.github_run_url, + sheets_url=settings.resolved_google_sheet_url(), + sheets_tab=settings.google_sheet_rank_tab or "Ranked", + max_items=int(settings.teams_match_max_items), + teams_webhook_url=settings.resolved_match_webhook_url(), + teams_enabled=bool(settings.teams_match_notify_enabled), + slack_bot_token=settings.resolved_slack_bot_token(), + slack_channel_id=settings.resolved_slack_channel_id(), + slack_webhook_url=settings.resolved_slack_match_webhook_url(), + slack_enabled=bool(settings.slack_match_notify_enabled), + ) + print( + f"match_notify teams={dispatched.teams_posted} " + f"slack={dispatched.slack_posted} " + f"above_threshold={dispatched.match_count} threshold={thr}" + ) + return 0 + + +def cmd_sync_rank_sheets( + args: argparse.Namespace, + settings: Settings | None = None, +) -> int: + """Push an existing interpret-rank JSON report to the Ranked Google Sheet tab.""" + try: + settings = settings or get_settings() + except Exception as exc: + print(f"error: invalid settings: {exc}", file=sys.stderr) + return EXIT_USAGE + setup_logging(settings.log_level) + + from_json = Path(args.from_json) if args.from_json else None + if from_json is None: + latest = find_latest_rankings_json(settings.data_dir / "rankings") + if latest is None: + print( + "error: no interpret-*.json under data/rankings; " + "run interpret-rank first or pass --from-json", + file=sys.stderr, + ) + return EXIT_USAGE + from_json = latest + print(f"Using latest rankings file {from_json}") + + try: + run_id, model, ranked = load_rankings_json(from_json) + count, sid, tab = _push_rankings_to_sheet( + ranked_rows=ranked, + run_id=run_id, + model=model, + settings=settings, + sheet_id=args.sheet_id, + rank_tab=args.rank_tab, + ) + except SheetsSyncError as exc: + print(f"error: rank sheets sync failed: {exc}", file=sys.stderr) + return EXIT_FAILURE + + print(f"Synced {count} ranking rows to sheet {sid} tab {tab!r}") + return 0 + + def build_parser() -> argparse.ArgumentParser: # Omit prog= so usage reflects the invoked name (module or console script). parser = argparse.ArgumentParser( @@ -336,6 +592,126 @@ def build_parser() -> argparse.ArgumentParser: help="Worksheet tab name to replace (default: Ingest / GOOGLE_SHEET_TAB)", ) + # interpret-rank — Grok rephrase + fit ranking (report only) + ir_p = sub.add_parser( + "interpret-rank", + help=( + "Grok-assisted plain-English rewrite + rank stored opportunities " + "against config/objectives.yaml (writes reports; does not mutate store)" + ), + ) + ir_p.add_argument( + "--objectives", + dest="objectives", + metavar="PATH", + default=None, + help="Company objectives YAML (default: OBJECTIVES_PATH / config/objectives.yaml)", + ) + ir_p.add_argument( + "--out-dir", + dest="out_dir", + metavar="DIR", + default=None, + help="Directory for JSON/Markdown reports (default: data/rankings)", + ) + ir_p.add_argument( + "--status", + dest="status", + metavar="STATUS", + default=None, + help="Only interpret rows with this Status (e.g. New)", + ) + ir_p.add_argument( + "--limit", + dest="limit", + type=int, + metavar="N", + default=None, + help="Max opportunities to send to Grok (after sort by rule score)", + ) + ir_p.add_argument( + "--min-score", + dest="min_score", + type=int, + metavar="N", + default=None, + help="Only rows with rule RelevanceScore >= N", + ) + ir_p.add_argument( + "--batch-size", + dest="batch_size", + type=int, + metavar="N", + default=None, + help=f"Opportunities per Grok call (default {DEFAULT_BATCH_SIZE})", + ) + ir_sync = ir_p.add_mutually_exclusive_group() + ir_sync.add_argument( + "--sync-sheets", + dest="sync_sheets", + action="store_true", + help=( + "Full-replace Google Sheet Ranked tab after ranking " + "(default when GOOGLE_SHEET_ID is set)" + ), + ) + ir_sync.add_argument( + "--no-sync-sheets", + dest="no_sync_sheets", + action="store_true", + help="Do not push rankings to Google Sheets even if GOOGLE_SHEET_ID is set", + ) + ir_p.add_argument( + "--sheet-id", + dest="sheet_id", + metavar="ID", + default=None, + help="Spreadsheet ID for rank sync (default: GOOGLE_SHEET_ID)", + ) + ir_p.add_argument( + "--rank-tab", + dest="rank_tab", + metavar="NAME", + default=None, + help="Rankings worksheet tab (default: Ranked / GOOGLE_SHEET_RANK_TAB)", + ) + ir_p.add_argument( + "--no-teams", + dest="no_teams", + action="store_true", + help="Skip Teams high-match capture ping after ranking", + ) + + # sync-rank-sheets — push existing ranking JSON without re-calling Grok + srs_p = sub.add_parser( + "sync-rank-sheets", + help=( + "Full-replace Google Sheet Ranked tab from an interpret-rank JSON " + "(latest under data/rankings or --from-json); does not call Grok" + ), + ) + srs_p.add_argument( + "--from-json", + dest="from_json", + metavar="PATH", + default=None, + help="Path to interpret-*.json (default: newest in data/rankings)", + ) + srs_p.add_argument( + "--sheet-id", + dest="sheet_id", + metavar="ID", + default=None, + help="Spreadsheet ID (default: GOOGLE_SHEET_ID env)", + ) + srs_p.add_argument( + "--rank-tab", + dest="rank_tab", + metavar="NAME", + default=None, + help=f"Worksheet tab to replace (default: {DEFAULT_RANK_TAB} / GOOGLE_SHEET_RANK_TAB)", + ) + return parser @@ -359,6 +735,10 @@ def main(argv: list[str] | None = None) -> int: return cmd_export_csv(args.out_path) if args.command == "sync-sheets": return cmd_sync_sheets(args.sheet_id, args.tab) + if args.command == "interpret-rank": + return cmd_interpret_rank(args) + if args.command == "sync-rank-sheets": + return cmd_sync_rank_sheets(args) raise AssertionError(f"unhandled command: {args.command!r}") diff --git a/src/opportunity_ingest/config.py b/src/opportunity_ingest/config.py index 1f06aca..1717f8d 100644 --- a/src/opportunity_ingest/config.py +++ b/src/opportunity_ingest/config.py @@ -29,8 +29,25 @@ class Settings(BaseSettings): description="Explicit SQLite path; default {DATA_DIR}/contract_opportunities.db", ) keywords_path: Path = Field(default=Path("config/keywords.yaml")) + objectives_path: Path = Field( + default=Path("config/objectives.yaml"), + description="Company objectives for interpret-rank (Grok)", + ) state_path: Path = Field(default=Path("state/zero_new_streak.json")) log_level: str = Field(default="INFO") + # Grok / xAI (optional; required only for interpret-rank) + xai_api_key: str | None = Field( + default=None, + description="xAI API key (XAI_API_KEY); never commit", + ) + xai_base_url: str = Field( + default="https://api.x.ai/v1", + description="OpenAI-compatible xAI base URL", + ) + xai_model: str = Field( + default="grok-4.5", + description="Grok model id for interpret-rank", + ) # Soft create-attempt budget (KD-17). 0 = unlimited; negatives rejected. max_create: int | None = Field( default=50, @@ -47,7 +64,63 @@ class Settings(BaseSettings): http_timeout_seconds: float = Field(default=120.0) zero_new_streak_threshold: int = Field(default=3) partial_error_exit_threshold: int = Field(default=5) - teams_webhook_url: str | None = Field(default=None) + teams_webhook_url: str | None = Field( + default=None, + description="Teams Workflows webhook for ops alerts (and match fallback)", + ) + teams_match_webhook_url: str | None = Field( + default=None, + description=( + "Optional dedicated webhook for high-match opportunity pings; " + "falls back to TEAMS_WEBHOOK_URL when unset" + ), + ) + teams_match_notify_enabled: bool = Field( + default=True, + description="Post Teams card when new matches meet score threshold", + ) + teams_match_score_threshold: int = Field( + default=40, + description="Min score 0–100 to ping Teams (RelevanceScore or Grok fit)", + ) + teams_match_max_items: int = Field( + default=8, + description="Max opportunities listed on one match Adaptive Card / Slack msg", + ) + # Slack CLI / Bolt credentials (preferred for match pings) + # https://docs.slack.dev/tools/slack-cli/ — paste tokens from CLI-created app + slack_bot_token: str | None = Field( + default=None, + description="Bot User OAuth Token xoxb-... (Slack CLI / Bolt SLACK_BOT_TOKEN)", + ) + slack_app_token: str | None = Field( + default=None, + description=( + "App-level token xapp-... (SLACK_APP_TOKEN; Socket Mode / slack run). " + "Not required for outbound chat.postMessage match alerts." + ), + ) + slack_channel_id: str | None = Field( + default=None, + description="Channel ID C... or #name for match posts (chat.postMessage)", + ) + # Legacy Incoming Webhooks (fallback if bot token unset) + slack_webhook_url: str | None = Field( + default=None, + description="Legacy Incoming Webhook URL (prefer SLACK_BOT_TOKEN)", + ) + slack_match_webhook_url: str | None = Field( + default=None, + description="Legacy dedicated match webhook; falls back to SLACK_WEBHOOK_URL", + ) + slack_match_notify_enabled: bool = Field( + default=True, + description="Post Slack message when new matches meet score threshold", + ) + notify_config_path: Path = Field( + default=Path("config/notify.yaml"), + description="Optional YAML for Teams/Slack match notify defaults", + ) github_run_url: str | None = Field( default=None, description="GitHub Actions run URL for Teams cards (GITHUB_RUN_URL)", @@ -65,9 +138,20 @@ class Settings(BaseSettings): default=None, description="Spreadsheet ID from the sheet URL", ) + google_sheet_url: str | None = Field( + default=None, + description=( + "Optional full spreadsheet share/deep link for Teams/Slack cards; " + "when unset, built from GOOGLE_SHEET_ID" + ), + ) google_sheet_tab: str = Field( default="Ingest", - description="Worksheet tab name fully replaced on each sync", + description="Worksheet tab name fully replaced on each opportunity sync", + ) + google_sheet_rank_tab: str = Field( + default="Ranked", + description="Worksheet tab full-replaced by Grok interpret-rank sync", ) google_service_account_file: Path | None = Field( default=None, @@ -93,13 +177,138 @@ def _max_create_non_negative(cls, v: int | None) -> int | None: raise ValueError("max_create must be >= 0 (0=unlimited)") return v + @field_validator("teams_match_score_threshold") + @classmethod + def _threshold_0_100(cls, v: int) -> int: + n = int(v) + if n < 0 or n > 100: + raise ValueError("teams_match_score_threshold must be 0–100") + return n + + @field_validator("teams_match_max_items") + @classmethod + def _max_items_positive(cls, v: int) -> int: + n = int(v) + if n < 1: + raise ValueError("teams_match_max_items must be >= 1") + return n + def resolved_sqlite_path(self) -> Path: """Return SQLITE_PATH or ``{DATA_DIR}/contract_opportunities.db``.""" if self.sqlite_path is not None: return self.sqlite_path return self.data_dir / "contract_opportunities.db" + def resolved_match_webhook_url(self) -> str | None: + """Teams webhook for high-match pings (dedicated URL or ops fallback).""" + for raw in (self.teams_match_webhook_url, self.teams_webhook_url): + if raw and str(raw).strip(): + return str(raw).strip() + return None + + def resolved_google_sheet_url(self) -> str | None: + """Browser URL for the opportunity spreadsheet (match-card CTA). + + Uses ``GOOGLE_SHEET_URL`` when set; otherwise + ``https://docs.google.com/spreadsheets/d/{GOOGLE_SHEET_ID}/edit``. + """ + if self.google_sheet_url and str(self.google_sheet_url).strip(): + u = str(self.google_sheet_url).strip() + if u.startswith(("http://", "https://")): + return u + sid = (self.google_sheet_id or "").strip() + if not sid: + return None + return f"https://docs.google.com/spreadsheets/d/{sid}/edit" + + def resolved_slack_match_webhook_url(self) -> str | None: + """Legacy Incoming Webhook for match pings (if bot token path unused).""" + for raw in (self.slack_match_webhook_url, self.slack_webhook_url): + if raw and str(raw).strip(): + return str(raw).strip() + return None + + def resolved_slack_bot_token(self) -> str | None: + """Bot token for Web API (Slack CLI / Bolt ``SLACK_BOT_TOKEN``).""" + if self.slack_bot_token and str(self.slack_bot_token).strip(): + return str(self.slack_bot_token).strip() + return None + + def resolved_slack_channel_id(self) -> str | None: + """Target channel for ``chat.postMessage``.""" + if self.slack_channel_id and str(self.slack_channel_id).strip(): + return str(self.slack_channel_id).strip() + return None + + def slack_web_api_configured(self) -> bool: + """True when Slack CLI-style bot token + channel are both set.""" + return bool(self.resolved_slack_bot_token() and self.resolved_slack_channel_id()) + + +def load_notify_yaml_overrides(path: Path | str | None = None) -> dict[str, object]: + """Load optional config/notify.yaml teams/slack sections (soft; empty if missing).""" + import yaml + + p = Path(path) if path is not None else Path("config/notify.yaml") + if not p.is_file(): + return {} + try: + raw = yaml.safe_load(p.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError): + return {} + if not isinstance(raw, dict): + return {} + out: dict[str, object] = {} + + # Shared threshold / max items (root or teams — teams keeps back-compat) + shared = raw.get("match") if isinstance(raw.get("match"), dict) else {} + teams = raw.get("teams") if isinstance(raw.get("teams"), dict) else {} + slack = raw.get("slack") if isinstance(raw.get("slack"), dict) else {} + + thr = shared.get("score_threshold", teams.get("match_score_threshold")) + if thr is not None: + try: + out["teams_match_score_threshold"] = int(thr) + except (TypeError, ValueError): + pass + max_items = shared.get("max_items_per_message", teams.get("max_items_per_message")) + if max_items is not None: + try: + out["teams_match_max_items"] = int(max_items) + except (TypeError, ValueError): + pass + + if "match_notify_enabled" in teams: + out["teams_match_notify_enabled"] = bool(teams["match_notify_enabled"]) + if "match_notify_enabled" in slack: + out["slack_match_notify_enabled"] = bool(slack["match_notify_enabled"]) + return out + def get_settings() -> Settings: - """Load settings from environment (and optional ``.env``).""" - return Settings() + """Load settings from environment / ``.env``, with soft notify.yaml overrides. + + Env vars always win for secrets and when ``TEAMS_MATCH_*`` / ``SLACK_*`` is set. + ``config/notify.yaml`` fills threshold/enabled/max items when those env + keys are unset. + """ + import os + + base = Settings() + yaml_over = load_notify_yaml_overrides(base.notify_config_path) + if not yaml_over: + return base + + updates: dict[str, object] = {} + env_map = { + "teams_match_notify_enabled": "TEAMS_MATCH_NOTIFY_ENABLED", + "teams_match_score_threshold": "TEAMS_MATCH_SCORE_THRESHOLD", + "teams_match_max_items": "TEAMS_MATCH_MAX_ITEMS", + "slack_match_notify_enabled": "SLACK_MATCH_NOTIFY_ENABLED", + } + for field_name, env_name in env_map.items(): + if env_name not in os.environ and field_name in yaml_over: + updates[field_name] = yaml_over[field_name] + if not updates: + return base + return base.model_copy(update=updates) diff --git a/src/opportunity_ingest/interpret_rank.py b/src/opportunity_ingest/interpret_rank.py new file mode 100644 index 0000000..99f5144 --- /dev/null +++ b/src/opportunity_ingest/interpret_rank.py @@ -0,0 +1,671 @@ +"""Grok-assisted interpretation and ranking of stored opportunities. + +Post-ingest only. Reads SQLite (or row dicts), never updates contract_opportunities +Status / Notes / RelevanceScore. Writes ranked JSON + Markdown reports under +``data/rankings/`` (or a caller-chosen path). + +Requires optional extra: ``pip install -e ".[ai]"`` and ``XAI_API_KEY``. +""" + +from __future__ import annotations + +import html +import json +import logging +import re +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Mapping, Protocol, Sequence + +import yaml + +UTC = timezone.utc +logger = logging.getLogger(__name__) + +DEFAULT_BATCH_SIZE = 12 +DEFAULT_DESCRIPTION_CHARS = 1200 +RANKINGS_SUBDIR = "rankings" + +SYSTEM_PROMPT = """\ +You are a capture analyst for a Canadian professional-services / IT firm. +You rank public tender opportunities against the firm's stated objectives. + +Rules: +1. Use ONLY the opportunity text and company objectives provided. Do not invent + buyers, budgets, or requirements not supported by the text. +2. Interpret the buyer's likely objective in plain language. +3. Rephrase the opportunity so a busy capture lead can scan it in 30 seconds. +4. Score fit_score 0–100 against the company's objectives (not keyword weight alone). +5. Be candid: false keyword hits and off-scope work get low scores. +6. Prefer technology / managed services / advisory that matches objectives; + deprioritize construction, facilities, food design, pure creative ads, etc. +7. Return valid JSON only, matching the schema in the user message. +""" + + +class InterpretRankError(Exception): + """User-facing failure for interpret-rank (config, API, parse).""" + + +class GrokClient(Protocol): + """Minimal protocol so tests can inject a fake client.""" + + def complete_json(self, *, system: str, user: str) -> dict[str, Any]: + """Return parsed JSON object from the model.""" + ... + + +@dataclass(frozen=True, slots=True) +class CompanyObjectives: + """Loaded from config/objectives.yaml.""" + + version: int + company: dict[str, Any] + objectives: list[dict[str, Any]] + guidance: dict[str, Any] + raw: dict[str, Any] = field(repr=False) + + def prompt_block(self) -> str: + """Compact YAML-ish block for the model prompt.""" + payload = { + "company": self.company, + "objectives": self.objectives, + "guidance": self.guidance, + } + return yaml.safe_dump(payload, sort_keys=False, allow_unicode=True) + + +@dataclass(frozen=True, slots=True) +class RankedOpportunity: + """One interpreted + ranked opportunity (report row).""" + + rank: int + opportunity_id: str + title: str + buyer: str | None + link: str + closing_date: str | None + keywords_matched: str | None + rule_relevance_score: int | None + fit_score: int + recommendation: str + plain_english: str + interpreted_objective: str + matched_objectives: list[str] + why_it_fits: str + risks_or_mismatches: str + next_action: str + + def as_dict(self) -> dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True, slots=True) +class InterpretRankResult: + """Full run output (in-memory + paths when written).""" + + run_id: str + model: str + ranked: list[RankedOpportunity] + input_count: int + json_path: Path | None = None + markdown_path: Path | None = None + objectives_path: Path | None = None + + def as_dict(self) -> dict[str, Any]: + return { + "run_id": self.run_id, + "model": self.model, + "input_count": self.input_count, + "ranked_count": len(self.ranked), + "objectives_path": str(self.objectives_path) if self.objectives_path else None, + "json_path": str(self.json_path) if self.json_path else None, + "markdown_path": str(self.markdown_path) if self.markdown_path else None, + "ranked": [r.as_dict() for r in self.ranked], + } + + +def load_objectives(path: Path | str) -> CompanyObjectives: + """Load and validate company objectives YAML.""" + p = Path(path) + if not p.is_file(): + raise InterpretRankError(f"objectives file not found: {p}") + try: + raw = yaml.safe_load(p.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError) as exc: + raise InterpretRankError(f"failed to load objectives from {p}: {exc}") from exc + if not isinstance(raw, dict): + raise InterpretRankError(f"objectives root must be a mapping: {p}") + company = raw.get("company") or {} + objectives = raw.get("objectives") or [] + guidance = raw.get("guidance") or {} + if not isinstance(company, dict): + raise InterpretRankError("objectives.company must be a mapping") + if not isinstance(objectives, list) or not objectives: + raise InterpretRankError("objectives.objectives must be a non-empty list") + if not isinstance(guidance, dict): + raise InterpretRankError("objectives.guidance must be a mapping") + version = int(raw.get("version") or 1) + return CompanyObjectives( + version=version, + company=company, + objectives=objectives, + guidance=guidance, + raw=raw, + ) + + +def _clean_text(value: object | None, max_chars: int | None = None) -> str: + if value is None: + return "" + text = html.unescape(str(value)).strip() + # Collapse whitespace / HTML-ish noise lightly. + text = re.sub(r"\s+", " ", text) + if max_chars is not None and len(text) > max_chars: + text = text[: max_chars - 1].rstrip() + "…" + return text + + +def prepare_opportunity_payload( + rows: Sequence[Mapping[str, Any]], + *, + description_chars: int = DEFAULT_DESCRIPTION_CHARS, +) -> list[dict[str, Any]]: + """Normalize store rows into compact model inputs.""" + out: list[dict[str, Any]] = [] + for row in rows: + oid = _clean_text(row.get("OpportunityID")) + title = _clean_text(row.get("Title")) + if not oid or not title: + continue + out.append( + { + "opportunity_id": oid, + "title": title, + "buyer": _clean_text(row.get("Buyer")) or None, + "category": _clean_text(row.get("Category")) or None, + "description": _clean_text(row.get("Description"), description_chars) or None, + "keywords_matched": _clean_text(row.get("KeywordsMatched")) or None, + "rule_relevance_score": _as_int(row.get("RelevanceScore")), + "status": _clean_text(row.get("Status")) or None, + "published_date": _clean_text(row.get("PublishedDate")) or None, + "closing_date": _clean_text(row.get("ClosingDate")) or None, + "link": _clean_text(row.get("Link")) or None, + } + ) + return out + + +def _as_int(value: object | None) -> int | None: + if value is None or value == "": + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def filter_rows( + rows: Sequence[Mapping[str, Any]], + *, + status: str | None = None, + limit: int | None = None, + min_rule_score: int | None = None, +) -> list[dict[str, Any]]: + """Filter/sort opportunity rows before AI ranking. + + Sort seed: RelevanceScore DESC, then OpportunityID for stability. + """ + items = [dict(r) for r in rows] + if status: + status_l = status.strip().lower() + items = [ + r + for r in items + if _clean_text(r.get("Status")).lower() == status_l + ] + if min_rule_score is not None: + items = [ + r + for r in items + if (_as_int(r.get("RelevanceScore")) or 0) >= min_rule_score + ] + + def sort_key(r: Mapping[str, Any]) -> tuple[int, str]: + score = _as_int(r.get("RelevanceScore")) or 0 + oid = _clean_text(r.get("OpportunityID")) + return (-score, oid) + + items.sort(key=sort_key) + if limit is not None and limit >= 0: + items = items[:limit] + return items + + +def build_user_prompt( + objectives: CompanyObjectives, + batch: Sequence[Mapping[str, Any]], +) -> str: + """User message asking for structured JSON ranking of one batch.""" + schema_hint = { + "rankings": [ + { + "opportunity_id": "string (must match input)", + "fit_score": "integer 0-100", + "recommendation": "pursue | watch | pass", + "plain_english": "2-4 sentence human rewrite", + "interpreted_objective": "what the buyer is trying to achieve", + "matched_objectives": ["objective id from company list"], + "why_it_fits": "1-3 sentences", + "risks_or_mismatches": "1-3 sentences", + "next_action": "concrete next step for capture lead", + } + ] + } + return ( + "## Company objectives (frame of reference)\n" + f"{objectives.prompt_block()}\n" + "## Opportunities to interpret and rank\n" + f"{json.dumps(list(batch), ensure_ascii=False, indent=2)}\n\n" + "## Required JSON schema\n" + f"{json.dumps(schema_hint, indent=2)}\n\n" + "Return a single JSON object with key `rankings` covering EVERY opportunity " + "in the batch (same opportunity_id values). Sort rankings by fit_score " + "descending within this batch." + ) + + +def _extract_json_object(text: str) -> dict[str, Any]: + """Parse JSON object from model text (tolerates fenced markdown).""" + raw = text.strip() + if not raw: + raise InterpretRankError("empty model response") + # Strip ```json ... ``` fences if present. + fence = re.search(r"```(?:json)?\s*([\s\S]*?)\s*```", raw, re.IGNORECASE) + if fence: + raw = fence.group(1).strip() + try: + data = json.loads(raw) + except json.JSONDecodeError: + # Last resort: first { ... last } + start = raw.find("{") + end = raw.rfind("}") + if start < 0 or end <= start: + raise InterpretRankError( + f"model response is not JSON (first 200 chars): {text[:200]!r}" + ) + try: + data = json.loads(raw[start : end + 1]) + except json.JSONDecodeError as exc: + raise InterpretRankError(f"failed to parse model JSON: {exc}") from exc + if not isinstance(data, dict): + raise InterpretRankError("model JSON root must be an object") + return data + + +class OpenAIGrokClient: + """xAI Grok client via OpenAI-compatible Chat Completions API.""" + + def __init__( + self, + *, + api_key: str, + base_url: str = "https://api.x.ai/v1", + model: str = "grok-4.5", + timeout: float = 120.0, + ) -> None: + try: + from openai import OpenAI + except ImportError as exc: + raise InterpretRankError( + 'openai package not installed; run: pip install -e ".[ai]"' + ) from exc + if not api_key or not api_key.strip(): + raise InterpretRankError("XAI_API_KEY is required for interpret-rank") + self.model = model + self._client = OpenAI(api_key=api_key.strip(), base_url=base_url, timeout=timeout) + + def complete_json(self, *, system: str, user: str) -> dict[str, Any]: + try: + resp = self._client.chat.completions.create( + model=self.model, + messages=[ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + temperature=0.2, + response_format={"type": "json_object"}, + ) + except Exception as exc: # noqa: BLE001 — surface any SDK/HTTP error + raise InterpretRankError(f"Grok API call failed: {exc}") from exc + try: + content = resp.choices[0].message.content or "" + except (AttributeError, IndexError, TypeError) as exc: + raise InterpretRankError(f"unexpected Grok response shape: {exc}") from exc + return _extract_json_object(content) + + +def _clamp_score(value: object) -> int: + try: + n = int(value) # type: ignore[arg-type] + except (TypeError, ValueError): + return 0 + return max(0, min(100, n)) + + +def _normalize_recommendation(value: object) -> str: + s = _clean_text(value).lower() + if s in {"pursue", "bid", "go", "relevant"}: + return "pursue" + if s in {"watch", "monitor", "maybe", "review"}: + return "watch" + if s in {"pass", "skip", "discard", "no"}: + return "pass" + return s or "watch" + + +def merge_model_rankings( + inputs: Sequence[Mapping[str, Any]], + model_payload: Mapping[str, Any], +) -> list[RankedOpportunity]: + """Join model rankings back to inputs; stable global rank by fit_score.""" + by_id = {str(item["opportunity_id"]): item for item in inputs} + raw_list = model_payload.get("rankings") + if not isinstance(raw_list, list): + raise InterpretRankError("model JSON missing rankings[] array") + + seen: set[str] = set() + partial: list[dict[str, Any]] = [] + for entry in raw_list: + if not isinstance(entry, dict): + continue + oid = _clean_text(entry.get("opportunity_id")) + if not oid or oid not in by_id or oid in seen: + continue + seen.add(oid) + src = by_id[oid] + matched = entry.get("matched_objectives") or [] + if isinstance(matched, str): + matched_list = [matched] + elif isinstance(matched, list): + matched_list = [str(x) for x in matched if x is not None] + else: + matched_list = [] + partial.append( + { + "opportunity_id": oid, + "title": src.get("title") or "", + "buyer": src.get("buyer"), + "link": src.get("link") or "", + "closing_date": src.get("closing_date"), + "keywords_matched": src.get("keywords_matched"), + "rule_relevance_score": src.get("rule_relevance_score"), + "fit_score": _clamp_score(entry.get("fit_score")), + "recommendation": _normalize_recommendation(entry.get("recommendation")), + "plain_english": _clean_text(entry.get("plain_english")) + or _clean_text(src.get("description"), 400) + or (src.get("title") or ""), + "interpreted_objective": _clean_text(entry.get("interpreted_objective")) + or "Not stated clearly in the notice text.", + "matched_objectives": matched_list, + "why_it_fits": _clean_text(entry.get("why_it_fits")) or "", + "risks_or_mismatches": _clean_text(entry.get("risks_or_mismatches")) or "", + "next_action": _clean_text(entry.get("next_action")) + or "Open the notice Link and review eligibility.", + } + ) + + # Any inputs the model skipped get a low default so the report is complete. + for oid, src in by_id.items(): + if oid in seen: + continue + logger.warning("model omitted opportunity_id=%s; applying default pass", oid) + partial.append( + { + "opportunity_id": oid, + "title": src.get("title") or "", + "buyer": src.get("buyer"), + "link": src.get("link") or "", + "closing_date": src.get("closing_date"), + "keywords_matched": src.get("keywords_matched"), + "rule_relevance_score": src.get("rule_relevance_score"), + "fit_score": 0, + "recommendation": "watch", + "plain_english": _clean_text(src.get("description"), 400) + or (src.get("title") or ""), + "interpreted_objective": "Model did not return an interpretation.", + "matched_objectives": [], + "why_it_fits": "", + "risks_or_mismatches": "Missing model output for this row.", + "next_action": "Re-run interpret-rank or review manually via Link.", + } + ) + + partial.sort( + key=lambda r: (-int(r["fit_score"]), str(r["opportunity_id"])), + ) + ranked: list[RankedOpportunity] = [] + for i, row in enumerate(partial, start=1): + ranked.append( + RankedOpportunity( + rank=i, + opportunity_id=str(row["opportunity_id"]), + title=str(row["title"]), + buyer=row.get("buyer"), # type: ignore[arg-type] + link=str(row.get("link") or ""), + closing_date=row.get("closing_date"), # type: ignore[arg-type] + keywords_matched=row.get("keywords_matched"), # type: ignore[arg-type] + rule_relevance_score=row.get("rule_relevance_score"), # type: ignore[arg-type] + fit_score=int(row["fit_score"]), + recommendation=str(row["recommendation"]), + plain_english=str(row["plain_english"]), + interpreted_objective=str(row["interpreted_objective"]), + matched_objectives=list(row["matched_objectives"]), + why_it_fits=str(row["why_it_fits"]), + risks_or_mismatches=str(row["risks_or_mismatches"]), + next_action=str(row["next_action"]), + ) + ) + return ranked + + +def _chunk(items: Sequence[Any], size: int) -> list[list[Any]]: + if size <= 0: + return [list(items)] + return [list(items[i : i + size]) for i in range(0, len(items), size)] + + +def rank_opportunities( + rows: Sequence[Mapping[str, Any]], + objectives: CompanyObjectives, + client: GrokClient, + *, + model_name: str = "grok-4.5", + batch_size: int = DEFAULT_BATCH_SIZE, + description_chars: int = DEFAULT_DESCRIPTION_CHARS, + run_id: str | None = None, +) -> InterpretRankResult: + """Call Grok in batches and return a globally ranked result (in-memory).""" + prepared = prepare_opportunity_payload(rows, description_chars=description_chars) + if not prepared: + rid = run_id or datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + return InterpretRankResult( + run_id=rid, + model=model_name, + ranked=[], + input_count=0, + ) + + all_partial: list[RankedOpportunity] = [] + for batch in _chunk(prepared, batch_size): + user = build_user_prompt(objectives, batch) + payload = client.complete_json(system=SYSTEM_PROMPT, user=user) + batch_ranked = merge_model_rankings(batch, payload) + all_partial.extend(batch_ranked) + + # Re-rank across batches. + all_partial.sort(key=lambda r: (-r.fit_score, r.opportunity_id)) + final: list[RankedOpportunity] = [] + for i, r in enumerate(all_partial, start=1): + final.append( + RankedOpportunity( + rank=i, + opportunity_id=r.opportunity_id, + title=r.title, + buyer=r.buyer, + link=r.link, + closing_date=r.closing_date, + keywords_matched=r.keywords_matched, + rule_relevance_score=r.rule_relevance_score, + fit_score=r.fit_score, + recommendation=r.recommendation, + plain_english=r.plain_english, + interpreted_objective=r.interpreted_objective, + matched_objectives=list(r.matched_objectives), + why_it_fits=r.why_it_fits, + risks_or_mismatches=r.risks_or_mismatches, + next_action=r.next_action, + ) + ) + + rid = run_id or datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + return InterpretRankResult( + run_id=rid, + model=model_name, + ranked=final, + input_count=len(prepared), + ) + + +def render_markdown(result: InterpretRankResult, objectives: CompanyObjectives) -> str: + """Human-readable ranked brief for capture leads.""" + company_name = _clean_text((objectives.company or {}).get("name")) or "Company" + lines: list[str] = [ + f"# Opportunity ranking brief — {result.run_id}", + "", + f"**Frame of reference:** {company_name} ", + f"**Model:** `{result.model}` ", + f"**Opportunities interpreted:** {result.input_count} ", + "", + "Scores are **Grok fit** against `config/objectives.yaml`, not the rule-based " + "`RelevanceScore` from ingest. Ingest rows are **not** modified.", + "", + "---", + "", + ] + if not result.ranked: + lines.append("_No opportunities to rank._") + lines.append("") + return "\n".join(lines) + + for r in result.ranked: + buyer = r.buyer or "Unknown buyer" + rule = r.rule_relevance_score if r.rule_relevance_score is not None else "—" + matched = ", ".join(r.matched_objectives) if r.matched_objectives else "—" + lines.extend( + [ + f"## #{r.rank} — {r.title}", + "", + f"- **Fit score:** {r.fit_score}/100 ", + f"- **Recommendation:** `{r.recommendation}` ", + f"- **Buyer:** {buyer} ", + f"- **OpportunityID:** `{r.opportunity_id}` ", + f"- **Closing:** {r.closing_date or '—'} ", + f"- **Keywords (ingest):** {r.keywords_matched or '—'} ", + f"- **Rule RelevanceScore:** {rule} ", + f"- **Matched objectives:** {matched} ", + f"- **Link:** {r.link or '—'} ", + "", + "### Plain English", + r.plain_english or "—", + "", + "### Interpreted buyer objective", + r.interpreted_objective or "—", + "", + "### Why it fits", + r.why_it_fits or "—", + "", + "### Risks / mismatches", + r.risks_or_mismatches or "—", + "", + "### Next action", + r.next_action or "—", + "", + "---", + "", + ] + ) + return "\n".join(lines) + + +def write_reports( + result: InterpretRankResult, + objectives: CompanyObjectives, + *, + out_dir: Path, + objectives_path: Path | None = None, +) -> InterpretRankResult: + """Write JSON + Markdown under out_dir; return result with paths set.""" + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + json_path = out_dir / f"interpret-{result.run_id}.json" + md_path = out_dir / f"interpret-{result.run_id}.md" + payload = result.as_dict() + payload["company"] = objectives.company + payload["objectives"] = objectives.objectives + json_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + md_path.write_text(render_markdown(result, objectives), encoding="utf-8") + return InterpretRankResult( + run_id=result.run_id, + model=result.model, + ranked=result.ranked, + input_count=result.input_count, + json_path=json_path, + markdown_path=md_path, + objectives_path=objectives_path, + ) + + +def run_interpret_rank( + rows: Sequence[Mapping[str, Any]], + *, + objectives_path: Path, + api_key: str, + base_url: str = "https://api.x.ai/v1", + model: str = "grok-4.5", + out_dir: Path, + status: str | None = None, + limit: int | None = None, + min_rule_score: int | None = None, + batch_size: int = DEFAULT_BATCH_SIZE, + client_factory: Callable[..., GrokClient] | None = None, +) -> InterpretRankResult: + """End-to-end: filter rows → Grok rank → write reports.""" + objectives = load_objectives(objectives_path) + filtered = filter_rows( + rows, + status=status, + limit=limit, + min_rule_score=min_rule_score, + ) + if client_factory is not None: + client = client_factory(api_key=api_key, base_url=base_url, model=model) + else: + client = OpenAIGrokClient(api_key=api_key, base_url=base_url, model=model) + result = rank_opportunities( + filtered, + objectives, + client, + model_name=model, + batch_size=batch_size, + ) + return write_reports( + result, + objectives, + out_dir=out_dir, + objectives_path=objectives_path, + ) diff --git a/src/opportunity_ingest/notify.py b/src/opportunity_ingest/notify.py index 2a97049..a291b5e 100644 --- a/src/opportunity_ingest/notify.py +++ b/src/opportunity_ingest/notify.py @@ -1,17 +1,30 @@ -"""Teams Workflows webhook notifications (Adaptive Card). +"""Channel notifications: Teams Workflows + Slack Incoming Webhooks. -Python owns primary notify for hard fail, high partial errors, and zero-new -streak. On successful Teams POST, sets GitHub Actions step output +Pipelines: + +1. **Ops alerts (Teams)** — hard fail, partial errors, zero-new streak + (``notify_ingest_alert`` / ``TEAMS_WEBHOOK_URL``). +2. **Match alerts** — new opportunities or Grok ranks at/above score threshold + (default **40/100**): + - Teams Adaptive Card: ``notify_match_alerts`` / ``TEAMS_MATCH_WEBHOOK_URL`` + - Slack (preferred): **Slack CLI / Bolt config** — ``SLACK_BOT_TOKEN`` (``xoxb-``) + + ``SLACK_CHANNEL_ID`` via Web API ``chat.postMessage`` (Block Kit). + Optional ``SLACK_APP_TOKEN`` (``xapp-``) is documented for CLI/Bolt apps but + not required for outbound posts. + - Slack (legacy fallback): Incoming Webhook URL if bot token unset. + - Multi-channel: ``dispatch_match_notifications`` + - Optional Google Sheets link (``GOOGLE_SHEET_ID`` / ``GOOGLE_SHEET_URL``) + as fact + OpenUrl / button on the card. + +Match cards include truncated summaries + open-notice CTAs (buttons / OpenUrl). + +On successful **Teams ops** notify POST, sets GitHub Actions step output ``notified=true`` so the workflow backup step can skip double-notify (``failure() && steps.ingest.outputs.notified != 'true'``). -**GITHUB_OUTPUT ownership (KD-18):** When ``GITHUB_OUTPUT`` is set, notify +**GITHUB_OUTPUT ownership (KD-18):** When ``GITHUB_OUTPUT`` is set, ops notify ownership is complete only if that file is written after a successful webhook -POST. Write is retried once; persistent failure logs at ERROR and returns -``False`` (does not claim ``notified=true``). In that edge case Actions may -still double-notify on job failure — operators should treat GITHUB_OUTPUT I/O -errors as critical. When ``GITHUB_OUTPUT`` is unset (local runs), POST success -alone is enough. +POST. Match alerts (Teams or Slack) do **not** set ``notified=true``. """ from __future__ import annotations @@ -19,17 +32,52 @@ import json import logging import os +import re import urllib.error import urllib.request +from dataclasses import dataclass, field from typing import Any, Mapping, Sequence logger = logging.getLogger(__name__) DEFAULT_TIMEOUT_SECONDS = 30.0 +DEFAULT_MATCH_THRESHOLD = 40 +DEFAULT_MATCH_MAX_ITEMS = 8 +DEFAULT_TITLE_MAX = 120 +DEFAULT_SUMMARY_MAX = 280 +# Slack allows many blocks but keep action buttons modest. +SLACK_MAX_ACTION_BUTTONS = 5 class NotifyError(Exception): - """Teams webhook or GITHUB_OUTPUT handoff failure (non-fatal to exit matrix).""" + """Webhook or GITHUB_OUTPUT handoff failure (non-fatal to exit matrix).""" + + +@dataclass(frozen=True, slots=True) +class MatchAlertItem: + """One high-match opportunity for capture-channel pings (Teams and/or Slack).""" + + title: str + opportunity_id: str + link: str + score: int + score_kind: str # e.g. RelevanceScore | GrokFit + buyer: str | None = None + keywords: str | None = None + summary: str | None = None + recommendation: str | None = None + closing_date: str | None = None + + +@dataclass(slots=True) +class MatchNotifyResult: + """Outcome of multi-channel match dispatch.""" + + match_count: int = 0 + teams_posted: bool = False + slack_posted: bool = False + any_posted: bool = False + details: dict[str, str] = field(default_factory=dict) def set_github_output_notified(notified: bool = True) -> bool: @@ -67,11 +115,24 @@ def set_github_output_notified(notified: bool = True) -> bool: ) +def _truncate(text: str | None, max_chars: int) -> str: + if not text: + return "" + s = re.sub(r"\s+", " ", str(text)).strip() + if len(s) <= max_chars: + return s + if max_chars <= 1: + return s[:max_chars] + return s[: max_chars - 1].rstrip() + "…" + + def build_adaptive_card_payload( *, title: str, facts: Sequence[Mapping[str, str]] | None = None, body_text: str | None = None, + actions: Sequence[Mapping[str, Any]] | None = None, + extra_body: Sequence[Mapping[str, Any]] | None = None, ) -> dict[str, Any]: """Build a Teams Workflows message with an Adaptive Card attachment.""" body: list[dict[str, Any]] = [ @@ -85,6 +146,8 @@ def build_adaptive_card_payload( ] if body_text: body.append({"type": "TextBlock", "text": body_text, "wrap": True}) + if extra_body: + body.extend(dict(b) for b in extra_body) if facts: body.append( { @@ -95,23 +158,179 @@ def build_adaptive_card_payload( ], } ) + card: dict[str, Any] = { + "$schema": "http://adaptivecards.io/schemas/adaptive-card.json", + "type": "AdaptiveCard", + "version": "1.4", + "body": body, + } + if actions: + card["actions"] = [dict(a) for a in actions] return { "type": "message", "attachments": [ { "contentType": "application/vnd.microsoft.card.adaptive", "contentUrl": None, - "content": { - "$schema": "http://adaptivecards.io/schemas/adaptive-card.json", - "type": "AdaptiveCard", - "version": "1.4", - "body": body, - }, + "content": card, } ], } +def filter_match_items( + items: Sequence[MatchAlertItem], + *, + threshold: int = DEFAULT_MATCH_THRESHOLD, +) -> list[MatchAlertItem]: + """Return items with score >= threshold, highest score first.""" + thr = max(0, min(100, int(threshold))) + filtered = [i for i in items if int(i.score) >= thr and (i.link or "").strip()] + filtered.sort(key=lambda i: (-int(i.score), i.opportunity_id)) + return filtered + + +def google_sheet_edit_url( + spreadsheet_id: str | None, + *, + override_url: str | None = None, +) -> str | None: + """Build a browser URL for the workbook (Teams/Slack CTA). + + Prefer ``override_url`` when set (full share/deep link). Otherwise + ``https://docs.google.com/spreadsheets/d/{id}/edit`` from the sheet id. + """ + if override_url and str(override_url).strip(): + u = str(override_url).strip() + if u.startswith(("http://", "https://")): + return u + sid = (spreadsheet_id or "").strip() + if not sid: + return None + return f"https://docs.google.com/spreadsheets/d/{sid}/edit" + + +def build_match_alert_payload( + items: Sequence[MatchAlertItem], + *, + threshold: int = DEFAULT_MATCH_THRESHOLD, + source: str = "ingest", + run_url: str | None = None, + sheets_url: str | None = None, + sheets_tab: str | None = None, + max_items: int = DEFAULT_MATCH_MAX_ITEMS, + title_max_chars: int = DEFAULT_TITLE_MAX, + summary_max_chars: int = DEFAULT_SUMMARY_MAX, + card_title: str | None = None, + cta_label: str = "Open notice", + sheets_cta_label: str = "Open Google Sheet", +) -> dict[str, Any]: + """Adaptive Card: high-match opportunities with summary + OpenUrl CTAs. + + ``source`` is ``ingest`` (rule score) or ``interpret-rank`` (Grok fit). + When ``sheets_url`` is set, the card includes a spreadsheet fact + action + so channel members can open the full grid (Ingest / Ranked). + """ + matched = filter_match_items(items, threshold=threshold) + if not matched: + raise NotifyError("no match items at or above threshold") + + cap = max(1, int(max_items)) + shown = matched[:cap] + hidden = len(matched) - len(shown) + + if card_title: + title = card_title + elif source == "interpret-rank": + title = f"Grok fit matches (≥{threshold})" + else: + title = f"New opportunity matches (≥{threshold})" + + intro = ( + f"**{len(matched)}** high-match opportunit" + f"{'y' if len(matched) == 1 else 'ies'} " + f"(threshold **{threshold}/100**, source `{source}`). " + "Review links and act — Status stays human-owned in the store." + ) + if hidden > 0: + intro += f" Showing top {len(shown)}; **{hidden}** more not listed." + sheet_link = (sheets_url or "").strip() + if sheet_link.startswith(("http://", "https://")): + tab_hint = f" (tab **{sheets_tab}**)" if (sheets_tab or "").strip() else "" + intro += f" Full list in Google Sheets{tab_hint}." + + extra_body: list[dict[str, Any]] = [] + actions: list[dict[str, Any]] = [] + for idx, item in enumerate(shown, start=1): + t = _truncate(item.title, title_max_chars) or item.opportunity_id + summary = _truncate(item.summary, summary_max_chars) + buyer = _truncate(item.buyer, 80) or "—" + kws = _truncate(item.keywords, 100) or "—" + rec = (item.recommendation or "").strip() + lines = [ + f"**#{idx} · {item.score}/100** ({item.score_kind})" + + (f" · `{rec}`" if rec else ""), + t, + f"Buyer: {buyer} · ID: `{item.opportunity_id}`", + f"Keywords: {kws}", + ] + if item.closing_date: + lines.append(f"Closing: {item.closing_date}") + if summary: + lines.append(summary) + extra_body.append( + { + "type": "TextBlock", + "text": "\n\n".join(lines), + "wrap": True, + "separator": True, + } + ) + link = (item.link or "").strip() + if link.startswith(("http://", "https://")): + # Teams shows a limited number of card actions; label uniquely. + label = f"{cta_label} #{idx}" + if len(label) > 40: + label = f"Open #{idx}" + actions.append( + { + "type": "Action.OpenUrl", + "title": label, + "url": link, + } + ) + + facts: list[dict[str, str]] = [ + {"title": "Threshold", "value": str(threshold)}, + {"title": "Matches", "value": str(len(matched))}, + {"title": "Source", "value": source}, + ] + if run_url: + facts.append({"title": "Run", "value": run_url}) + if sheet_link.startswith(("http://", "https://")): + facts.append({"title": "Google Sheet", "value": sheet_link}) + if (sheets_tab or "").strip(): + facts.append({"title": "Sheet tab", "value": str(sheets_tab).strip()}) + # Prefer sheet CTA first so it survives action truncation. + actions.insert( + 0, + { + "type": "Action.OpenUrl", + "title": _truncate(sheets_cta_label, 40) or "Open Google Sheet", + "url": sheet_link, + }, + ) + + # Adaptive Cards: too many actions can be dropped by Teams; keep first 6. + return build_adaptive_card_payload( + title=title, + body_text=intro, + extra_body=extra_body, + facts=facts, + actions=actions[:6] if actions else None, + ) + + def build_ingest_alert_payload( *, reason: str, @@ -138,37 +357,121 @@ def build_ingest_alert_payload( return build_adaptive_card_payload(title=title, facts=facts) -def post_teams_webhook( +def post_json_webhook( webhook_url: str, payload: Mapping[str, Any], *, timeout: float = DEFAULT_TIMEOUT_SECONDS, + channel_label: str = "webhook", ) -> None: - """POST JSON payload to Teams Workflows webhook URL. + """POST JSON payload to an Incoming Webhook / Workflows URL. Raises ``NotifyError`` on transport / non-2xx responses. """ if not webhook_url or not webhook_url.strip(): - raise NotifyError("TEAMS_WEBHOOK_URL is empty") + raise NotifyError(f"{channel_label} URL is empty") data = json.dumps(payload).encode("utf-8") req = urllib.request.Request( webhook_url.strip(), data=data, - headers={"Content-Type": "application/json"}, + headers={"Content-Type": "application/json; charset=utf-8"}, method="POST", ) try: with urllib.request.urlopen(req, timeout=timeout) as resp: status = getattr(resp, "status", None) or resp.getcode() if status is not None and int(status) >= 300: - raise NotifyError(f"Teams webhook returned HTTP {status}") + raise NotifyError(f"{channel_label} returned HTTP {status}") except urllib.error.HTTPError as exc: - raise NotifyError(f"Teams webhook HTTP error: {exc.code} {exc.reason}") from exc + raise NotifyError( + f"{channel_label} HTTP error: {exc.code} {exc.reason}" + ) from exc except urllib.error.URLError as exc: - raise NotifyError(f"Teams webhook request failed: {exc.reason}") from exc + raise NotifyError(f"{channel_label} request failed: {exc.reason}") from exc except TimeoutError as exc: - raise NotifyError("Teams webhook request timed out") from exc + raise NotifyError(f"{channel_label} request timed out") from exc + + +def post_teams_webhook( + webhook_url: str, + payload: Mapping[str, Any], + *, + timeout: float = DEFAULT_TIMEOUT_SECONDS, +) -> None: + """POST JSON payload to Teams Workflows webhook URL.""" + post_json_webhook( + webhook_url, payload, timeout=timeout, channel_label="Teams webhook" + ) + + +def post_slack_webhook( + webhook_url: str, + payload: Mapping[str, Any], + *, + timeout: float = DEFAULT_TIMEOUT_SECONDS, +) -> None: + """POST JSON to a legacy Slack Incoming Webhook URL.""" + post_json_webhook( + webhook_url, payload, timeout=timeout, channel_label="Slack webhook" + ) + + +def post_slack_chat_message( + *, + bot_token: str, + channel: str, + text: str, + blocks: Sequence[Mapping[str, Any]] | None = None, + timeout: float = DEFAULT_TIMEOUT_SECONDS, +) -> None: + """Post via Slack Web API ``chat.postMessage`` (Slack CLI / Bolt bot token). + + Requires ``slack-sdk`` (``pip install -e ".[slack]"``) and a bot token + ``xoxb-...`` with ``chat:write``, installed to the workspace. Channel is + an ID (``C...`` / ``G...``) or public channel name (``#alerts``). + + See: https://docs.slack.dev/tools/slack-cli/ and Bolt Python env vars + ``SLACK_BOT_TOKEN`` / ``SLACK_APP_TOKEN``. + """ + token = (bot_token or "").strip() + ch = (channel or "").strip() + if not token: + raise NotifyError("SLACK_BOT_TOKEN is empty") + if not token.startswith("xoxb-"): + logger.warning( + "SLACK_BOT_TOKEN does not start with xoxb-; expected Slack CLI / " + "Bolt bot user OAuth token" + ) + if not ch: + raise NotifyError("SLACK_CHANNEL_ID is empty") + + try: + from slack_sdk import WebClient + from slack_sdk.errors import SlackApiError + except ImportError as exc: + raise NotifyError( + 'slack-sdk not installed; run: pip install -e ".[slack]"' + ) from exc + + client = WebClient(token=token, timeout=timeout) + try: + resp = client.chat_postMessage( + channel=ch, + text=text, + blocks=list(blocks) if blocks else None, + ) + except SlackApiError as exc: + err = "" + if exc.response is not None: + err = str(exc.response.get("error") or exc.response) + raise NotifyError(f"Slack chat.postMessage failed: {err or exc}") from exc + except Exception as exc: # noqa: BLE001 — network / SDK + raise NotifyError(f"Slack Web API request failed: {exc}") from exc + + if not getattr(resp, "get", lambda _k, _d=None: None)("ok", True): + # slack_sdk returns SlackResponse mapping-like + raise NotifyError(f"Slack chat.postMessage not ok: {resp}") def notify_ingest_alert( @@ -224,3 +527,461 @@ def notify_ingest_alert( ) return False return True + + +def build_slack_match_payload( + items: Sequence[MatchAlertItem], + *, + threshold: int = DEFAULT_MATCH_THRESHOLD, + source: str = "ingest", + run_url: str | None = None, + sheets_url: str | None = None, + sheets_tab: str | None = None, + max_items: int = DEFAULT_MATCH_MAX_ITEMS, + title_max_chars: int = DEFAULT_TITLE_MAX, + summary_max_chars: int = DEFAULT_SUMMARY_MAX, + card_title: str | None = None, + cta_label: str = "Open notice", + sheets_cta_label: str = "Open Google Sheet", +) -> dict[str, Any]: + """Slack Block Kit payload for high-match opportunities (Incoming Webhook).""" + matched = filter_match_items(items, threshold=threshold) + if not matched: + raise NotifyError("no match items at or above threshold") + + cap = max(1, int(max_items)) + shown = matched[:cap] + hidden = len(matched) - len(shown) + + if card_title: + header = card_title + elif source == "interpret-rank": + header = f"Grok fit matches (≥{threshold})" + else: + header = f"New opportunity matches (≥{threshold})" + + # Slack header plain_text max ~150 chars + header = _truncate(header, 140) + + intro = ( + f"*{len(matched)}* high-match opportunit" + f"{'y' if len(matched) == 1 else 'ies'} " + f"(threshold *{threshold}/100*, source `{source}`). " + "Review links and act — store Status stays human-owned." + ) + if hidden > 0: + intro += f" Showing top {len(shown)}; *{hidden}* more not listed." + if run_url: + intro += f"\nRun: <{run_url}|workflow>" + sheet_link = (sheets_url or "").strip() + if sheet_link.startswith(("http://", "https://")): + tab_hint = f" (tab *{sheets_tab}*)" if (sheets_tab or "").strip() else "" + intro += f"\nFull list: <{sheet_link}|Google Sheet>{tab_hint}" + + blocks: list[dict[str, Any]] = [ + { + "type": "header", + "text": {"type": "plain_text", "text": header, "emoji": True}, + }, + {"type": "section", "text": {"type": "mrkdwn", "text": intro}}, + {"type": "divider"}, + ] + + for idx, item in enumerate(shown, start=1): + t = _truncate(item.title, title_max_chars) or item.opportunity_id + summary = _truncate(item.summary, summary_max_chars) + buyer = _truncate(item.buyer, 80) or "—" + kws = _truncate(item.keywords, 100) or "—" + rec = (item.recommendation or "").strip() + link = (item.link or "").strip() + title_line = f"*<{link}|{t}>*" if link.startswith("http") else f"*{t}*" + rec_bit = f" · `{rec}`" if rec else "" + body = ( + f"*#{idx} · {item.score}/100* ({item.score_kind}){rec_bit}\n" + f"{title_line}\n" + f"Buyer: {buyer} · ID: `{item.opportunity_id}`\n" + f"Keywords: {kws}" + ) + if item.closing_date: + body += f"\nClosing: {item.closing_date}" + if summary: + body += f"\n>{summary}" + blocks.append({"type": "section", "text": {"type": "mrkdwn", "text": body}}) + + # Action buttons (Slack limit: keep few). Sheet button first when present. + elements: list[dict[str, Any]] = [] + if sheet_link.startswith(("http://", "https://")): + elements.append( + { + "type": "button", + "text": { + "type": "plain_text", + "text": _truncate(sheets_cta_label, 75) or "Open Google Sheet", + "emoji": True, + }, + "url": sheet_link, + "action_id": "open_google_sheet", + } + ) + notice_budget = max(0, SLACK_MAX_ACTION_BUTTONS - len(elements)) + for idx, item in enumerate(shown[:notice_budget], start=1): + link = (item.link or "").strip() + if not link.startswith(("http://", "https://")): + continue + elements.append( + { + "type": "button", + "text": { + "type": "plain_text", + "text": _truncate(f"{cta_label} #{idx}", 75), + "emoji": True, + }, + "url": link, + "action_id": f"open_notice_{idx}", + } + ) + if elements: + blocks.append({"type": "actions", "elements": elements}) + + fallback = f"{header}: {len(matched)} match(es) ≥ {threshold}" + return {"text": fallback, "blocks": blocks} + + +def notify_match_alerts( + webhook_url: str | None, + items: Sequence[MatchAlertItem], + *, + threshold: int = DEFAULT_MATCH_THRESHOLD, + source: str = "ingest", + run_url: str | None = None, + sheets_url: str | None = None, + sheets_tab: str | None = None, + max_items: int = DEFAULT_MATCH_MAX_ITEMS, + enabled: bool = True, + timeout: float = DEFAULT_TIMEOUT_SECONDS, + title_max_chars: int = DEFAULT_TITLE_MAX, + summary_max_chars: int = DEFAULT_SUMMARY_MAX, + card_title: str | None = None, + cta_label: str = "Open notice", +) -> bool: + """Post a Teams capture-channel Adaptive Card for high-match opportunities. + + Returns True when a card was POSTed successfully. + Returns False when disabled, no webhook, no items above threshold, or POST fails. + Does **not** write GITHUB_OUTPUT (ops double-notify handoff is separate). + """ + if not enabled: + logger.info("Teams match notify disabled; skipping (source=%s)", source) + return False + if not webhook_url or not str(webhook_url).strip(): + logger.warning( + "Teams match notify skipped (source=%s): no TEAMS_MATCH_WEBHOOK_URL / " + "TEAMS_WEBHOOK_URL", + source, + ) + return False + + matched = filter_match_items(items, threshold=threshold) + if not matched: + logger.info( + "Teams match notify: no items with score>=%s (source=%s, candidates=%s)", + threshold, + source, + len(items), + ) + return False + + try: + payload = build_match_alert_payload( + matched, + threshold=threshold, + source=source, + run_url=run_url, + sheets_url=sheets_url, + sheets_tab=sheets_tab, + max_items=max_items, + title_max_chars=title_max_chars, + summary_max_chars=summary_max_chars, + card_title=card_title, + cta_label=cta_label, + ) + post_teams_webhook(str(webhook_url), payload, timeout=timeout) + except NotifyError as exc: + logger.error("Teams match notify failed (source=%s): %s", source, exc) + return False + + logger.info( + "Teams match notify sent (source=%s, matches=%s, threshold=%s, sheets=%s)", + source, + len(matched), + threshold, + bool(sheets_url), + ) + return True + + +def notify_slack_match_alerts( + items: Sequence[MatchAlertItem], + *, + bot_token: str | None = None, + channel_id: str | None = None, + webhook_url: str | None = None, + threshold: int = DEFAULT_MATCH_THRESHOLD, + source: str = "ingest", + run_url: str | None = None, + sheets_url: str | None = None, + sheets_tab: str | None = None, + max_items: int = DEFAULT_MATCH_MAX_ITEMS, + enabled: bool = True, + timeout: float = DEFAULT_TIMEOUT_SECONDS, + title_max_chars: int = DEFAULT_TITLE_MAX, + summary_max_chars: int = DEFAULT_SUMMARY_MAX, + card_title: str | None = None, + cta_label: str = "Open notice", +) -> bool: + """Post a Slack Block Kit message for high-match opportunities. + + **Preferred (Slack CLI / Bolt):** ``bot_token`` (``SLACK_BOT_TOKEN``) + + ``channel_id`` (``SLACK_CHANNEL_ID``) → Web API ``chat.postMessage``. + + **Legacy fallback:** Incoming Webhook ``webhook_url`` if bot token unset. + + Returns True on successful post. Does not set GITHUB_OUTPUT. + """ + if not enabled: + logger.info("Slack match notify disabled; skipping (source=%s)", source) + return False + + matched = filter_match_items(items, threshold=threshold) + if not matched: + logger.info( + "Slack match notify: no items with score>=%s (source=%s, candidates=%s)", + threshold, + source, + len(items), + ) + return False + + try: + payload = build_slack_match_payload( + matched, + threshold=threshold, + source=source, + run_url=run_url, + sheets_url=sheets_url, + sheets_tab=sheets_tab, + max_items=max_items, + title_max_chars=title_max_chars, + summary_max_chars=summary_max_chars, + card_title=card_title, + cta_label=cta_label, + ) + except NotifyError as exc: + logger.error("Slack match payload failed (source=%s): %s", source, exc) + return False + + token = (bot_token or "").strip() + channel = (channel_id or "").strip() + hook = (webhook_url or "").strip() + + try: + if token and channel: + post_slack_chat_message( + bot_token=token, + channel=channel, + text=str(payload.get("text") or "Opportunity matches"), + blocks=payload.get("blocks"), # type: ignore[arg-type] + timeout=timeout, + ) + mode = "web_api" + elif hook: + logger.warning( + "Slack match using legacy Incoming Webhook; prefer Slack CLI " + "SLACK_BOT_TOKEN + SLACK_CHANNEL_ID (see scripts/slack_cli_setup.md)" + ) + post_slack_webhook(hook, payload, timeout=timeout) + mode = "webhook" + else: + logger.warning( + "Slack match notify skipped (source=%s): set SLACK_BOT_TOKEN + " + "SLACK_CHANNEL_ID (Slack CLI/Bolt) or legacy SLACK_WEBHOOK_URL", + source, + ) + return False + except NotifyError as exc: + logger.error("Slack match notify failed (source=%s): %s", source, exc) + return False + + logger.info( + "Slack match notify sent (source=%s, mode=%s, matches=%s, threshold=%s, sheets=%s)", + source, + mode, + len(matched), + threshold, + bool(sheets_url), + ) + return True + + +def dispatch_match_notifications( + items: Sequence[MatchAlertItem], + *, + threshold: int = DEFAULT_MATCH_THRESHOLD, + source: str = "ingest", + run_url: str | None = None, + sheets_url: str | None = None, + sheets_tab: str | None = None, + max_items: int = DEFAULT_MATCH_MAX_ITEMS, + teams_webhook_url: str | None = None, + teams_enabled: bool = True, + slack_bot_token: str | None = None, + slack_channel_id: str | None = None, + slack_webhook_url: str | None = None, + slack_enabled: bool = True, + timeout: float = DEFAULT_TIMEOUT_SECONDS, +) -> MatchNotifyResult: + """Send high-match alerts to Teams and/or Slack (same score criteria). + + Slack prefers bot token + channel (CLI/Bolt); webhook is legacy fallback. + When ``sheets_url`` is set, cards include an Open Google Sheet CTA. + """ + matched = filter_match_items(items, threshold=threshold) + result = MatchNotifyResult(match_count=len(matched)) + if not matched: + result.details["reason"] = "no_items_above_threshold" + return result + + if teams_enabled: + result.teams_posted = notify_match_alerts( + teams_webhook_url, + matched, + threshold=threshold, + source=source, + run_url=run_url, + sheets_url=sheets_url, + sheets_tab=sheets_tab, + max_items=max_items, + enabled=True, + timeout=timeout, + ) + result.details["teams"] = "posted" if result.teams_posted else "failed_or_skipped" + else: + result.details["teams"] = "disabled" + + if slack_enabled: + result.slack_posted = notify_slack_match_alerts( + matched, + bot_token=slack_bot_token, + channel_id=slack_channel_id, + webhook_url=slack_webhook_url, + threshold=threshold, + source=source, + run_url=run_url, + sheets_url=sheets_url, + sheets_tab=sheets_tab, + max_items=max_items, + enabled=True, + timeout=timeout, + ) + result.details["slack"] = "posted" if result.slack_posted else "failed_or_skipped" + else: + result.details["slack"] = "disabled" + + result.any_posted = bool(result.teams_posted or result.slack_posted) + return result + + +def match_items_from_opportunity_fields( + fields_list: Sequence[Any], + *, + threshold: int = DEFAULT_MATCH_THRESHOLD, +) -> list[MatchAlertItem]: + """Build match items from ``OpportunityFields`` (or duck-typed) for ingest.""" + items: list[MatchAlertItem] = [] + for f in fields_list: + try: + score = int(getattr(f, "RelevanceScore", 0) or 0) + except (TypeError, ValueError): + score = 0 + if score < threshold: + continue + link = str(getattr(f, "Link", "") or "").strip() + if not link: + continue + desc = getattr(f, "Description", None) + items.append( + MatchAlertItem( + title=str(getattr(f, "Title", "") or ""), + opportunity_id=str(getattr(f, "OpportunityID", "") or ""), + link=link, + score=score, + score_kind="RelevanceScore", + buyer=getattr(f, "Buyer", None), + keywords=getattr(f, "KeywordsMatched", None), + summary=str(desc) if desc else None, + recommendation=None, + closing_date=( + str(getattr(f, "ClosingDate")) + if getattr(f, "ClosingDate", None) is not None + else None + ), + ) + ) + return items + + +def match_items_from_ranked( + ranked: Sequence[Any], + *, + threshold: int = DEFAULT_MATCH_THRESHOLD, +) -> list[MatchAlertItem]: + """Build match items from ``RankedOpportunity`` (or dicts) for interpret-rank.""" + items: list[MatchAlertItem] = [] + for r in ranked: + if isinstance(r, Mapping): + score = int(r.get("fit_score") or 0) + link = str(r.get("link") or "").strip() + if score < threshold or not link: + continue + items.append( + MatchAlertItem( + title=str(r.get("title") or ""), + opportunity_id=str(r.get("opportunity_id") or ""), + link=link, + score=score, + score_kind="GrokFit", + buyer=r.get("buyer"), # type: ignore[arg-type] + keywords=r.get("keywords_matched"), # type: ignore[arg-type] + summary=str(r.get("plain_english") or r.get("interpreted_objective") or ""), + recommendation=str(r.get("recommendation") or "") or None, + closing_date=str(r.get("closing_date") or "") or None, + ) + ) + continue + try: + score = int(getattr(r, "fit_score", 0) or 0) + except (TypeError, ValueError): + score = 0 + link = str(getattr(r, "link", "") or "").strip() + if score < threshold or not link: + continue + items.append( + MatchAlertItem( + title=str(getattr(r, "title", "") or ""), + opportunity_id=str(getattr(r, "opportunity_id", "") or ""), + link=link, + score=score, + score_kind="GrokFit", + buyer=getattr(r, "buyer", None), + keywords=getattr(r, "keywords_matched", None), + summary=str( + getattr(r, "plain_english", None) + or getattr(r, "interpreted_objective", None) + or "" + ) + or None, + recommendation=getattr(r, "recommendation", None), + closing_date=getattr(r, "closing_date", None), + ) + ) + return items diff --git a/src/opportunity_ingest/pipeline.py b/src/opportunity_ingest/pipeline.py index 3a9a014..2c393df 100644 --- a/src/opportunity_ingest/pipeline.py +++ b/src/opportunity_ingest/pipeline.py @@ -27,7 +27,11 @@ ) from opportunity_ingest.map_fields import MapError, map_to_opportunity_fields from opportunity_ingest.models import ExistingKeys, OpportunityFields -from opportunity_ingest.notify import notify_ingest_alert +from opportunity_ingest.notify import ( + dispatch_match_notifications, + match_items_from_opportunity_fields, + notify_ingest_alert, +) from opportunity_ingest.parse import ParseError, parse_csv_file, parse_csv_text from opportunity_ingest.score import compute_score from opportunity_ingest.state import ( @@ -77,6 +81,9 @@ class RunMetrics: exit_code: int = EXIT_OK notified: bool = False notify_reason: str | None = None + match_notified: bool = False + match_notify_count: int = 0 + slack_match_notified: bool = False csv_source: str | None = None extra: dict[str, Any] = field(default_factory=dict) @@ -201,14 +208,19 @@ def _process_candidates( keys: ExistingKeys, budget: AttemptBudget, write: bool, -) -> dict[str, int]: - """Dedupe + create loop. On dry-run, counts would-create without store writes.""" +) -> dict[str, Any]: + """Dedupe + create loop. On dry-run, counts would-create without store writes. + + Write path also returns ``created``: list of successfully created OpportunityFields + (for Teams high-match pings). + """ added = 0 errors = 0 skipped_dup = 0 skipped_max = 0 would_create = 0 attempts = 0 + created: list[OpportunityFields] = [] for fields in candidates: if is_duplicate(fields.OpportunityID, fields.Link, keys): @@ -264,6 +276,7 @@ def _process_candidates( else: register_created(keys, fields.OpportunityID, fields.Link) added += 1 + created.append(fields) logger.info( "Created opportunity %s (score=%s)", fields.OpportunityID, @@ -277,6 +290,7 @@ def _process_candidates( "skipped_max_create": skipped_max, "would_create": would_create, "attempts": attempts, + "created": created, } @@ -395,6 +409,7 @@ def run_pipeline( metrics.skipped_max_create_count = stats["skipped_max_create"] metrics.would_create_count = stats["would_create"] metrics.create_attempts = stats["attempts"] + created_fields: list[OpportunityFields] = list(stats.get("created") or []) # Streak: write runs only (UTC calendar-day semantics). streak_reached = False @@ -451,9 +466,38 @@ def run_pipeline( extra_facts=extra, ) + # High-match capture pings → Teams and/or Slack (new creates only). + if write and created_fields and ( + settings.teams_match_notify_enabled or settings.slack_match_notify_enabled + ): + thr = int(settings.teams_match_score_threshold) + match_items = match_items_from_opportunity_fields( + created_fields, threshold=thr + ) + metrics.match_notify_count = len(match_items) + if match_items: + dispatched = dispatch_match_notifications( + match_items, + threshold=thr, + source="ingest", + run_url=settings.github_run_url, + sheets_url=settings.resolved_google_sheet_url(), + sheets_tab=settings.google_sheet_tab or "Ingest", + max_items=int(settings.teams_match_max_items), + teams_webhook_url=settings.resolved_match_webhook_url(), + teams_enabled=bool(settings.teams_match_notify_enabled), + slack_bot_token=settings.resolved_slack_bot_token(), + slack_channel_id=settings.resolved_slack_channel_id(), + slack_webhook_url=settings.resolved_slack_match_webhook_url(), + slack_enabled=bool(settings.slack_match_notify_enabled), + ) + metrics.match_notified = dispatched.teams_posted + metrics.slack_match_notified = dispatched.slack_posted + logger.info( "Run complete: write=%s added=%s errors=%s would_create=%s " - "skipped_dup=%s skipped_max=%s exit=%s notified=%s", + "skipped_dup=%s skipped_max=%s exit=%s notified=%s " + "match_notified=%s slack_match_notified=%s match_count=%s", write, metrics.added_count, metrics.error_count, @@ -462,6 +506,9 @@ def run_pipeline( metrics.skipped_max_create_count, metrics.exit_code, metrics.notified, + metrics.match_notified, + metrics.slack_match_notified, + metrics.match_notify_count, ) except PipelineHardFail as exc: diff --git a/src/opportunity_ingest/sheets_sync.py b/src/opportunity_ingest/sheets_sync.py index aff8ac7..6e54763 100644 --- a/src/opportunity_ingest/sheets_sync.py +++ b/src/opportunity_ingest/sheets_sync.py @@ -1,6 +1,9 @@ -"""Sync SQLite opportunities to Google Sheets (service account + Sheets API). +"""Sync SQLite opportunities and Grok rankings to Google Sheets. + +Free path: full replace of worksheet tabs via service account + Sheets API. +- ``Ingest`` (default): opportunity rows from SQLite +- ``Ranked`` (default for interpret-rank): Grok fit-rank report (separate tab) -Free path: full replace of one worksheet tab (default ``Ingest``). Requires optional deps: ``pip install gspread google-auth`` (or ``pip install -e ".[sheets]"``). """ @@ -11,13 +14,36 @@ import logging import os from pathlib import Path -from typing import Any +from typing import Any, Mapping, Sequence from opportunity_ingest.storage.sqlite_store import SqliteOpportunityStore logger = logging.getLogger(__name__) DEFAULT_TAB = "Ingest" +DEFAULT_RANK_TAB = "Ranked" + +# Columns written to the Ranked tab (full replace each interpret-rank sync). +RANKING_EXPORT_COLUMNS: tuple[str, ...] = ( + "RunId", + "Model", + "Rank", + "FitScore", + "Recommendation", + "Title", + "OpportunityID", + "Buyer", + "ClosingDate", + "KeywordsMatched", + "RuleRelevanceScore", + "MatchedObjectives", + "PlainEnglish", + "InterpretedObjective", + "WhyItFits", + "RisksOrMismatches", + "NextAction", + "Link", +) class SheetsSyncError(Exception): @@ -72,18 +98,13 @@ def _import_gspread(): return gspread, Credentials -def sync_sqlite_to_sheet( - store: SqliteOpportunityStore, - *, +def _open_spreadsheet( spreadsheet_id: str, - worksheet_title: str = DEFAULT_TAB, + *, service_account_file: str | Path | None = None, service_account_json: str | None = None, -) -> int: - """Replace worksheet contents with all SQLite opportunity rows. - - Returns number of data rows written (excluding header). - """ +): + """Authorize and open a spreadsheet by id. Returns (gspread_module, spreadsheet).""" if not spreadsheet_id or not spreadsheet_id.strip(): raise SheetsSyncError("GOOGLE_SHEET_ID is required") @@ -103,24 +124,23 @@ def sync_sqlite_to_sheet( f"Failed to open spreadsheet {spreadsheet_id!r}. " f"Share the sheet with the service account email as Editor. Detail: {exc}" ) from exc + return gspread, spreadsheet - title = (worksheet_title or DEFAULT_TAB).strip() or DEFAULT_TAB + +def _get_or_create_worksheet(spreadsheet: Any, title: str, *, cols: int = 20) -> Any: + gspread, _ = _import_gspread() try: try: - worksheet = spreadsheet.worksheet(title) + return spreadsheet.worksheet(title) except gspread.WorksheetNotFound: - worksheet = spreadsheet.add_worksheet(title=title, rows=1000, cols=20) + worksheet = spreadsheet.add_worksheet(title=title, rows=1000, cols=cols) logger.info("Created worksheet tab %r", title) + return worksheet except Exception as exc: raise SheetsSyncError(f"Cannot open/create worksheet {title!r}: {exc}") from exc - store.health_check() - rows = store.list_rows() - headers = list(SqliteOpportunityStore.EXPORT_COLUMNS) - values: list[list[Any]] = [headers] - for row in rows: - values.append([_cell(row.get(col)) for col in headers]) +def _replace_worksheet_values(worksheet: Any, values: list[list[Any]]) -> None: try: worksheet.clear() # Batch update is one efficient API call for the full grid. @@ -128,19 +148,169 @@ def sync_sqlite_to_sheet( except Exception as exc: raise SheetsSyncError(f"Failed to write sheet values: {exc}") from exc + +def sync_sqlite_to_sheet( + store: SqliteOpportunityStore, + *, + spreadsheet_id: str, + worksheet_title: str = DEFAULT_TAB, + service_account_file: str | Path | None = None, + service_account_json: str | None = None, +) -> int: + """Replace worksheet contents with all SQLite opportunity rows. + + Returns number of data rows written (excluding header). + """ + gspread_mod, spreadsheet = _open_spreadsheet( + spreadsheet_id, + service_account_file=service_account_file, + service_account_json=service_account_json, + ) + del gspread_mod # only needed for WorksheetNotFound inside helper + + title = (worksheet_title or DEFAULT_TAB).strip() or DEFAULT_TAB + worksheet = _get_or_create_worksheet( + spreadsheet, title, cols=max(20, len(SqliteOpportunityStore.EXPORT_COLUMNS)) + ) + + store.health_check() + rows = store.list_rows() + headers = list(SqliteOpportunityStore.EXPORT_COLUMNS) + values: list[list[Any]] = [headers] + for row in rows: + values.append([_cell(row.get(col)) for col in headers]) + + _replace_worksheet_values(worksheet, values) + data_rows = len(rows) logger.info( - "Synced %s rows to sheet %s tab %r", + "Synced %s opportunity rows to sheet %s tab %r", + data_rows, + spreadsheet_id, + title, + ) + return data_rows + + +def rankings_to_sheet_values( + ranked_rows: Sequence[Mapping[str, Any]], + *, + run_id: str, + model: str, +) -> list[list[Any]]: + """Build header + data grid for the Ranked tab from ranked dicts.""" + headers = list(RANKING_EXPORT_COLUMNS) + values: list[list[Any]] = [headers] + for row in ranked_rows: + matched = row.get("matched_objectives") + if isinstance(matched, list): + matched_s = ", ".join(str(x) for x in matched) + else: + matched_s = matched or "" + values.append( + [ + _cell(run_id), + _cell(model), + _cell(row.get("rank")), + _cell(row.get("fit_score")), + _cell(row.get("recommendation")), + _cell(row.get("title")), + _cell(row.get("opportunity_id")), + _cell(row.get("buyer")), + _cell(row.get("closing_date")), + _cell(row.get("keywords_matched")), + _cell(row.get("rule_relevance_score")), + _cell(matched_s), + _cell(row.get("plain_english")), + _cell(row.get("interpreted_objective")), + _cell(row.get("why_it_fits")), + _cell(row.get("risks_or_mismatches")), + _cell(row.get("next_action")), + _cell(row.get("link")), + ] + ) + return values + + +def sync_rankings_to_sheet( + ranked_rows: Sequence[Mapping[str, Any]], + *, + run_id: str, + model: str, + spreadsheet_id: str, + worksheet_title: str = DEFAULT_RANK_TAB, + service_account_file: str | Path | None = None, + service_account_json: str | None = None, +) -> int: + """Full-replace the Ranked tab with Grok interpret-rank output. + + **Never** targets the opportunity ``Ingest`` tab (refused). + Returns number of data rows written (excluding header). + """ + title = (worksheet_title or DEFAULT_RANK_TAB).strip() or DEFAULT_RANK_TAB + if title.lower() == DEFAULT_TAB.lower(): + raise SheetsSyncError( + f"Refusing to write Grok rankings to the {DEFAULT_TAB!r} tab " + f"(opportunity view). Use {DEFAULT_RANK_TAB!r} or another tab name." + ) + + _gspread, spreadsheet = _open_spreadsheet( + spreadsheet_id, + service_account_file=service_account_file, + service_account_json=service_account_json, + ) + worksheet = _get_or_create_worksheet( + spreadsheet, title, cols=max(20, len(RANKING_EXPORT_COLUMNS)) + ) + values = rankings_to_sheet_values(ranked_rows, run_id=run_id, model=model) + _replace_worksheet_values(worksheet, values) + + data_rows = max(0, len(values) - 1) + logger.info( + "Synced %s ranking rows to sheet %s tab %r (run_id=%s)", data_rows, spreadsheet_id, title, + run_id, ) return data_rows +def load_rankings_json(path: str | Path) -> tuple[str, str, list[dict[str, Any]]]: + """Load interpret-rank JSON report → (run_id, model, ranked list).""" + p = Path(path) + if not p.is_file(): + raise SheetsSyncError(f"Rankings JSON not found: {p}") + try: + data = json.loads(p.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise SheetsSyncError(f"Cannot read rankings JSON {p}: {exc}") from exc + if not isinstance(data, dict): + raise SheetsSyncError("Rankings JSON root must be an object") + run_id = str(data.get("run_id") or p.stem) + model = str(data.get("model") or "") + ranked = data.get("ranked") + if not isinstance(ranked, list): + raise SheetsSyncError("Rankings JSON missing ranked[] array") + rows: list[dict[str, Any]] = [] + for item in ranked: + if isinstance(item, dict): + rows.append(item) + return run_id, model, rows + + +def find_latest_rankings_json(directory: str | Path) -> Path | None: + """Return newest ``interpret-*.json`` under directory, or None.""" + d = Path(directory) + if not d.is_dir(): + return None + files = sorted(d.glob("interpret-*.json"), key=lambda p: p.stat().st_mtime, reverse=True) + return files[0] if files else None + + def _cell(value: Any) -> str | int | float: if value is None: return "" - if isinstance(value, (int, float)): + if isinstance(value, (int, float)) and not isinstance(value, bool): return value return str(value) diff --git a/tests/test_filter.py b/tests/test_filter.py index 12eb383..e19b047 100644 --- a/tests/test_filter.py +++ b/tests/test_filter.py @@ -91,17 +91,20 @@ def test_load_shipped_keywords_yaml(): group_ids = {t.group for t in cfg.terms} assert group_ids == { "microsoft_cloud", - "managed_services", - "itsm_servicenow", + "operating_authority", "advisory_consulting", "automation_process", "ai_operations", + "knowledge_flow", } - # No bare noisy terms in shipped defaults + # No bare noisy terms in shipped defaults (Atlas-tuned) bare = {t.term.lower() for t in cfg.terms} assert "teams" not in bare assert "strategy" not in bare + assert "service desk" not in bare # classic MSP noise; not Atlas offer assert "microsoft teams" in bare + assert "operating model" in bare + assert "copilot" in bare @pytest.mark.parametrize( @@ -285,13 +288,15 @@ def test_suppress_nested_terms_helper(): ] -def test_shipped_config_managed_services_not_double_counted(): +def test_shipped_config_nested_operating_terms_not_double_counted(): + """Longer phrase wins when nested shorter is fully covered by span rules.""" cfg = load_keyword_config(KEYWORDS_PATH) - hit = _tender(title="managed services for IT") + hit = _tender(title="operating contracts and registries for governance") result = match_keywords(hit, cfg) - assert "managed services" in result.terms - assert "managed service" not in result.terms - assert result.weights["managed services"] == 16 + assert "operating contracts" in result.terms + # singular nested form should not also count when only plural phrase hit + assert result.matched is True + assert "operating_authority" in result.groups def test_filter_tenders_drops_non_matches(): diff --git a/tests/test_interpret_rank.py b/tests/test_interpret_rank.py new file mode 100644 index 0000000..7e54a4e --- /dev/null +++ b/tests/test_interpret_rank.py @@ -0,0 +1,252 @@ +"""Unit tests for Grok interpret-rank (mocked client; no live API).""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from opportunity_ingest.interpret_rank import ( + CompanyObjectives, + InterpretRankError, + _extract_json_object, + build_user_prompt, + filter_rows, + load_objectives, + merge_model_rankings, + prepare_opportunity_payload, + rank_opportunities, + render_markdown, + run_interpret_rank, + write_reports, +) + + +class FakeGrok: + """Deterministic client for tests.""" + + def __init__(self, *args, **kwargs) -> None: # noqa: ANN002, ANN003 + self.calls = 0 + + def complete_json(self, *, system: str, user: str) -> dict: + self.calls += 1 + # Pull opportunity ids from the JSON block in the user prompt. + start = user.find("## Opportunities to interpret") + assert start >= 0 + # Find first [ after that header for the opportunities array. + arr_start = user.find("[", start) + arr_end = user.find("\n## Required JSON schema", arr_start) + blob = user[arr_start:arr_end].strip() + opps = json.loads(blob) + rankings = [] + for i, o in enumerate(opps): + oid = o["opportunity_id"] + # Higher for known tech-ish ids in fixtures. + fit = 90 - i * 5 if "TECH" in oid else 40 - i + rankings.append( + { + "opportunity_id": oid, + "fit_score": fit, + "recommendation": "pursue" if fit >= 70 else "watch", + "plain_english": f"Plain summary for {oid}", + "interpreted_objective": f"Buyer wants outcome for {oid}", + "matched_objectives": ["microsoft_cloud"], + "why_it_fits": "Matches cloud skills.", + "risks_or_mismatches": "Limited description.", + "next_action": "Open the Link and confirm scope.", + } + ) + return {"rankings": rankings} + + +def _sample_objectives(tmp_path: Path) -> Path: + data = { + "version": 1, + "company": {"name": "Test Co", "summary": "Cloud services"}, + "objectives": [ + { + "id": "microsoft_cloud", + "label": "M365", + "priority": "critical", + "description": "Microsoft work", + } + ], + "guidance": {"prefer": ["IT"], "deprioritize": ["construction"]}, + } + path = tmp_path / "objectives.yaml" + path.write_text(yaml.safe_dump(data), encoding="utf-8") + return path + + +def test_load_objectives_ok(tmp_path: Path): + path = _sample_objectives(tmp_path) + obj = load_objectives(path) + assert obj.version == 1 + assert obj.company["name"] == "Test Co" + assert obj.objectives[0]["id"] == "microsoft_cloud" + + +def test_load_objectives_missing(tmp_path: Path): + with pytest.raises(InterpretRankError, match="not found"): + load_objectives(tmp_path / "nope.yaml") + + +def test_filter_rows_status_limit_and_score(): + rows = [ + {"OpportunityID": "A", "Status": "New", "RelevanceScore": 10}, + {"OpportunityID": "B", "Status": "New", "RelevanceScore": 50}, + {"OpportunityID": "C", "Status": "Discarded", "RelevanceScore": 99}, + ] + out = filter_rows(rows, status="New", min_rule_score=20, limit=1) + assert len(out) == 1 + assert out[0]["OpportunityID"] == "B" + + +def test_prepare_and_merge_roundtrip(): + rows = [ + { + "Title": "Cloud RFP", + "OpportunityID": "TECH-1", + "Buyer": "Agency", + "Description": "Need SharePoint & Power Apps", + "KeywordsMatched": "sharepoint", + "RelevanceScore": 40, + "Link": "https://example.com/a", + "ClosingDate": "2026-09-01T00:00:00Z", + } + ] + prepared = prepare_opportunity_payload(rows) + assert prepared[0]["description"] == "Need SharePoint & Power Apps" + model = { + "rankings": [ + { + "opportunity_id": "TECH-1", + "fit_score": 88, + "recommendation": "pursue", + "plain_english": "Buyer needs Power Platform help.", + "interpreted_objective": "Modernize collaboration tools.", + "matched_objectives": ["microsoft_cloud"], + "why_it_fits": "Direct stack match.", + "risks_or_mismatches": "None major.", + "next_action": "Download full package.", + } + ] + } + ranked = merge_model_rankings(prepared, model) + assert len(ranked) == 1 + assert ranked[0].rank == 1 + assert ranked[0].fit_score == 88 + assert ranked[0].recommendation == "pursue" + + +def test_merge_fills_missing_model_rows(): + prepared = [ + {"opportunity_id": "X", "title": "T", "buyer": None, "link": "u", + "closing_date": None, "keywords_matched": None, "rule_relevance_score": 1, + "description": "d"}, + ] + ranked = merge_model_rankings(prepared, {"rankings": []}) + assert ranked[0].fit_score == 0 + assert "Model did not return" in ranked[0].interpreted_objective + + +def test_extract_json_fenced(): + data = _extract_json_object('```json\n{"rankings": []}\n```') + assert data == {"rankings": []} + + +def test_rank_and_write_reports(tmp_path: Path): + path = _sample_objectives(tmp_path) + objectives = load_objectives(path) + rows = [ + { + "Title": "Tech deal", + "OpportunityID": "TECH-9", + "Buyer": "B", + "Description": "Azure migration", + "KeywordsMatched": "azure", + "RelevanceScore": 30, + "Link": "https://example.com/t", + "Status": "New", + }, + { + "Title": "Other", + "OpportunityID": "OTH-1", + "Buyer": "B", + "Description": "Something else", + "KeywordsMatched": "service desk", + "RelevanceScore": 10, + "Link": "https://example.com/o", + "Status": "New", + }, + ] + client = FakeGrok() + result = rank_opportunities(rows, objectives, client, model_name="fake", batch_size=10) + assert client.calls == 1 + assert result.input_count == 2 + assert result.ranked[0].opportunity_id == "TECH-9" + assert result.ranked[0].fit_score >= result.ranked[1].fit_score + + written = write_reports(result, objectives, out_dir=tmp_path / "rankings", objectives_path=path) + assert written.json_path is not None and written.json_path.is_file() + assert written.markdown_path is not None and written.markdown_path.is_file() + md = written.markdown_path.read_text(encoding="utf-8") + assert "Fit score" in md + assert "TECH-9" in md + payload = json.loads(written.json_path.read_text(encoding="utf-8")) + assert payload["ranked_count"] == 2 + + +def test_run_interpret_rank_with_factory(tmp_path: Path): + path = _sample_objectives(tmp_path) + rows = [ + { + "Title": "Tech deal", + "OpportunityID": "TECH-1", + "Buyer": "B", + "Description": "Power Platform", + "KeywordsMatched": "power platform", + "RelevanceScore": 50, + "Link": "https://example.com/a", + "Status": "New", + } + ] + result = run_interpret_rank( + rows, + objectives_path=path, + api_key="test-key", + model="fake-model", + out_dir=tmp_path / "out", + client_factory=FakeGrok, + ) + assert result.ranked[0].plain_english.startswith("Plain summary") + assert result.markdown_path is not None + + +def test_build_user_prompt_includes_objectives(): + obj = CompanyObjectives( + version=1, + company={"name": "Co"}, + objectives=[{"id": "x", "label": "X"}], + guidance={}, + raw={}, + ) + prompt = build_user_prompt( + obj, + [{"opportunity_id": "1", "title": "T"}], + ) + assert "Company objectives" in prompt + assert "opportunity_id" in prompt + + +def test_render_markdown_empty(): + from opportunity_ingest.interpret_rank import InterpretRankResult + + obj = CompanyObjectives(1, {"name": "Co"}, [{"id": "a"}], {}, {}) + md = render_markdown( + InterpretRankResult("rid", "m", [], 0), + obj, + ) + assert "No opportunities" in md diff --git a/tests/test_notify.py b/tests/test_notify.py index 519613c..7adea24 100644 --- a/tests/test_notify.py +++ b/tests/test_notify.py @@ -9,10 +9,19 @@ import pytest from opportunity_ingest.notify import ( + MatchAlertItem, NotifyError, build_adaptive_card_payload, build_ingest_alert_payload, + build_match_alert_payload, + build_slack_match_payload, + dispatch_match_notifications, + filter_match_items, + google_sheet_edit_url, + match_items_from_ranked, notify_ingest_alert, + notify_match_alerts, + notify_slack_match_alerts, post_teams_webhook, set_github_output_notified, ) @@ -163,3 +172,248 @@ def test_notify_returns_false_when_github_output_handoff_fails( ) assert post.called assert ok is False + + +def _sample_match(score: int = 55, oid: str = "ID-1") -> MatchAlertItem: + return MatchAlertItem( + title="Cloud RFP for Copilot enablement", + opportunity_id=oid, + link=f"https://example.com/notice/{oid}", + score=score, + score_kind="RelevanceScore", + buyer="PSPC", + keywords="copilot, power platform", + summary="Buyer wants Microsoft-native AI under human governance.", + recommendation="pursue", + closing_date="2026-09-01T00:00:00Z", + ) + + +def test_filter_match_items_threshold(): + items = [_sample_match(55), _sample_match(30, "LOW"), _sample_match(90, "HI")] + out = filter_match_items(items, threshold=40) + assert [i.opportunity_id for i in out] == ["HI", "ID-1"] + + +def test_google_sheet_edit_url(): + assert google_sheet_edit_url(None) is None + assert google_sheet_edit_url(" ") is None + assert ( + google_sheet_edit_url("abc123XYZ") + == "https://docs.google.com/spreadsheets/d/abc123XYZ/edit" + ) + assert ( + google_sheet_edit_url( + "abc", + override_url="https://docs.google.com/spreadsheets/d/abc/edit#gid=0", + ) + == "https://docs.google.com/spreadsheets/d/abc/edit#gid=0" + ) + + +def test_build_match_alert_payload_has_cta_and_truncation(): + items = [_sample_match(52, "WS-AI"), _sample_match(41, "WS-IAM")] + payload = build_match_alert_payload(items, threshold=40, source="ingest") + card = payload["attachments"][0]["content"] + assert card["type"] == "AdaptiveCard" + assert "actions" in card + assert card["actions"][0]["type"] == "Action.OpenUrl" + assert "https://example.com/notice/WS-AI" in card["actions"][0]["url"] + body_text = " ".join( + str(b.get("text", "")) for b in card["body"] if isinstance(b, dict) + ) + assert "52/100" in body_text or "52" in body_text + + +def test_build_match_alert_payload_includes_sheets_link(): + sheet = "https://docs.google.com/spreadsheets/d/SHEETID/edit" + payload = build_match_alert_payload( + [_sample_match(60, "TOP")], + threshold=40, + source="ingest", + sheets_url=sheet, + sheets_tab="Ingest", + ) + card = payload["attachments"][0]["content"] + assert card["actions"][0]["title"] == "Open Google Sheet" + assert card["actions"][0]["url"] == sheet + facts = next( + b["facts"] for b in card["body"] if isinstance(b, dict) and "facts" in b + ) + fact_map = {f["title"]: f["value"] for f in facts} + assert fact_map["Google Sheet"] == sheet + assert fact_map["Sheet tab"] == "Ingest" + body_text = " ".join( + str(b.get("text", "")) for b in card["body"] if isinstance(b, dict) + ) + assert "Google Sheets" in body_text + + +def test_notify_match_alerts_success(): + with patch("opportunity_ingest.notify.post_teams_webhook") as post: + ok = notify_match_alerts( + "https://example.com/hook", + [_sample_match(60)], + threshold=40, + source="ingest", + ) + assert ok is True + post.assert_called_once() + + +def test_notify_match_alerts_below_threshold_skips(): + with patch("opportunity_ingest.notify.post_teams_webhook") as post: + ok = notify_match_alerts( + "https://example.com/hook", + [_sample_match(10)], + threshold=40, + ) + assert ok is False + post.assert_not_called() + + +def test_notify_match_alerts_disabled(): + with patch("opportunity_ingest.notify.post_teams_webhook") as post: + ok = notify_match_alerts( + "https://example.com/hook", + [_sample_match(90)], + enabled=False, + ) + assert ok is False + post.assert_not_called() + + +def test_match_items_from_ranked_dicts(): + ranked = [ + { + "fit_score": 52, + "title": "AI ITQ", + "opportunity_id": "X", + "link": "https://example.com/x", + "plain_english": "Source list for AI", + "recommendation": "watch", + "buyer": "PSPC", + "keywords_matched": "artificial intelligence", + }, + { + "fit_score": 5, + "title": "Elevator", + "opportunity_id": "Y", + "link": "https://example.com/y", + }, + ] + items = match_items_from_ranked(ranked, threshold=40) + assert len(items) == 1 + assert items[0].score == 52 + assert items[0].score_kind == "GrokFit" + + +def test_build_slack_match_payload_blocks_and_buttons(): + items = [_sample_match(52, "WS-AI"), _sample_match(41, "WS-IAM")] + payload = build_slack_match_payload(items, threshold=40, source="ingest") + assert "blocks" in payload + assert payload["text"] + types = [b.get("type") for b in payload["blocks"]] + assert "header" in types + assert "section" in types + assert "actions" in types + actions = next(b for b in payload["blocks"] if b.get("type") == "actions") + assert actions["elements"][0]["type"] == "button" + assert "https://example.com/notice/WS-AI" in actions["elements"][0]["url"] + + +def test_build_slack_match_payload_includes_sheets_button(): + sheet = "https://docs.google.com/spreadsheets/d/SHEETID/edit" + payload = build_slack_match_payload( + [_sample_match(55)], + threshold=40, + sheets_url=sheet, + sheets_tab="Ranked", + ) + intro = payload["blocks"][1]["text"]["text"] + assert "Google Sheet" in intro + assert "SHEETID" in intro + actions = next(b for b in payload["blocks"] if b.get("type") == "actions") + assert actions["elements"][0]["url"] == sheet + assert "Sheet" in actions["elements"][0]["text"]["text"] + + +def test_notify_slack_match_alerts_web_api_success(): + with patch("opportunity_ingest.notify.post_slack_chat_message") as post: + ok = notify_slack_match_alerts( + [_sample_match(60)], + bot_token="xoxb-test-token", + channel_id="C123", + threshold=40, + source="ingest", + ) + assert ok is True + post.assert_called_once() + kwargs = post.call_args.kwargs + assert kwargs["bot_token"] == "xoxb-test-token" + assert kwargs["channel"] == "C123" + assert kwargs["blocks"] + + +def test_notify_slack_match_alerts_legacy_webhook(): + with patch("opportunity_ingest.notify.post_slack_webhook") as post: + ok = notify_slack_match_alerts( + [_sample_match(60)], + webhook_url="https://hooks.slack.com/services/T/B/x", + threshold=40, + ) + assert ok is True + post.assert_called_once() + + +def test_notify_slack_match_alerts_no_credentials(): + with ( + patch("opportunity_ingest.notify.post_slack_chat_message") as cpost, + patch("opportunity_ingest.notify.post_slack_webhook") as wpost, + ): + ok = notify_slack_match_alerts([_sample_match(90)], threshold=40) + assert ok is False + cpost.assert_not_called() + wpost.assert_not_called() + + +def test_dispatch_match_notifications_both_channels(): + items = [_sample_match(55)] + with ( + patch("opportunity_ingest.notify.post_teams_webhook") as tpost, + patch("opportunity_ingest.notify.post_slack_chat_message") as spost, + ): + result = dispatch_match_notifications( + items, + threshold=40, + teams_webhook_url="https://example.com/teams", + teams_enabled=True, + slack_bot_token="xoxb-test", + slack_channel_id="C99", + slack_enabled=True, + ) + assert result.match_count == 1 + assert result.teams_posted is True + assert result.slack_posted is True + assert result.any_posted is True + tpost.assert_called_once() + spost.assert_called_once() + + +def test_dispatch_slack_only_web_api(): + with ( + patch("opportunity_ingest.notify.post_teams_webhook") as tpost, + patch("opportunity_ingest.notify.post_slack_chat_message") as spost, + ): + result = dispatch_match_notifications( + [_sample_match(80)], + threshold=40, + teams_enabled=False, + slack_bot_token="xoxb-test", + slack_channel_id="#alerts", + slack_enabled=True, + ) + assert result.teams_posted is False + assert result.slack_posted is True + tpost.assert_not_called() + spost.assert_called_once() diff --git a/tests/test_sheets_sync.py b/tests/test_sheets_sync.py index 22e4b1c..9ed2705 100644 --- a/tests/test_sheets_sync.py +++ b/tests/test_sheets_sync.py @@ -8,8 +8,14 @@ import pytest from opportunity_ingest.sheets_sync import ( + DEFAULT_RANK_TAB, + RANKING_EXPORT_COLUMNS, SheetsSyncError, _load_service_account_info, + find_latest_rankings_json, + load_rankings_json, + rankings_to_sheet_values, + sync_rankings_to_sheet, ) @@ -37,3 +43,95 @@ def test_load_service_account_missing_raises() -> None: def test_load_service_account_bad_json() -> None: with pytest.raises(SheetsSyncError, match="not valid JSON"): _load_service_account_info(None, "{not-json") + + +def test_rankings_to_sheet_values_shape() -> None: + rows = [ + { + "rank": 1, + "fit_score": 88, + "recommendation": "pursue", + "title": "Cloud RFP", + "opportunity_id": "TECH-1", + "buyer": "Agency", + "closing_date": "2026-09-01", + "keywords_matched": "azure", + "rule_relevance_score": 40, + "matched_objectives": ["microsoft_cloud", "ai_operations"], + "plain_english": "Plain text", + "interpreted_objective": "Modernize", + "why_it_fits": "Stack match", + "risks_or_mismatches": "None", + "next_action": "Open link", + "link": "https://example.com/a", + } + ] + values = rankings_to_sheet_values(rows, run_id="run1", model="grok-4.5") + assert values[0] == list(RANKING_EXPORT_COLUMNS) + assert len(values) == 2 + assert values[1][0] == "run1" + assert values[1][1] == "grok-4.5" + assert values[1][2] == 1 + assert values[1][3] == 88 + assert values[1][11] == "microsoft_cloud, ai_operations" + assert values[1][-1] == "https://example.com/a" + + +def test_sync_rankings_refuses_ingest_tab() -> None: + with pytest.raises(SheetsSyncError, match="Refusing"): + sync_rankings_to_sheet( + [], + run_id="r", + model="m", + spreadsheet_id="sheet123", + worksheet_title="Ingest", + service_account_json=json.dumps({"type": "service_account"}), + ) + + +def test_load_rankings_json_and_latest(tmp_path: Path) -> None: + older = tmp_path / "interpret-20260101T000000Z.json" + newer = tmp_path / "interpret-20260811T120000Z.json" + payload = { + "run_id": "20260811T120000Z", + "model": "grok-4.5", + "ranked": [ + { + "rank": 1, + "opportunity_id": "X", + "title": "T", + "fit_score": 50, + "recommendation": "watch", + "matched_objectives": [], + "plain_english": "p", + "interpreted_objective": "i", + "why_it_fits": "w", + "risks_or_mismatches": "r", + "next_action": "n", + "link": "https://example.com", + } + ], + } + older.write_text(json.dumps({"run_id": "old", "model": "m", "ranked": []}), encoding="utf-8") + newer.write_text(json.dumps(payload), encoding="utf-8") + # Ensure mtime ordering + import os + import time + + os.utime(older, (time.time() - 100, time.time() - 100)) + os.utime(newer, (time.time(), time.time())) + + run_id, model, ranked = load_rankings_json(newer) + assert run_id == "20260811T120000Z" + assert model == "grok-4.5" + assert ranked[0]["opportunity_id"] == "X" + + latest = find_latest_rankings_json(tmp_path) + assert latest == newer + assert find_latest_rankings_json(tmp_path / "missing") is None + assert DEFAULT_RANK_TAB == "Ranked" + + +def test_load_rankings_json_missing(tmp_path: Path) -> None: + with pytest.raises(SheetsSyncError, match="not found"): + load_rankings_json(tmp_path / "nope.json") diff --git a/tests/test_smoke.py b/tests/test_smoke.py index 2695226..e297582 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -32,6 +32,8 @@ def test_cli_parser_has_frozen_subcommands(): "check-store", "export-csv", "sync-sheets", + "interpret-rank", + "sync-rank-sheets", }