From 2a99fcdfbd791f44820dd0d572164f235d29d2e7 Mon Sep 17 00:00:00 2001 From: DoRmAmMu1997 Date: Mon, 10 Aug 2026 19:22:23 +0530 Subject: [PATCH 1/5] chore(repo): add architecture docs and consolidate tests under Tests/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two structural changes, no runtime behaviour touched. docs/ — the folder held only Superpowers session artefacts. Those are now gitignored and untracked (kept on disk), matching the existing .superpowers/ rule, and docs/ carries a committed architecture set instead: docs/README.md index and reading order docs/hld/ one high-level design for the whole repository docs/lld/ 12 low-level designs, one per component docs/adr/ 11 decision records for choices already in force The ADRs are retrospective: they record why the safety model looks the way it does (typed order outcomes, the two-flag live gate, the per-strategy size multiplier, host-owned gates around the LLM agents), which is the part that is expensive to reconstruct later. Tests/ — every test file and test conftest moved into a top-level Tests/ tree mirroring the source layout, so runtime folders hold only runtime code. Each mirrored folder for a spaced-name source folder carries a conftest.py that puts the SOURCE folder on sys.path, never the test folder, preserving the narrow import scope the co-located conftests had. Path anchors now resolve from the repository root. Updated for the new paths: the CI workflow (both jobs), pyproject (Ruff per-file-ignores and coverage omit), README.md, CLAUDE.md, AGENTS.md and the per-folder Readmes. CLAUDE.md and AGENTS.md remain byte-identical after the "What this project is" marker, as test_repository_policy requires. Verified by exact count rather than by a green run: 487 master + 26 market-data-health + 1089 pytest = 1602, before and after. Also green: ruff, mypy (53 files), compileall, bandit, pre-commit config validation, the branch-enabled coverage run (71.4%, floor 54.7%) with zero Tests/ files leaking into the report, and the per-module coverage policy gate. Co-Authored-By: Claude Opus 5 --- .github/workflows/quality-and-security.yml | 17 +- .gitignore | 4 + AGENTS.md | 31 +- CLAUDE.md | 31 +- Dependencies/Readme.md | 5 +- README.md | 24 +- Signal Generators/Readme.md | 2 +- .../REGIME_PORTING_NOTES.md | 4 +- .../Regime Adaptive Strategy/conftest.py | 23 - .../SL Hunting AI Agent/README.md | 6 +- .../SL Hunting AI Agent/conftest.py | 16 - .../test_index_fetch_construction.py | 4 +- .../Dhan API/test_dhan_execution.py | 3 +- .../Flattrade API/test_flattrade_execution.py | 5 +- Tests/Dependencies/conftest.py | 31 ++ .../Dependencies}/test_broker_contract.py | 3 +- .../Dependencies}/test_check_env_config.py | 3 +- .../Dependencies}/test_dhan_token_setup.py | 0 .../test_diagnostic_preflight.py | 3 +- .../Dependencies}/test_execution_ledger.py | 0 .../Dependencies}/test_next_open_entry.py | 0 .../Dependencies}/test_order_splitting.py | 0 .../Dependencies}/test_repository_policy.py | 3 +- .../Dependencies}/test_risk_sizing.py | 0 .../Dependencies}/test_secret_redaction.py | 0 .../Dependencies}/test_startup_exposure.py | 0 .../Dependencies}/test_tick_bar_builder.py | 0 .../Dependencies}/test_trading_lifecycle.py | 0 .../CPR AI Agent/conftest.py | 13 +- .../CPR AI Agent}/test_cpr_ai_context.py | 0 .../CPR AI Agent}/test_cpr_ai_core.py | 0 .../test_cpr_ai_master_integration.py | 6 +- .../CPR AI Agent}/test_cpr_ai_runtime.py | 0 .../test_cpr_strategy_signal_generators.py | 3 +- .../Regime Adaptive Strategy/conftest.py | 31 ++ .../test_regime_adaptive.py | 5 +- .../SL Hunting AI Agent/conftest.py | 23 + .../test_sl_hunting_agent.py | 0 .../test_sl_hunting_indicators.py | 0 .../test_sl_hunting_journal.py | 0 .../test_sl_hunting_lessons.py | 0 .../test_sl_hunting_premarket.py | 15 +- .../test_sl_hunting_runner.py | 0 .../test_sl_hunting_schema.py | 0 .../test_sl_hunting_v2.py | 0 ...est_subhamoy_strategy_signal_generators.py | 5 +- .../test_deterministic_strategy_safety.py | 3 +- .../Signal Generators}/test_renko_bounds.py | 3 +- .../test_trading_bot_ports.py | 3 +- .../test_market_data_health.py | 0 .../test_nifty_multi_strategy_master.py | 10 +- docs/README.md | 74 ++++ ...0001-single-process-thread-per-strategy.md | 109 +++++ ...0002-broker-agnostic-execution-contract.md | 109 +++++ .../adr/0003-acknowledgement-is-not-a-fill.md | 98 +++++ docs/adr/0004-paper-by-default-double-gate.md | 109 +++++ .../adr/0005-rest-vs-websocket-market-data.md | 116 ++++++ docs/adr/0006-per-strategy-size-multiplier.md | 115 +++++ docs/adr/0007-llm-agents-as-opt-in-workers.md | 129 ++++++ docs/adr/0008-single-env-as-config-source.md | 107 +++++ ...-importlib-loading-for-spaced-filenames.md | 116 ++++++ .../0010-tests-in-a-mirrored-tests-tree.md | 133 ++++++ ...11-committed-docs-untracked-superpowers.md | 125 ++++++ docs/hld/system-overview.md | 335 +++++++++++++++ docs/lld/configuration.md | 132 ++++++ docs/lld/cpr-codex-ai-agent.md | 186 +++++++++ docs/lld/data-and-backtesting.md | 133 ++++++ docs/lld/execution-and-brokers.md | 238 +++++++++++ docs/lld/market-data.md | 182 ++++++++ docs/lld/master-runner.md | 170 ++++++++ docs/lld/regime-adaptive.md | 102 +++++ docs/lld/reporting-and-observability.md | 137 ++++++ docs/lld/risk-and-safety.md | 220 ++++++++++ docs/lld/sl-hunting-ai-agent.md | 216 ++++++++++ docs/lld/strategy-workers.md | 189 +++++++++ docs/lld/testing-and-ci.md | 220 ++++++++++ .../2026-07-27-mat-111-sl-hunting-cooldown.md | 393 ------------------ ...7-27-mat-111-sl-hunting-cooldown-design.md | 112 ----- pyproject.toml | 10 +- 79 files changed, 4037 insertions(+), 616 deletions(-) delete mode 100644 Signal Generators/Regime Adaptive Strategy/conftest.py delete mode 100644 Signal Generators/SL Hunting AI Agent/conftest.py rename {Data Extractors => Tests/Data Extractors}/test_index_fetch_construction.py (96%) rename {Dependencies => Tests/Dependencies}/Dhan API/test_dhan_execution.py (99%) rename {Dependencies => Tests/Dependencies}/Flattrade API/test_flattrade_execution.py (95%) create mode 100644 Tests/Dependencies/conftest.py rename {Dependencies => Tests/Dependencies}/test_broker_contract.py (99%) rename {Dependencies => Tests/Dependencies}/test_check_env_config.py (98%) rename {Dependencies => Tests/Dependencies}/test_dhan_token_setup.py (100%) rename {Dependencies => Tests/Dependencies}/test_diagnostic_preflight.py (97%) rename {Dependencies => Tests/Dependencies}/test_execution_ledger.py (100%) rename {Dependencies => Tests/Dependencies}/test_next_open_entry.py (100%) rename {Dependencies => Tests/Dependencies}/test_order_splitting.py (100%) rename {Dependencies => Tests/Dependencies}/test_repository_policy.py (99%) rename {Dependencies => Tests/Dependencies}/test_risk_sizing.py (100%) rename {Dependencies => Tests/Dependencies}/test_secret_redaction.py (100%) rename {Dependencies => Tests/Dependencies}/test_startup_exposure.py (100%) rename {Dependencies => Tests/Dependencies}/test_tick_bar_builder.py (100%) rename {Dependencies => Tests/Dependencies}/test_trading_lifecycle.py (100%) rename {Signal Generators => Tests/Signal Generators}/CPR AI Agent/conftest.py (66%) rename {Signal Generators/CPR AI Agent/tests => Tests/Signal Generators/CPR AI Agent}/test_cpr_ai_context.py (100%) rename {Signal Generators/CPR AI Agent/tests => Tests/Signal Generators/CPR AI Agent}/test_cpr_ai_core.py (100%) rename {Signal Generators/CPR AI Agent/tests => Tests/Signal Generators/CPR AI Agent}/test_cpr_ai_master_integration.py (75%) rename {Signal Generators/CPR AI Agent/tests => Tests/Signal Generators/CPR AI Agent}/test_cpr_ai_runtime.py (100%) rename {Signal Generators => Tests/Signal Generators}/CPR Strategy/test_cpr_strategy_signal_generators.py (99%) create mode 100644 Tests/Signal Generators/Regime Adaptive Strategy/conftest.py rename {Signal Generators => Tests/Signal Generators}/Regime Adaptive Strategy/test_regime_adaptive.py (98%) create mode 100644 Tests/Signal Generators/SL Hunting AI Agent/conftest.py rename {Signal Generators/SL Hunting AI Agent/tests => Tests/Signal Generators/SL Hunting AI Agent}/test_sl_hunting_agent.py (100%) rename {Signal Generators/SL Hunting AI Agent/tests => Tests/Signal Generators/SL Hunting AI Agent}/test_sl_hunting_indicators.py (100%) rename {Signal Generators/SL Hunting AI Agent/tests => Tests/Signal Generators/SL Hunting AI Agent}/test_sl_hunting_journal.py (100%) rename {Signal Generators/SL Hunting AI Agent/tests => Tests/Signal Generators/SL Hunting AI Agent}/test_sl_hunting_lessons.py (100%) rename {Signal Generators/SL Hunting AI Agent/tests => Tests/Signal Generators/SL Hunting AI Agent}/test_sl_hunting_premarket.py (95%) rename {Signal Generators/SL Hunting AI Agent/tests => Tests/Signal Generators/SL Hunting AI Agent}/test_sl_hunting_runner.py (100%) rename {Signal Generators/SL Hunting AI Agent/tests => Tests/Signal Generators/SL Hunting AI Agent}/test_sl_hunting_schema.py (100%) rename {Signal Generators/SL Hunting AI Agent/tests => Tests/Signal Generators/SL Hunting AI Agent}/test_sl_hunting_v2.py (100%) rename {Signal Generators => Tests/Signal Generators}/Subhamoy Strategies/test_subhamoy_strategy_signal_generators.py (97%) rename {Signal Generators => Tests/Signal Generators}/test_deterministic_strategy_safety.py (98%) rename {Signal Generators => Tests/Signal Generators}/test_renko_bounds.py (92%) rename {Signal Generators => Tests/Signal Generators}/test_trading_bot_ports.py (99%) rename test_market_data_health.py => Tests/test_market_data_health.py (100%) rename test_nifty_multi_strategy_master.py => Tests/test_nifty_multi_strategy_master.py (99%) create mode 100644 docs/README.md create mode 100644 docs/adr/0001-single-process-thread-per-strategy.md create mode 100644 docs/adr/0002-broker-agnostic-execution-contract.md create mode 100644 docs/adr/0003-acknowledgement-is-not-a-fill.md create mode 100644 docs/adr/0004-paper-by-default-double-gate.md create mode 100644 docs/adr/0005-rest-vs-websocket-market-data.md create mode 100644 docs/adr/0006-per-strategy-size-multiplier.md create mode 100644 docs/adr/0007-llm-agents-as-opt-in-workers.md create mode 100644 docs/adr/0008-single-env-as-config-source.md create mode 100644 docs/adr/0009-importlib-loading-for-spaced-filenames.md create mode 100644 docs/adr/0010-tests-in-a-mirrored-tests-tree.md create mode 100644 docs/adr/0011-committed-docs-untracked-superpowers.md create mode 100644 docs/hld/system-overview.md create mode 100644 docs/lld/configuration.md create mode 100644 docs/lld/cpr-codex-ai-agent.md create mode 100644 docs/lld/data-and-backtesting.md create mode 100644 docs/lld/execution-and-brokers.md create mode 100644 docs/lld/market-data.md create mode 100644 docs/lld/master-runner.md create mode 100644 docs/lld/regime-adaptive.md create mode 100644 docs/lld/reporting-and-observability.md create mode 100644 docs/lld/risk-and-safety.md create mode 100644 docs/lld/sl-hunting-ai-agent.md create mode 100644 docs/lld/strategy-workers.md create mode 100644 docs/lld/testing-and-ci.md delete mode 100644 docs/superpowers/plans/2026-07-27-mat-111-sl-hunting-cooldown.md delete mode 100644 docs/superpowers/specs/2026-07-27-mat-111-sl-hunting-cooldown-design.md diff --git a/.github/workflows/quality-and-security.yml b/.github/workflows/quality-and-security.yml index 741f9d6..54e34d9 100644 --- a/.github/workflows/quality-and-security.yml +++ b/.github/workflows/quality-and-security.yml @@ -56,14 +56,15 @@ jobs: run: python -m pre_commit validate-config .pre-commit-config.yaml - name: Run branch-enabled test coverage - # The master is importlib-loaded because its filename has spaces. Append - # every suite into one branch-enabled runtime report, then enforce the - # 54.7% repository baseline plus the stricter per-safety/broker budgets. + # Every suite lives under Tests/, mirroring the source tree. The master is + # importlib-loaded because its filename has spaces. Append every suite into + # one branch-enabled runtime report, then enforce the 54.7% repository + # baseline plus the stricter per-safety/broker budgets. run: | python -m coverage erase - python -m coverage run -m unittest test_nifty_multi_strategy_master - python -m coverage run --append -m unittest test_market_data_health - python -m coverage run --append -m pytest "Signal Generators" "Dependencies" "Data Extractors" -q + python -m coverage run -m unittest Tests.test_nifty_multi_strategy_master + python -m coverage run --append -m unittest Tests.test_market_data_health + python -m coverage run --append -m pytest "Tests/Signal Generators" "Tests/Dependencies" "Tests/Data Extractors" -q python -m coverage json -o coverage.json python -m coverage report python scripts/check_coverage_thresholds.py coverage.json @@ -123,6 +124,6 @@ jobs: - name: Run broker contract and adapter suites run: > python -m pytest - "Dependencies/test_broker_contract.py" - "Dependencies/Flattrade API/test_flattrade_execution.py" + "Tests/Dependencies/test_broker_contract.py" + "Tests/Dependencies/Flattrade API/test_flattrade_execution.py" -q diff --git a/.gitignore b/.gitignore index 8c7589b..da232f5 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,7 @@ htmlcov/ # reports here. They describe one development session rather than the product, # so keep the complete workspace-private directory out of source control. .superpowers/ +# Same reasoning for the plans/specs Superpowers writes under docs/: they are a +# per-session working record, not product documentation. docs/ itself is the +# committed architecture set (hld/, lld/, adr/) -- only this subtree is ignored. +docs/superpowers/ diff --git a/AGENTS.md b/AGENTS.md index c0f7475..11bb9ff 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -79,8 +79,9 @@ One process, cooperating threads: ``` Nifty Multi Strategy Front Test - Master File.py # the multithreaded paper/live runner (the "big one") algo.py # unified CLI: fetch-data / backtest / run / setup-token / diagnose / check-env -test_nifty_multi_strategy_master.py # unittest suite for the master -test_market_data_health.py # unittest suite for the shared feed-health gates +Tests/ # EVERY test, mirroring the source tree (docs/adr/0010) + test_nifty_multi_strategy_master.py # unittest suite for the master + test_market_data_health.py # unittest suite for the shared feed-health gates requirements.txt # exact core runtime dependencies requirements-brokers.txt # exact Kotak/Shoonya optional live set requirements-ai.txt # exact optional Claude Agent SDK stack @@ -98,12 +99,13 @@ Dependencies/ check_env_config.py # `algo.py check-env` config-drift audit (read-only) Kotak API/ -> kotak_execution.py, diagnose_kotak_symbol.py Shoonya API/ -> NorenApi.py (vendored client), shoonya_execution.py, diagnose_shoonya_symbol.py - Flattrade API/ -> flattrade_execution.py, diagnose_flattrade_symbol.py, - test_flattrade_execution.py - Dhan API/ -> dhan_execution.py, diagnose_dhan_symbol.py, test_dhan_execution.py + Flattrade API/ -> flattrade_execution.py, diagnose_flattrade_symbol.py + Dhan API/ -> dhan_execution.py, diagnose_dhan_symbol.py pyproject.toml # ruff + mypy quality-gate configuration .github/workflows/quality-and-security.yml # CI: tests + compileall + ruff + mypy + bandit scripts/check_coverage_thresholds.py # branch-coverage policy gate +docs/ # committed architecture set: hld/, lld/, adr/ + # (docs/superpowers/ is a session scratchpad, gitignored) Backtest Outputs/ # generated CSVs/logs (gitignored) ``` @@ -175,15 +177,20 @@ Backtest Outputs/ # generated CSVs/logs (gitign `Dependencies/.env` against `env.example` and against the keys the code's `_env_*` calls actually read, reporting settings missing from `.env` (an unseen in-code default is in force), mistyped or stale keys, and knobs missing from the template. Read-only, and it prints key NAMES only — never a - value out of `.env` — so its output is safe to share. `test_repository_policy.py` imports the same + value out of `.env` — so its output is safe to share. `Tests/Dependencies/test_repository_policy.py` imports the same helpers so CI fails when a new `_env_*` key lands without an `env.example` entry. -- **Tests:** `python -m unittest test_nifty_multi_strategy_master` (loads the master via `importlib`, - mocks `dhanhq`; broker/SDK-specific cases skip when those deps are absent). Signal-generator tests live - under `Signal Generators/`. +- **Tests:** EVERY suite lives under `Tests/`, mirroring the source tree — the test for + `Signal Generators/` sits at `Tests/Signal Generators/`. Run the master suite with + `python -m unittest Tests.test_nifty_multi_strategy_master` (loads the master via `importlib`, + mocks `dhanhq`; broker/SDK-specific cases skip when those deps are absent). Two rules when adding + a test: put it at the mirrored path, and keep its FILENAME unique repository-wide (pytest keys + modules by basename — there are no `__init__.py` files). A `Tests/` folder mirroring a + spaced-name source folder carries a `conftest.py` that puts the SOURCE folder on `sys.path`, + never the test folder, so tests exercise the same import resolution production uses. - **Quality gates (run before pushing; CI enforces on Python 3.12 + 3.13):** - `python -m unittest test_nifty_multi_strategy_master`, - `python -m unittest test_market_data_health`, - `python -m pytest "Signal Generators" "Dependencies" "Data Extractors" -q`, + `python -m unittest Tests.test_nifty_multi_strategy_master`, + `python -m unittest Tests.test_market_data_health`, + `python -m pytest "Tests/Signal Generators" "Tests/Dependencies" "Tests/Data Extractors" -q`, the branch-enabled Coverage.py run plus `scripts/check_coverage_thresholds.py`, pip-audit of committed pins locally plus the clean resolved CI environment, Ruff, mypy, compileall, diff --git a/CLAUDE.md b/CLAUDE.md index 86287cc..6c34263 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -79,8 +79,9 @@ One process, cooperating threads: ``` Nifty Multi Strategy Front Test - Master File.py # the multithreaded paper/live runner (the "big one") algo.py # unified CLI: fetch-data / backtest / run / setup-token / diagnose / check-env -test_nifty_multi_strategy_master.py # unittest suite for the master -test_market_data_health.py # unittest suite for the shared feed-health gates +Tests/ # EVERY test, mirroring the source tree (docs/adr/0010) + test_nifty_multi_strategy_master.py # unittest suite for the master + test_market_data_health.py # unittest suite for the shared feed-health gates requirements.txt # exact core runtime dependencies requirements-brokers.txt # exact Kotak/Shoonya optional live set requirements-ai.txt # exact optional Claude Agent SDK stack @@ -98,12 +99,13 @@ Dependencies/ check_env_config.py # `algo.py check-env` config-drift audit (read-only) Kotak API/ -> kotak_execution.py, diagnose_kotak_symbol.py Shoonya API/ -> NorenApi.py (vendored client), shoonya_execution.py, diagnose_shoonya_symbol.py - Flattrade API/ -> flattrade_execution.py, diagnose_flattrade_symbol.py, - test_flattrade_execution.py - Dhan API/ -> dhan_execution.py, diagnose_dhan_symbol.py, test_dhan_execution.py + Flattrade API/ -> flattrade_execution.py, diagnose_flattrade_symbol.py + Dhan API/ -> dhan_execution.py, diagnose_dhan_symbol.py pyproject.toml # ruff + mypy quality-gate configuration .github/workflows/quality-and-security.yml # CI: tests + compileall + ruff + mypy + bandit scripts/check_coverage_thresholds.py # branch-coverage policy gate +docs/ # committed architecture set: hld/, lld/, adr/ + # (docs/superpowers/ is a session scratchpad, gitignored) Backtest Outputs/ # generated CSVs/logs (gitignored) ``` @@ -175,15 +177,20 @@ Backtest Outputs/ # generated CSVs/logs (gitign `Dependencies/.env` against `env.example` and against the keys the code's `_env_*` calls actually read, reporting settings missing from `.env` (an unseen in-code default is in force), mistyped or stale keys, and knobs missing from the template. Read-only, and it prints key NAMES only — never a - value out of `.env` — so its output is safe to share. `test_repository_policy.py` imports the same + value out of `.env` — so its output is safe to share. `Tests/Dependencies/test_repository_policy.py` imports the same helpers so CI fails when a new `_env_*` key lands without an `env.example` entry. -- **Tests:** `python -m unittest test_nifty_multi_strategy_master` (loads the master via `importlib`, - mocks `dhanhq`; broker/SDK-specific cases skip when those deps are absent). Signal-generator tests live - under `Signal Generators/`. +- **Tests:** EVERY suite lives under `Tests/`, mirroring the source tree — the test for + `Signal Generators/` sits at `Tests/Signal Generators/`. Run the master suite with + `python -m unittest Tests.test_nifty_multi_strategy_master` (loads the master via `importlib`, + mocks `dhanhq`; broker/SDK-specific cases skip when those deps are absent). Two rules when adding + a test: put it at the mirrored path, and keep its FILENAME unique repository-wide (pytest keys + modules by basename — there are no `__init__.py` files). A `Tests/` folder mirroring a + spaced-name source folder carries a `conftest.py` that puts the SOURCE folder on `sys.path`, + never the test folder, so tests exercise the same import resolution production uses. - **Quality gates (run before pushing; CI enforces on Python 3.12 + 3.13):** - `python -m unittest test_nifty_multi_strategy_master`, - `python -m unittest test_market_data_health`, - `python -m pytest "Signal Generators" "Dependencies" "Data Extractors" -q`, + `python -m unittest Tests.test_nifty_multi_strategy_master`, + `python -m unittest Tests.test_market_data_health`, + `python -m pytest "Tests/Signal Generators" "Tests/Dependencies" "Tests/Data Extractors" -q`, the branch-enabled Coverage.py run plus `scripts/check_coverage_thresholds.py`, pip-audit of committed pins locally plus the clean resolved CI environment, Ruff, mypy, compileall, diff --git a/Dependencies/Readme.md b/Dependencies/Readme.md index bf50df1..7818cf5 100644 --- a/Dependencies/Readme.md +++ b/Dependencies/Readme.md @@ -49,8 +49,9 @@ log), the nightly-refreshed `all_instrument .csv` Dhan instrument master ( for option-contract resolution by the runner and the Dhan adapter), and the Google Sheets OAuth token cache when the EOD sheet writer is enabled. -(`test_market_data_health.py` lives at the repo root next to the master's own suite; -everything else in this folder is tested right here.) +(Tests for everything in this folder live at the mirrored path under `Tests/Dependencies/`, +alongside `Tests/test_market_data_health.py` and the master's own suite. See +`docs/adr/0010-tests-in-a-mirrored-tests-tree.md` for why.) ## Keeping `.env` honest Configuration drifts in three directions at once: your `.env`, the committed diff --git a/README.md b/README.md index d505097..5344336 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Although I own the code, the coding itself was done entirely using GPT-5.4-xhigh - **SL Hunting AI Agent — BankNIFTY mirror basket + newer knowledge (v3c–v3e).** The agent now trades Intraday Hunter's multi-index style: every NIFTY entry is mirrored with an **equal-lot BankNIFTY ATM** leg (`SL_HUNTING_BNF_MIRROR`, default true). The two legs are **tied for hard risk** (stop/target, max-loss, 15:15 square-off close both) but the agent evaluates each leg's **premise independently** and can cut one alone via the EXIT `exit_leg` selector (`NIFTY` | `BNF` | `BOTH`). Entry stays NIFTY-only (the mirror copies it). Its knowledge also grew several distilled-from-video layers — a scoped **gap-up opening-drive**, a **2-week verbatim transcript sweep**, and a **live-day match** against the agent's own journal (details in `Signal Generators/SL Hunting AI Agent/README.md`). - **Optional LLM trading agent — the "SL Hunting AI Agent" (opt-in worker).** A Claude agent (via the [`claude-agent-sdk`](https://pypi.org/project/claude-agent-sdk/) on your Claude subscription — **no API key**) trades the discretionary *SL Hunting* price-action method on NIFTY ATM options. Once per completed 1-min bar (the method's native timeframe) it reads the NIFTY chart (with **BankNIFTY cross-confirmation**) and — only on a confirmed setup at a real level — acts through the SAME tested `enter_position`/`exit_position` path as every other worker. Position sizing floors affordable whole lots, never exceeds `SL_HUNTING_RISK_BUDGET`, skips one-lot-over-budget setups, and caps at `SL_HUNTING_MAX_LOTS` (default 5); the equal-lot BankNIFTY mirror can roughly double basket risk. It **stops opening new positions after 10:30** (`SL_HUNTING_NO_NEW_ENTRY_HOUR`/`_MINUTE`, default 10:30) — *not* a square-off: open positions, their stops/targets, and the 15:15 square-off are unaffected. Its post-exit cooldown starts only when the whole NIFTY/BankNIFTY basket is confirmed flat, so an independently surviving or partly closed leg cannot run the timer down; exits never consult this guard, while unreadable guard state rejects new live entries. It is **off by default** (`SL_HUNTING_ENABLED`), trades **paper** unless both `LIVE_TRADING_ENABLED` and `SL_HUNTING_LIVE_TRADING` are set, and is **fail-soft** — any agent/SDK error becomes a safe HOLD while its separate mechanical risk loop keeps checking stop, target, max-loss, stale data, and square-off. It can also **learn from its own trades** through a tool-free, schema-validated reflection coach with digest-bound human approval (paper-first, off by default). Install the exact optional stack with `pip install -r requirements-ai.txt` and run one-time `claude setup-token` (keep `ANTHROPIC_API_KEY` **UNSET** so it bills your Claude plan, not per-token API). Full details — knowledge, tools, safety model, the learning loop — are in `Signal Generators/SL Hunting AI Agent/README.md`. It joins the configuration-dependent worker roster only when enabled. - **CPR Algo 3 (multi-instrument) is now wired into the front test.** A new `CPRAlgo3StrategyWorker` runs the "CPR basic setup" strategy, which watches THREE charts at once — the NIFTY spot plus a ~ITM CE and a ~ITM PE of the current-week expiry — and only fires when VWAP and the CPR band align across all three (RSI/ARSI on spot). The two ITM options are **observation only**: a signal still BUYS the ATM CE/PE of the next-next expiry through the same tested path as the other directional workers, so it shares CPR's risk knobs (tunable via `CPR_ALGO3_*` in `.env`, including `CPR_ALGO3_ITM_OFFSET`). It fetches the two option 1-min OHLC feeds on demand and drives its own spot target/stop exit. It belongs to the core roster, while the enabled total remains configuration-dependent. (The standalone Algo 3 signal generator + its unit tests live under `Signal Generators/CPR Strategy/`.) -- **Code-quality pass.** Added a `requirements.txt`; gave every Shoonya broker HTTP call a timeout (a hung call could otherwise stall a worker thread and the shared broker lock); removed hardcoded credentials from the vendored Shoonya client; routed the execution layer's status/errors through `logging` instead of `print()`; and ported the master test suite into the repo (`test_nifty_multi_strategy_master.py` — see Tests below). +- **Code-quality pass.** Added a `requirements.txt`; gave every Shoonya broker HTTP call a timeout (a hung call could otherwise stall a worker thread and the shared broker lock); removed hardcoded credentials from the vendored Shoonya client; routed the execution layer's status/errors through `logging` instead of `print()`; and ported the master test suite into the repo (`Tests/test_nifty_multi_strategy_master.py` — see Tests below). - **Live broker execution is broker-selectable (Kotak Neo, Shoonya, or Flattrade).** `LIVE_BROKER` picks `KOTAK`, `SHOONYA`, or `FLATTRADE`, and every real order goes through one generic `execution_client`. The global `LIVE_TRADING_ENABLED` kill-switch and each strategy's `_LIVE_TRADING` flag must both be true; unknown broker names fail closed to paper. Each broker folder contains an execution client and a read-only diagnostic with an optional, typed-`YES`, round-trip test order. Flattrade uses its official Pi v2 browser-token flow, exact NFO index scrip master, documented request limits, market-order protection, and `SingleOrdHist` fill confirmation. Everything still defaults to paper. (Shoonya's legacy QuickAuth endpoint is being decommissioned by Finvasia.) - **End-of-day P&L is now written to a Google Sheet.** When all workers exit on a clean end of day, the master parses the run's log for each strategy's realised P&L and writes it into a tracker sheet — one row per strategy, one column per calendar day — overwriting today's cell and backfilling any blank earlier-this-month cells from the (append-mode) log. Auth is OAuth user-token via `gspread`; configure `GSHEET_ID` + an OAuth client in `.env` (see Setup). It's a safe no-op when unconfigured, so it never disturbs shutdown. - **13 TradingBot signal-generator ports.** Thirteen ATM single-leg strategies were ported into `Signal Generators/` (SMA Crossover, Bollinger Bands, Keltner Squeeze, Mean Reversion Z-Score, ML Ensemble, Multi-Timeframe, Opening Range Breakout, Parabolic SAR, RSI Divergence, RSI Reversal, Stochastic, Supertrend, Volatility Breakout), all sharing `misc_strategy_common.py` and the mandatory TA-Lib 0.6.8 indicator backend. They're wired through the shared `AtmSingleLegStrategyWorker` factory and each is tunable from `.env` by its own prefix. ML Ensemble needs `scikit-learn`. @@ -49,6 +49,8 @@ You might have to adjust the import addresses from which the files are to be imp ├── Data Extractors/ # 1m OHLC downloaders + shared helper ├── My Backtest Files (For Reference)/ # backtesting.py-based backtests ├── Signal Generators/ # strategy / signal logic modules +├── Tests/ # EVERY test, mirroring the tree above +├── docs/ # architecture docs: hld/, lld/, adr/ └── Dependencies/ # shared config + live-execution layer ├── env.example # copy to Dependencies/.env and fill in ├── dhan_token_setup.py # one-time DhanHQ OAuth token setup @@ -137,11 +139,15 @@ Run `python algo.py --help`, or `python algo.py --help`, for the detai The `Backtest Outputs/` folder is `.gitignore`-d, so generated CSVs/logs stay local. # Tests +**Every test in this repository lives under `Tests/`, mirroring the source tree** — the tests for `Signal Generators/SL Hunting AI Agent/` are at `Tests/Signal Generators/SL Hunting AI Agent/`, and so on. Runtime folders contain only runtime code. + The front-test master has a unittest suite — env toggles, broker paper/live routing and the fail-closed `LIVE_BROKER` switch, order fill-confirmation, and symbol resolution. Run it from the repo root: ``` -python -m unittest test_nifty_multi_strategy_master +python -m unittest Tests.test_nifty_multi_strategy_master ``` -Broker/SDK-specific cases skip automatically when optional dependencies are absent, and all broker HTTP/browser/order behaviour is mocked. Signal generators, execution/reconciliation primitives, data extractors, and repository-policy checks have focused suites under their respective folders. CI runs the whole quality gate on every push/PR — see "Quality gates & CI" below. +Broker/SDK-specific cases skip automatically when optional dependencies are absent, and all broker HTTP/browser/order behaviour is mocked. Signal generators, execution/reconciliation primitives, data extractors, and repository-policy checks have focused suites at their mirrored paths. CI runs the whole quality gate on every push/PR — see "Quality gates & CI" below. + +Adding a test? Put it at the mirrored path, and give it a filename that is unique across the repository — pytest keys modules by basename (there are no `__init__.py` files). Where a `Tests/` folder mirrors a source folder whose name contains spaces, it carries a `conftest.py` that puts the **source** folder on `sys.path`, so tests resolve imports exactly the way the runner does. The reasoning is in [`docs/adr/0010-tests-in-a-mirrored-tests-tree.md`](docs/adr/0010-tests-in-a-mirrored-tests-tree.md). # Quality gates & CI A GitHub Actions workflow (`.github/workflows/quality-and-security.yml`) runs on every push and pull request across Python 3.12 and 3.13. Locally, the same gate is: @@ -149,13 +155,13 @@ A GitHub Actions workflow (`.github/workflows/quality-and-security.yml`) runs on pip install -r requirements-dev.txt pip install -r requirements-ai.txt pip install -r requirements-codex-ai.txt -python -m unittest test_nifty_multi_strategy_master -python -m unittest test_market_data_health -python -m pytest "Signal Generators" "Dependencies" "Data Extractors" -q +python -m unittest Tests.test_nifty_multi_strategy_master +python -m unittest Tests.test_market_data_health +python -m pytest "Tests/Signal Generators" "Tests/Dependencies" "Tests/Data Extractors" -q python -m coverage erase -python -m coverage run -m unittest test_nifty_multi_strategy_master -python -m coverage run --append -m unittest test_market_data_health -python -m coverage run --append -m pytest "Signal Generators" "Dependencies" "Data Extractors" -q +python -m coverage run -m unittest Tests.test_nifty_multi_strategy_master +python -m coverage run --append -m unittest Tests.test_market_data_health +python -m coverage run --append -m pytest "Tests/Signal Generators" "Tests/Dependencies" "Tests/Data Extractors" -q python -m coverage json -o coverage.json python scripts/check_coverage_thresholds.py coverage.json python -m pip_audit -r requirements.txt --no-deps --progress-spinner off diff --git a/Signal Generators/Readme.md b/Signal Generators/Readme.md index 48aa12f..3846d26 100644 --- a/Signal Generators/Readme.md +++ b/Signal Generators/Readme.md @@ -78,8 +78,8 @@ Everything for it lives in its own folder, `Regime Adaptive Strategy/`: | `Nifty Regime Adaptive Signal Generator.py` | the router — the only worker of the three | | `regime_candidates.py` | the two candidate rules, as pure column-producing functions | | `regime_common.py` | session date, session VWAP, session opening range — and this folder's **only** `sys.path` bootstrap, which is why it re-exports the shared indicators from `misc_strategy_common` one level up | -| `conftest.py` | the pytest equivalent of that bootstrap (same pattern as `SL Hunting AI Agent/`) | | `REGIME_PORTING_NOTES.md` | **read before enabling live** — what was dropped and why | +| _(tests)_ | at the mirrored path `Tests/Signal Generators/Regime Adaptive Strategy/`, whose `conftest.py` is the pytest equivalent of that bootstrap (same pattern as `SL Hunting AI Agent/`) | Two things to know before touching it: diff --git a/Signal Generators/Regime Adaptive Strategy/REGIME_PORTING_NOTES.md b/Signal Generators/Regime Adaptive Strategy/REGIME_PORTING_NOTES.md index 315372c..515ed20 100644 --- a/Signal Generators/Regime Adaptive Strategy/REGIME_PORTING_NOTES.md +++ b/Signal Generators/Regime Adaptive Strategy/REGIME_PORTING_NOTES.md @@ -13,8 +13,8 @@ attributed in each new module's docstring. | `regime_common.py` | Session date, session VWAP, session opening range | | `regime_candidates.py` | The two candidate rules, as pure column-producing functions | | `Nifty Regime Adaptive Signal Generator.py` | The router — the only new worker | -| `conftest.py` | pytest `sys.path` bootstrap for this folder | -| `test_regime_adaptive.py` | Behaviour tests for the above | +| `Tests/Signal Generators/Regime Adaptive Strategy/conftest.py` | pytest `sys.path` bootstrap pointing back at this folder | +| `Tests/Signal Generators/Regime Adaptive Strategy/test_regime_adaptive.py` | Behaviour tests for the above | **One note on imports.** The master's `load_module()` puts only the loaded file's own directory on `sys.path`, so a module in here cannot see diff --git a/Signal Generators/Regime Adaptive Strategy/conftest.py b/Signal Generators/Regime Adaptive Strategy/conftest.py deleted file mode 100644 index 95322b4..0000000 --- a/Signal Generators/Regime Adaptive Strategy/conftest.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Pytest bootstrap for the Regime Adaptive strategy folder. - -This folder's name contains spaces and its modules import each other by bare -name (`import regime_common`, etc.), so this adds the folder to ``sys.path`` -before the tests import anything -- the same pattern as -``SL Hunting AI Agent/conftest.py``. - -The parent ("Signal Generators/") is added too, because ``regime_common`` -re-exports the shared indicators from ``misc_strategy_common`` which lives one -level up. At runtime ``regime_common`` bootstraps that itself; under pytest the -import can happen through a different entry point, so it is done here as well. -""" - -from __future__ import annotations - -import os -import sys - -_HERE = os.path.dirname(os.path.abspath(__file__)) -_SIGNAL_GEN_DIR = os.path.dirname(_HERE) -for _path in (_SIGNAL_GEN_DIR, _HERE): - if _path not in sys.path: - sys.path.insert(0, _path) diff --git a/Signal Generators/SL Hunting AI Agent/README.md b/Signal Generators/SL Hunting AI Agent/README.md index c533c32..93eec0c 100644 --- a/Signal Generators/SL Hunting AI Agent/README.md +++ b/Signal Generators/SL Hunting AI Agent/README.md @@ -55,7 +55,6 @@ kill-switch (also scaled) still caps the day. | `sl_hunting_coach.py` | v3 reflection coach — proposes lessons from the journal (`--reflect`/`--promote`/`--list`). | | `sl_hunting_lessons.py` | v3 lessons store (schema, consolidate, `format_lessons` for prompt injection). | | `lessons.json` | The APPROVED (live) lessons the agent injects — starts empty; you promote into it. | -| `tests/` | pytest suite — runs with a fake runner, no SDK/CLI/network. | ## Setup (one-time) ```bash @@ -139,8 +138,11 @@ cutoff the agent isn't called at all, so it makes **no LLM calls for the rest of worker — the rest of the master and its test suite are unaffected. ## Tests +The suite lives at the mirrored path under `Tests/` (see +`docs/adr/0010-tests-in-a-mirrored-tests-tree.md`). It runs with a fake runner — +no SDK, CLI, or network. ```bash -pytest "Signal Generators/SL Hunting AI Agent/tests" +pytest "Tests/Signal Generators/SL Hunting AI Agent" ``` ## BankNIFTY cross-confirmation (v2) diff --git a/Signal Generators/SL Hunting AI Agent/conftest.py b/Signal Generators/SL Hunting AI Agent/conftest.py deleted file mode 100644 index 69ea35a..0000000 --- a/Signal Generators/SL Hunting AI Agent/conftest.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Pytest bootstrap for the SL Hunting AI Agent. - -The folder name contains spaces and the modules import each other by bare name -(`import sl_hunting_tools`, etc.), so we add this folder to ``sys.path`` before the -tests import anything. pytest auto-loads this ``conftest.py`` because it sits above -the ``tests/`` directory. -""" - -from __future__ import annotations - -import os -import sys - -_HERE = os.path.dirname(os.path.abspath(__file__)) -if _HERE not in sys.path: - sys.path.insert(0, _HERE) diff --git a/Data Extractors/test_index_fetch_construction.py b/Tests/Data Extractors/test_index_fetch_construction.py similarity index 96% rename from Data Extractors/test_index_fetch_construction.py rename to Tests/Data Extractors/test_index_fetch_construction.py index 1edec15..1c1b2f7 100644 --- a/Data Extractors/test_index_fetch_construction.py +++ b/Tests/Data Extractors/test_index_fetch_construction.py @@ -19,7 +19,9 @@ import pandas as pd import pytest -MODULE_PATH = Path(__file__).resolve().parent / "index_1m_5y_data_fetch_dhan_common.py" +# Tests/Data Extractors/ -> the repository root is two levels up. +_REPO_ROOT = Path(__file__).resolve().parents[2] +MODULE_PATH = _REPO_ROOT / "Data Extractors" / "index_1m_5y_data_fetch_dhan_common.py" spec = importlib.util.spec_from_file_location("index_1m_5y_data_fetch_dhan_common", MODULE_PATH) fetcher = importlib.util.module_from_spec(spec) sys.modules["index_1m_5y_data_fetch_dhan_common"] = fetcher diff --git a/Dependencies/Dhan API/test_dhan_execution.py b/Tests/Dependencies/Dhan API/test_dhan_execution.py similarity index 99% rename from Dependencies/Dhan API/test_dhan_execution.py rename to Tests/Dependencies/Dhan API/test_dhan_execution.py index bff32ee..d062090 100644 --- a/Dependencies/Dhan API/test_dhan_execution.py +++ b/Tests/Dependencies/Dhan API/test_dhan_execution.py @@ -15,7 +15,8 @@ import pytest -ROOT = Path(__file__).resolve().parents[2] +# Tests/Dependencies/Dhan API/ -> the repository root is three levels up. +ROOT = Path(__file__).resolve().parents[3] def _load_module() -> ModuleType: diff --git a/Dependencies/Flattrade API/test_flattrade_execution.py b/Tests/Dependencies/Flattrade API/test_flattrade_execution.py similarity index 95% rename from Dependencies/Flattrade API/test_flattrade_execution.py rename to Tests/Dependencies/Flattrade API/test_flattrade_execution.py index bd9241e..1e10ac6 100644 --- a/Dependencies/Flattrade API/test_flattrade_execution.py +++ b/Tests/Dependencies/Flattrade API/test_flattrade_execution.py @@ -18,7 +18,10 @@ import pandas as pd import pytest -sys.path.insert(0, str(Path(__file__).resolve().parent)) +# The adapter and its diagnostic import each other by bare name, so the SOURCE +# folder -- not this test folder -- goes on sys.path. +# Tests/Dependencies/Flattrade API/ -> repository root is three up. +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "Dependencies" / "Flattrade API")) import diagnose_flattrade_symbol as diag import flattrade_execution as fe diff --git a/Tests/Dependencies/conftest.py b/Tests/Dependencies/conftest.py new file mode 100644 index 0000000..b6401cd --- /dev/null +++ b/Tests/Dependencies/conftest.py @@ -0,0 +1,31 @@ +"""Pytest bootstrap for the shared ``Dependencies/`` suites. + +These tests used to live inside ``Dependencies/`` itself, where pytest's default +``prepend`` import mode put that folder on ``sys.path`` for free -- which is how +bare imports such as ``from check_env_config import audit`` and +``from order_splitting import split_order_quantity`` resolved. + +Now that the tests live under ``Tests/``, that no longer happens, so the SOURCE +folder is added here instead. Note it is the source ``Dependencies/`` directory +that goes on the path, NOT this test directory: the modules import each other by +bare name at runtime (the diagnostics run as standalone scripts), and the tests +must exercise that same resolution rather than a test-only arrangement. + +Only this one directory is inserted. Adding the repository root or a wider set +would let a test resolve an import production never performs, which could hide a +missing dependency. +""" + +from __future__ import annotations + +import os +import sys + +# Tests/Dependencies/ -> the repository root is two levels up. +_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +_SOURCE_DIR = os.path.join(_REPO_ROOT, "Dependencies") + +# Insert only when absent so repeated collection does not grow or reorder +# ``sys.path`` unnecessarily. +if _SOURCE_DIR not in sys.path: + sys.path.insert(0, _SOURCE_DIR) diff --git a/Dependencies/test_broker_contract.py b/Tests/Dependencies/test_broker_contract.py similarity index 99% rename from Dependencies/test_broker_contract.py rename to Tests/Dependencies/test_broker_contract.py index 95257b0..2cf0cb5 100644 --- a/Dependencies/test_broker_contract.py +++ b/Tests/Dependencies/test_broker_contract.py @@ -31,7 +31,8 @@ import pandas as pd import pytest -ROOT = Path(__file__).resolve().parents[1] +# Tests/Dependencies/ -> the repository root is two levels up. +ROOT = Path(__file__).resolve().parents[2] class _NeoApiTestDouble: diff --git a/Dependencies/test_check_env_config.py b/Tests/Dependencies/test_check_env_config.py similarity index 98% rename from Dependencies/test_check_env_config.py rename to Tests/Dependencies/test_check_env_config.py index 0693fa4..04ebdad 100644 --- a/Dependencies/test_check_env_config.py +++ b/Tests/Dependencies/test_check_env_config.py @@ -19,7 +19,8 @@ source_files, ) -REPO_ROOT = Path(__file__).resolve().parent.parent +# Tests/Dependencies/ -> the repository root is two levels up. +REPO_ROOT = Path(__file__).resolve().parents[2] def _make_repo(tmp_path: Path, *, template: str, live: str | None, code: str = "") -> Path: diff --git a/Dependencies/test_dhan_token_setup.py b/Tests/Dependencies/test_dhan_token_setup.py similarity index 100% rename from Dependencies/test_dhan_token_setup.py rename to Tests/Dependencies/test_dhan_token_setup.py diff --git a/Dependencies/test_diagnostic_preflight.py b/Tests/Dependencies/test_diagnostic_preflight.py similarity index 97% rename from Dependencies/test_diagnostic_preflight.py rename to Tests/Dependencies/test_diagnostic_preflight.py index 16fcf1f..759bb15 100644 --- a/Dependencies/test_diagnostic_preflight.py +++ b/Tests/Dependencies/test_diagnostic_preflight.py @@ -13,7 +13,8 @@ import pytest -ROOT = Path(__file__).resolve().parents[1] +# Tests/Dependencies/ -> the repository root is two levels up. +ROOT = Path(__file__).resolve().parents[2] def _load(name: str, relative_path: str) -> ModuleType: diff --git a/Dependencies/test_execution_ledger.py b/Tests/Dependencies/test_execution_ledger.py similarity index 100% rename from Dependencies/test_execution_ledger.py rename to Tests/Dependencies/test_execution_ledger.py diff --git a/Dependencies/test_next_open_entry.py b/Tests/Dependencies/test_next_open_entry.py similarity index 100% rename from Dependencies/test_next_open_entry.py rename to Tests/Dependencies/test_next_open_entry.py diff --git a/Dependencies/test_order_splitting.py b/Tests/Dependencies/test_order_splitting.py similarity index 100% rename from Dependencies/test_order_splitting.py rename to Tests/Dependencies/test_order_splitting.py diff --git a/Dependencies/test_repository_policy.py b/Tests/Dependencies/test_repository_policy.py similarity index 99% rename from Dependencies/test_repository_policy.py rename to Tests/Dependencies/test_repository_policy.py index b456f9c..4a12721 100644 --- a/Dependencies/test_repository_policy.py +++ b/Tests/Dependencies/test_repository_policy.py @@ -15,7 +15,8 @@ import yaml from check_env_config import audit, env_keys_read_by, source_files -ROOT = Path(__file__).resolve().parent.parent +# Tests/Dependencies/ -> the repository root is two levels up. +ROOT = Path(__file__).resolve().parents[2] def _requirement_lines(name: str) -> list[str]: diff --git a/Dependencies/test_risk_sizing.py b/Tests/Dependencies/test_risk_sizing.py similarity index 100% rename from Dependencies/test_risk_sizing.py rename to Tests/Dependencies/test_risk_sizing.py diff --git a/Dependencies/test_secret_redaction.py b/Tests/Dependencies/test_secret_redaction.py similarity index 100% rename from Dependencies/test_secret_redaction.py rename to Tests/Dependencies/test_secret_redaction.py diff --git a/Dependencies/test_startup_exposure.py b/Tests/Dependencies/test_startup_exposure.py similarity index 100% rename from Dependencies/test_startup_exposure.py rename to Tests/Dependencies/test_startup_exposure.py diff --git a/Dependencies/test_tick_bar_builder.py b/Tests/Dependencies/test_tick_bar_builder.py similarity index 100% rename from Dependencies/test_tick_bar_builder.py rename to Tests/Dependencies/test_tick_bar_builder.py diff --git a/Dependencies/test_trading_lifecycle.py b/Tests/Dependencies/test_trading_lifecycle.py similarity index 100% rename from Dependencies/test_trading_lifecycle.py rename to Tests/Dependencies/test_trading_lifecycle.py diff --git a/Signal Generators/CPR AI Agent/conftest.py b/Tests/Signal Generators/CPR AI Agent/conftest.py similarity index 66% rename from Signal Generators/CPR AI Agent/conftest.py rename to Tests/Signal Generators/CPR AI Agent/conftest.py index eee4165..e15c7ab 100644 --- a/Signal Generators/CPR AI Agent/conftest.py +++ b/Tests/Signal Generators/CPR AI Agent/conftest.py @@ -1,10 +1,13 @@ """Make the independent CPR AI modules importable despite their space-containing folder. Python cannot use a normal dotted import for a directory named ``CPR AI Agent``. -Pytest loads this file before collecting nearby tests, so it adds only this +Pytest loads this file before collecting nearby tests, so it adds only that independent agent directory to the import search path. Production uses the master's existing ``load_module`` helper instead. Keeping this adjustment in test configuration avoids packaging or renaming the folder merely for pytest. + +Note the path points at the SOURCE agent folder under ``Signal Generators/``, +not at this mirrored test folder -- the tests import the real modules. """ from __future__ import annotations @@ -12,13 +15,17 @@ import os import sys -_HERE = os.path.dirname(os.path.abspath(__file__)) +# Tests/Signal Generators/CPR AI Agent/ -> repository root is three up. +_REPO_ROOT = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +_AGENT_DIR = os.path.join(_REPO_ROOT, "Signal Generators", "CPR AI Agent") # Only this package root is inserted. Adding a repository-wide directory would # make tests pass through imports that production never uses and could hide a # missing dependency or accidental legacy-CPR coupling. # # Insert only when absent so repeated collection does not grow or reorder # ``sys.path`` unnecessarily. -for _path in (_HERE,): +for _path in (_AGENT_DIR,): if _path not in sys.path: sys.path.insert(0, _path) diff --git a/Signal Generators/CPR AI Agent/tests/test_cpr_ai_context.py b/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_context.py similarity index 100% rename from Signal Generators/CPR AI Agent/tests/test_cpr_ai_context.py rename to Tests/Signal Generators/CPR AI Agent/test_cpr_ai_context.py diff --git a/Signal Generators/CPR AI Agent/tests/test_cpr_ai_core.py b/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_core.py similarity index 100% rename from Signal Generators/CPR AI Agent/tests/test_cpr_ai_core.py rename to Tests/Signal Generators/CPR AI Agent/test_cpr_ai_core.py diff --git a/Signal Generators/CPR AI Agent/tests/test_cpr_ai_master_integration.py b/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_master_integration.py similarity index 75% rename from Signal Generators/CPR AI Agent/tests/test_cpr_ai_master_integration.py rename to Tests/Signal Generators/CPR AI Agent/test_cpr_ai_master_integration.py index 77705ce..c82c8e9 100644 --- a/Signal Generators/CPR AI Agent/tests/test_cpr_ai_master_integration.py +++ b/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_master_integration.py @@ -13,7 +13,11 @@ def test_task_two_runtime_does_not_import_the_legacy_cpr_strategy_package(): """Runtime modules must stay independent before the master wires execution.""" - agent_directory = Path(__file__).resolve().parents[1] + # Tests/Signal Generators/CPR AI Agent/ -> the SOURCE agent + # folder lives three levels up, under "Signal Generators/". + agent_directory = ( + Path(__file__).resolve().parents[3] / "Signal Generators" / "CPR AI Agent" + ) runtime_sources = "\n".join( path.read_text(encoding="utf-8") for path in agent_directory.glob("cpr_ai_*.py") diff --git a/Signal Generators/CPR AI Agent/tests/test_cpr_ai_runtime.py b/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_runtime.py similarity index 100% rename from Signal Generators/CPR AI Agent/tests/test_cpr_ai_runtime.py rename to Tests/Signal Generators/CPR AI Agent/test_cpr_ai_runtime.py diff --git a/Signal Generators/CPR Strategy/test_cpr_strategy_signal_generators.py b/Tests/Signal Generators/CPR Strategy/test_cpr_strategy_signal_generators.py similarity index 99% rename from Signal Generators/CPR Strategy/test_cpr_strategy_signal_generators.py rename to Tests/Signal Generators/CPR Strategy/test_cpr_strategy_signal_generators.py index d4ced56..2623918 100644 --- a/Signal Generators/CPR Strategy/test_cpr_strategy_signal_generators.py +++ b/Tests/Signal Generators/CPR Strategy/test_cpr_strategy_signal_generators.py @@ -5,7 +5,8 @@ import pandas as pd -STRATEGY_DIR = Path(__file__).resolve().parent +# Tests/Signal Generators/CPR Strategy/ -> repository root is three up. +STRATEGY_DIR = Path(__file__).resolve().parents[3] / "Signal Generators" / "CPR Strategy" LOGIC_PATH = STRATEGY_DIR / "cpr_strategy_logic.py" BACKTEST_PATH = STRATEGY_DIR / "Nifty CPR Strategy Backtest.py" ALGO3_PATH = STRATEGY_DIR / "Nifty CPR Algo 3 Signal Generator.py" diff --git a/Tests/Signal Generators/Regime Adaptive Strategy/conftest.py b/Tests/Signal Generators/Regime Adaptive Strategy/conftest.py new file mode 100644 index 0000000..9e5176c --- /dev/null +++ b/Tests/Signal Generators/Regime Adaptive Strategy/conftest.py @@ -0,0 +1,31 @@ +"""Pytest bootstrap for the Regime Adaptive strategy tests. + +The strategy's folder name contains spaces and its modules import each other by +bare name (`import regime_common`, etc.), so that folder goes on ``sys.path`` +before the tests import anything -- the same pattern as +``SL Hunting AI Agent/conftest.py``. + +The parent (`Signal Generators/`) is added too, because ``regime_common`` +re-exports the shared indicators from ``misc_strategy_common`` which lives one +level up. At runtime ``regime_common`` bootstraps that itself; under pytest the +import can happen through a different entry point, so it is done here as well. + +Both paths point at the SOURCE tree, not at this mirrored test folder -- the +tests import the real modules. +""" + +from __future__ import annotations + +import os +import sys + +# Tests/Signal Generators/Regime Adaptive Strategy/ -> repository root +# is three levels up. +_REPO_ROOT = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +_SIGNAL_GEN_DIR = os.path.join(_REPO_ROOT, "Signal Generators") +_STRATEGY_DIR = os.path.join(_SIGNAL_GEN_DIR, "Regime Adaptive Strategy") +for _path in (_SIGNAL_GEN_DIR, _STRATEGY_DIR): + if _path not in sys.path: + sys.path.insert(0, _path) diff --git a/Signal Generators/Regime Adaptive Strategy/test_regime_adaptive.py b/Tests/Signal Generators/Regime Adaptive Strategy/test_regime_adaptive.py similarity index 98% rename from Signal Generators/Regime Adaptive Strategy/test_regime_adaptive.py rename to Tests/Signal Generators/Regime Adaptive Strategy/test_regime_adaptive.py index 6b053a5..d42a6be 100644 --- a/Signal Generators/Regime Adaptive Strategy/test_regime_adaptive.py +++ b/Tests/Signal Generators/Regime Adaptive Strategy/test_regime_adaptive.py @@ -19,7 +19,10 @@ import pandas as pd import pytest -STRATEGY_DIR = Path(__file__).resolve().parent +# Tests/Signal Generators/Regime Adaptive Strategy/ -> repository root +# is three up. This points at the SOURCE strategy folder, whose modules import +# each other (and `misc_strategy_common` one level up) by bare name. +STRATEGY_DIR = Path(__file__).resolve().parents[3] / "Signal Generators" / "Regime Adaptive Strategy" # conftest.py already does this for a normal pytest run; repeated here so the # file also works when run directly (python -m pytest on this one path). for _path in (str(STRATEGY_DIR.parent), str(STRATEGY_DIR)): diff --git a/Tests/Signal Generators/SL Hunting AI Agent/conftest.py b/Tests/Signal Generators/SL Hunting AI Agent/conftest.py new file mode 100644 index 0000000..dc344af --- /dev/null +++ b/Tests/Signal Generators/SL Hunting AI Agent/conftest.py @@ -0,0 +1,23 @@ +"""Pytest bootstrap for the SL Hunting AI Agent. + +The agent's folder name contains spaces and its modules import each other by +bare name (`import sl_hunting_tools`, etc.), so that folder goes on ``sys.path`` +before the tests import anything. + +The path points at the SOURCE agent folder under ``Signal Generators/``, not at +this mirrored test folder -- the tests import the real modules. +""" + +from __future__ import annotations + +import os +import sys + +# Tests/Signal Generators/SL Hunting AI Agent/ -> repository root is +# three levels up. +_REPO_ROOT = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +_AGENT_DIR = os.path.join(_REPO_ROOT, "Signal Generators", "SL Hunting AI Agent") +if _AGENT_DIR not in sys.path: + sys.path.insert(0, _AGENT_DIR) diff --git a/Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_agent.py b/Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_agent.py similarity index 100% rename from Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_agent.py rename to Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_agent.py diff --git a/Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_indicators.py b/Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_indicators.py similarity index 100% rename from Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_indicators.py rename to Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_indicators.py diff --git a/Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_journal.py b/Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_journal.py similarity index 100% rename from Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_journal.py rename to Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_journal.py diff --git a/Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_lessons.py b/Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_lessons.py similarity index 100% rename from Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_lessons.py rename to Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_lessons.py diff --git a/Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_premarket.py b/Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_premarket.py similarity index 95% rename from Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_premarket.py rename to Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_premarket.py index 6c25b17..ec2cee4 100644 --- a/Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_premarket.py +++ b/Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_premarket.py @@ -8,6 +8,7 @@ from __future__ import annotations import json +import os from datetime import date from sl_hunting_premarket import ( @@ -20,6 +21,14 @@ TODAY = date(2026, 7, 28) +# The shipped premarket_note.json lives with the agent, not with these tests. +# Tests/Signal Generators/SL Hunting AI Agent/ -> repository root is +# three levels up. +_REPO_ROOT = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +) +AGENT_DIR = os.path.join(_REPO_ROOT, "Signal Generators", "SL Hunting AI Agent") + def _note(**overrides): payload = { @@ -162,7 +171,7 @@ def test_shipped_note_file_is_valid(): """The note committed alongside the agent must itself parse and validate.""" import os - here = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + here = AGENT_DIR shipped = os.path.join(here, "premarket_note.json") if not os.path.exists(shipped): return @@ -183,7 +192,7 @@ def test_shipped_note_targets_the_next_TRADING_day_not_the_next_calendar_day(): import os from datetime import date as _date - here = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + here = AGENT_DIR note = load_premarket_note(os.path.join(here, "premarket_note.json")) assert note is not None assert _date.fromisoformat(note.for_date).weekday() < 5, ( @@ -207,7 +216,7 @@ def test_shipped_note_matches_august_10_intraday_hunter_plan(): """ import os - here = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + here = AGENT_DIR shipped = os.path.join(here, "premarket_note.json") note = load_premarket_note(shipped) diff --git a/Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_runner.py b/Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_runner.py similarity index 100% rename from Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_runner.py rename to Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_runner.py diff --git a/Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_schema.py b/Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_schema.py similarity index 100% rename from Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_schema.py rename to Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_schema.py diff --git a/Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_v2.py b/Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_v2.py similarity index 100% rename from Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_v2.py rename to Tests/Signal Generators/SL Hunting AI Agent/test_sl_hunting_v2.py diff --git a/Signal Generators/Subhamoy Strategies/test_subhamoy_strategy_signal_generators.py b/Tests/Signal Generators/Subhamoy Strategies/test_subhamoy_strategy_signal_generators.py similarity index 97% rename from Signal Generators/Subhamoy Strategies/test_subhamoy_strategy_signal_generators.py rename to Tests/Signal Generators/Subhamoy Strategies/test_subhamoy_strategy_signal_generators.py index 0c5cd89..cc71cab 100644 --- a/Signal Generators/Subhamoy Strategies/test_subhamoy_strategy_signal_generators.py +++ b/Tests/Signal Generators/Subhamoy Strategies/test_subhamoy_strategy_signal_generators.py @@ -6,7 +6,10 @@ import pandas as pd -STRATEGY_DIR = Path(__file__).resolve().parent +# Tests/Signal Generators/Subhamoy Strategies/ -> repository root is +# three up. STRATEGY_DIR points at the SOURCE folder, so REPO_ROOT still +# derives from it exactly as before the tests moved. +STRATEGY_DIR = Path(__file__).resolve().parents[3] / "Signal Generators" / "Subhamoy Strategies" REPO_ROOT = STRATEGY_DIR.parents[1] BACKTEST_DIR = REPO_ROOT / "My Backtest Files (For Reference)" / "Subhamoy Strategies" diff --git a/Signal Generators/test_deterministic_strategy_safety.py b/Tests/Signal Generators/test_deterministic_strategy_safety.py similarity index 98% rename from Signal Generators/test_deterministic_strategy_safety.py rename to Tests/Signal Generators/test_deterministic_strategy_safety.py index 0ff73ce..609dcdc 100644 --- a/Signal Generators/test_deterministic_strategy_safety.py +++ b/Tests/Signal Generators/test_deterministic_strategy_safety.py @@ -15,7 +15,8 @@ import pandas as pd import pytest -GENERATOR_DIR = Path(__file__).resolve().parent +# Tests/Signal Generators/ -> the repository root is two levels up. +GENERATOR_DIR = Path(__file__).resolve().parents[2] / "Signal Generators" def _load_module(relative_path: str, module_name: str): diff --git a/Signal Generators/test_renko_bounds.py b/Tests/Signal Generators/test_renko_bounds.py similarity index 92% rename from Signal Generators/test_renko_bounds.py rename to Tests/Signal Generators/test_renko_bounds.py index 1a2d797..9f92768 100644 --- a/Signal Generators/test_renko_bounds.py +++ b/Tests/Signal Generators/test_renko_bounds.py @@ -9,7 +9,8 @@ import pandas as pd import pytest -GEN_DIR = Path(__file__).resolve().parent +# Tests/Signal Generators/ -> the repository root is two levels up. +GEN_DIR = Path(__file__).resolve().parents[2] / "Signal Generators" @pytest.mark.parametrize( diff --git a/Signal Generators/test_trading_bot_ports.py b/Tests/Signal Generators/test_trading_bot_ports.py similarity index 99% rename from Signal Generators/test_trading_bot_ports.py rename to Tests/Signal Generators/test_trading_bot_ports.py index 6d12a39..5083d81 100644 --- a/Signal Generators/test_trading_bot_ports.py +++ b/Tests/Signal Generators/test_trading_bot_ports.py @@ -47,7 +47,8 @@ import pandas as pd import pytest -GEN_DIR = Path(__file__).resolve().parent +# Tests/Signal Generators/ -> the repository root is two levels up. +GEN_DIR = Path(__file__).resolve().parents[2] / "Signal Generators" # Ports that live in a subfolder import their siblings by bare name, so that # folder has to be importable before `_load_port` executes them. (The master's diff --git a/test_market_data_health.py b/Tests/test_market_data_health.py similarity index 100% rename from test_market_data_health.py rename to Tests/test_market_data_health.py diff --git a/test_nifty_multi_strategy_master.py b/Tests/test_nifty_multi_strategy_master.py similarity index 99% rename from test_nifty_multi_strategy_master.py rename to Tests/test_nifty_multi_strategy_master.py index 2f787bd..b5f4700 100644 --- a/test_nifty_multi_strategy_master.py +++ b/Tests/test_nifty_multi_strategy_master.py @@ -27,7 +27,11 @@ # Python usually expects file names without spaces (e.g. "my_script.py"). # Since the original strategy file has spaces in its name, we must use # 'importlib' to manually load the file as a module so we can test its contents. -file_path = Path(__file__).parent / "Nifty Multi Strategy Front Test - Master File.py" +# +# This suite lives under "Tests/", so every path below is anchored on the +# REPOSITORY ROOT rather than on this file's own folder. +REPO_ROOT = Path(__file__).resolve().parents[1] +file_path = REPO_ROOT / "Nifty Multi Strategy Front Test - Master File.py" spec = importlib.util.spec_from_file_location("master_file", file_path) master_file = importlib.util.module_from_spec(spec) sys.modules["master_file"] = master_file @@ -76,7 +80,7 @@ def tearDownModule(): # The Flattrade helper is loaded separately so its low-level REST behaviour can # be tested without starting the master runner or making any network requests. -flattrade_file_path = Path(__file__).parent / "Dependencies" / "Flattrade API" / "flattrade_execution.py" +flattrade_file_path = REPO_ROOT / "Dependencies" / "Flattrade API" / "flattrade_execution.py" flattrade_module = None if flattrade_file_path.is_file(): flattrade_spec = importlib.util.spec_from_file_location( @@ -87,7 +91,7 @@ def tearDownModule(): flattrade_spec.loader.exec_module(flattrade_module) flattrade_diagnostic_path = ( - Path(__file__).parent + REPO_ROOT / "Dependencies" / "Flattrade API" / "diagnose_flattrade_symbol.py" diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..0d84549 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,74 @@ +# Architecture documentation + +This folder is the committed design record for the NIFTY multi-strategy trading +system. It answers "how is this thing built, and why is it built that way" +without requiring a reader to open the 17k-line master runner first. + +It is **not** operator documentation. For setup, credentials, `.env` keys and +day-to-day commands, read the root [`README.md`](../README.md) and each +component folder's own `Readme.md`. + +## Layout + +| Folder | What lives there | When to read it | +|---|---|---| +| [`hld/`](hld/) | One high-level design covering the whole repository | First. Start with [`hld/system-overview.md`](hld/system-overview.md). | +| [`lld/`](lld/) | One low-level design per component | When changing that component. | +| [`adr/`](adr/) | Architecture Decision Records — the *why* behind decisions already baked in | Before proposing to reverse one. | +| `superpowers/` | Per-session agent plans and specs. **Untracked** (`.gitignore`). | Never — it is a working scratchpad, not product documentation. | + +## Reading order for a new contributor + +1. [`hld/system-overview.md`](hld/system-overview.md) — the process, the threads, the data flow. +2. [`lld/risk-and-safety.md`](lld/risk-and-safety.md) — this is live-money code; read the safety model before anything else. +3. [`lld/master-runner.md`](lld/master-runner.md) — startup gates, supervision, shutdown. +4. Then the LLD for whichever component you are touching. + +## Low-level designs + +| Component | Document | +|---|---| +| Process lifecycle, thread supervision, startup/shutdown | [`lld/master-runner.md`](lld/master-runner.md) | +| Market data: REST poller, websocket producer, shared store, health gates | [`lld/market-data.md`](lld/market-data.md) | +| Strategy worker family and the signal-generator factory | [`lld/strategy-workers.md`](lld/strategy-workers.md) | +| Option contract resolution and order placement | [`lld/execution-and-brokers.md`](lld/execution-and-brokers.md) | +| Risk, sizing, and the live-trading safety model | [`lld/risk-and-safety.md`](lld/risk-and-safety.md) | +| SL Hunting AI Agent (optional, Claude) | [`lld/sl-hunting-ai-agent.md`](lld/sl-hunting-ai-agent.md) | +| CPR Codex AI Agent (optional, Codex) | [`lld/cpr-codex-ai-agent.md`](lld/cpr-codex-ai-agent.md) | +| Regime Adaptive router | [`lld/regime-adaptive.md`](lld/regime-adaptive.md) | +| Configuration and drift detection | [`lld/configuration.md`](lld/configuration.md) | +| Telegram alerts, EOD P&L sheet, logging | [`lld/reporting-and-observability.md`](lld/reporting-and-observability.md) | +| Data extraction and backtesting | [`lld/data-and-backtesting.md`](lld/data-and-backtesting.md) | +| Test architecture, coverage budgets, CI | [`lld/testing-and-ci.md`](lld/testing-and-ci.md) | + +## Decision records + +| ADR | Decision | +|---|---| +| [0001](adr/0001-single-process-thread-per-strategy.md) | One process, one thread per strategy | +| [0002](adr/0002-broker-agnostic-execution-contract.md) | Broker-agnostic execution contract, fail-closed broker selection | +| [0003](adr/0003-acknowledgement-is-not-a-fill.md) | Typed order outcomes and a quantity-bearing ledger | +| [0004](adr/0004-paper-by-default-double-gate.md) | Paper by default behind a two-flag live gate | +| [0005](adr/0005-rest-vs-websocket-market-data.md) | REST polling as default, websocket as opt-in producer | +| [0006](adr/0006-per-strategy-size-multiplier.md) | Per-strategy size multiplier, deliberately not global | +| [0007](adr/0007-llm-agents-as-opt-in-workers.md) | LLM agents as opt-in workers with host-owned gates | +| [0008](adr/0008-single-env-as-config-source.md) | A single `.env` as the only configuration source | +| [0009](adr/0009-importlib-loading-for-spaced-filenames.md) | `importlib` loading instead of renaming spaced files | +| [0010](adr/0010-tests-in-a-mirrored-tests-tree.md) | Tests consolidated into a mirrored top-level `Tests/` tree | +| [0011](adr/0011-committed-docs-untracked-superpowers.md) | Committed `docs/` set; Superpowers workspace untracked | + +## Keeping these documents honest + +Architecture documents that drift are worse than no documents. Two habits keep +this set usable: + +- **Change the LLD in the same commit as the code.** If a pull request changes + how a component behaves, its LLD is part of that pull request. +- **Never restate a number the code owns.** Prefer "the default is defined in + `Dependencies/env.example`" to copying the value here. Where a specific value + genuinely matters to the design (the 10-second broker deadline, the 90% + coverage budget), it is stated once with the file that owns it named next to it. + +ADRs are append-only history. When a decision changes, add a new ADR that +supersedes the old one and mark the old one `Superseded`; do not edit the +original's decision. diff --git a/docs/adr/0001-single-process-thread-per-strategy.md b/docs/adr/0001-single-process-thread-per-strategy.md new file mode 100644 index 0000000..2b96de1 --- /dev/null +++ b/docs/adr/0001-single-process-thread-per-strategy.md @@ -0,0 +1,109 @@ +# ADR-0001: One process, one thread per strategy + +**Status:** Accepted +**Date:** 2026-08-10 (retrospective — records a decision already in force) +**Deciders:** repository owner + +## Context + +The system runs roughly 27 core strategies plus up to two optional AI agents +concurrently against one market-data feed, executing through one broker account. + +The workload's actual shape: + +- Strategies act on **completed 1-minute (or resampled 5-minute) candles**. A + 2–5 second data lag sits inside the decision granularity. +- Per-bar CPU work is small pandas operations on frames of a few hundred rows. +- Almost all wall-clock time is spent in HTTP calls and sleeps. +- One person maintains the whole thing. +- One trading session per day, on one machine. + +## Decision + +Run everything in **one Python process** with **one `threading.Thread` per +strategy**, one producer thread for market data, and one worker thread for +Telegram. Share exactly two mutable objects, both lock-guarded: +`SharedMarketDataStore` and the broker session behind `ExecutionSafetyCoordinator`. + +## Options considered + +### Option A: One process, thread per strategy (chosen) + +| Dimension | Assessment | +|---|---| +| Complexity | Low | +| Cost | Nil | +| Scalability | Bounded by one machine — irrelevant at this size | +| Team familiarity | Total | + +**Pros:** the concurrency review surface is two objects; one broker session and +one lock make order serialization trivial; one log file; deployment is +`python algo.py run`; a shared in-memory feed needs no serialization. +**Cons:** no horizontal scale; one crash stops every strategy; the GIL caps +CPU-bound work; a leaked exception in one thread can affect the process. + +### Option B: Process per strategy + +| Dimension | Assessment | +|---|---| +| Complexity | High | +| Cost | Nil in money, high in code | +| Scalability | Real, and unneeded | +| Team familiarity | Moderate | + +**Pros:** true isolation and parallelism; one strategy cannot take down another. +**Cons:** the broker session must be shared across processes, which turns the +simplest safety property in the system (one lock, one session, serialized +orders) into a distributed-locking problem. The market-data store would need IPC +or a broker. **The exposure ledger would have to become cross-process state** — +and that ledger is the thing that must never be wrong. + +### Option C: Single-threaded asyncio + +| Dimension | Assessment | +|---|---| +| Complexity | Medium | +| Cost | Nil | +| Scalability | Fine | +| Team familiarity | Lower | + +**Pros:** no locks; natural fit for an I/O-bound workload; cheap concurrency. +**Cons:** every broker SDK in use is synchronous and would need thread-pool +wrapping anyway, reintroducing the thread boundary at the least convenient +place. One blocking call in one strategy stalls **all** of them — the exact +failure the 10-second deadline exists to contain. The strategy code would have +to be rewritten in colour. + +## Trade-off analysis + +The deciding factor is not performance — no option is anywhere near a resource +limit. It is **how many places can be wrong about exposure.** + +Option A has one: an in-process ledger guarded by one lock. Option B has as many +as there are processes, plus the coordination between them. Option C has one, +but reintroduces threads at the broker boundary while making a single blocking +SDK call a process-wide stall. + +For live-money code maintained by one person, minimising the number of places +that can disagree about "how much am I holding?" beats every other consideration. + +## Consequences + +**Easier:** reasoning about concurrency (two objects); serializing orders; +deploying; debugging from one log; adding a strategy (add a thread). + +**Harder:** surviving a crash (everything stops together); CPU-bound work +(GIL); running strategies on separate machines (not possible without redesign). + +**To revisit when:** the runner must survive a mid-session restart unattended +(persist the ledger first — see the HLD §9), a second operator joins, or a +strategy genuinely needs sub-second reaction (that is a different system, not an +increment of this one). + +## Action items + +- [x] One lock-guarded `SharedMarketDataStore`. +- [x] One lock-guarded broker session behind `ExecutionSafetyCoordinator`. +- [x] Ten-second deadline **including** lock wait, so one hung call cannot stall + the other workers ([ADR-0002](0002-broker-agnostic-execution-contract.md)). +- [ ] Persist the execution ledger if unattended running is ever wanted. diff --git a/docs/adr/0002-broker-agnostic-execution-contract.md b/docs/adr/0002-broker-agnostic-execution-contract.md new file mode 100644 index 0000000..ce13b5f --- /dev/null +++ b/docs/adr/0002-broker-agnostic-execution-contract.md @@ -0,0 +1,109 @@ +# ADR-0002: Broker-agnostic execution contract, fail-closed broker selection + +**Status:** Accepted +**Date:** 2026-08-10 (retrospective) +**Deciders:** repository owner + +## Context + +Four brokers are usable for live execution: Kotak Neo, Shoonya (Finvasia), +Flattrade Pi v2, and Dhan. They differ in authentication (OAuth, browser token, +TOTP), symbol format, order-status vocabulary, error shapes, and rate limits. + +Two forces: + +1. Brokers are swapped for real reasons — Shoonya's legacy QuickAuth endpoint is + being decommissioned; Kotak rejects MIS orders on next-month contracts. The + system must not require a rewrite when that happens. +2. The runner's order path is the most safety-critical code in the repository. + It must be tested **once**, not once per broker. + +## Decision + +Define one broker-agnostic surface that every adapter implements, and route all +real orders through a single generic `execution_client`. The runner never +imports a broker module. + +``` +ensure_logged_in · is_logged_in · logout +preload_scrip_master · resolve_option_symbol +place_market_order · get_order_status · cancel_order +list_open_orders · list_open_positions +recover_after_reconciliation · extract_order_id +``` + +`LIVE_BROKER` selects the adapter. **An unrecognised value fails closed:** live +trading is disabled and the session runs on paper. It never falls back to a +default broker. + +Shared result types live in `Dependencies/broker_contract.py` +([ADR-0003](0003-acknowledgement-is-not-a-fill.md)). + +## Options considered + +### Option A: One contract, adapters contain the quirks (chosen) + +**Pros:** one tested order path; swapping brokers is a config change; each +broker's weirdness is isolated where it can be unit-tested; a new broker is +additive. +**Cons:** the contract must be the *intersection* of what four brokers can do; +an adapter that cannot honour a method must fail explicitly rather than +half-implement it; quirks that leak are hard to spot in review. + +### Option B: Broker-specific call sites in the runner + +**Pros:** no abstraction to design; each broker uses its most natural API. +**Cons:** the safety logic (fill confirmation, ledger updates, deadlines) would +be duplicated four times and would drift. This is disqualifying: four copies of +"is this a fill?" is four chances to get it wrong. + +### Option C: A third-party unified broker library + +**Pros:** no adapter code to maintain. +**Cons:** the failure semantics this system depends on — the difference between +a transport error and a rejection, cumulative-vs-absolute fill quantities — are +exactly what such libraries normalise away. Adding a dependency at the +live-order boundary also adds an unaudited party to the most sensitive path. + +## Trade-off analysis + +The contract's cost is real: it is the intersection of four brokers, so it +cannot expose anything one of them lacks. The benefit is that the roughly +30 workers share **one** execution path, and every safety rule is written and +tested exactly once. + +The fail-closed selection is the deciding detail. A typo in `LIVE_BROKER` +silently routing to a default broker would place real orders through an account +the operator did not intend. Refusing to trade live is always the cheaper error. + +## Consequences + +**Easier:** swapping brokers; testing execution once; adding a broker; reasoning +about the order path. + +**Harder:** using a broker-specific feature (it must be contained in the adapter +or added to the contract for all four); a broker whose model genuinely differs +would strain the intersection. + +**To revisit when:** a broker requires a capability the contract cannot express +(streaming order updates, bracket orders, basket submission) — extend the +contract deliberately rather than leaking the capability upward. + +## Consequences already realised + +- Dhan's transport errors are shape-identical to rejections; the adapter + contains that (see [ADR-0003](0003-acknowledgement-is-not-a-fill.md)). +- Kotak's pandas/requests pins conflict with the audited core set, so its + environment is validated in a separate CI job. +- Shoonya's `NorenApi` is vendored, excluded from lint/mypy/Bandit, and every + call has an explicit timeout. + +## Action items + +- [x] `Dependencies/broker_contract.py` holds the shared types and Protocol. +- [x] `_select_execution_client` fails closed on an unknown name. +- [x] Ten-second deadline per call, **including** shared lock and rate-limit wait. +- [x] 80% branch-coverage budget per adapter, enforced by + `scripts/check_coverage_thresholds.py`; a missing module is a failure. +- [x] Adding a broker requires a `BROKER_THRESHOLDS` row — treated as part of + the work, after the Dhan adapter first landed without one. diff --git a/docs/adr/0003-acknowledgement-is-not-a-fill.md b/docs/adr/0003-acknowledgement-is-not-a-fill.md new file mode 100644 index 0000000..7d976ec --- /dev/null +++ b/docs/adr/0003-acknowledgement-is-not-a-fill.md @@ -0,0 +1,98 @@ +# ADR-0003: An acknowledgement is not a fill — typed outcomes and a quantity ledger + +**Status:** Accepted +**Date:** 2026-08-10 (retrospective) +**Deciders:** repository owner + +## Context + +Broker APIs answer an order with an acknowledgement, an order ID, a truthy dict, +or an error. None of those is a fill. The states that actually occur: + +- The order fills completely. +- It fills partially, and the rest is pending or cancelled. +- It is rejected outright. +- **The response is lost** — the order may or may not exist at the broker. +- The status is reported cumulatively, late, out of order, or repeated. +- **Dhan's SDK returns `{'status': 'failure', 'remarks': str(exc)}` for + transport errors** — byte-identical in shape to a genuine rejection. + +The consequence of getting this wrong is not a bad trade. It is an untracked +live position. + +There is a second, subtler trap specific to this system: a rejected live **entry +with zero fill** is deliberately treated as a paper fallback. So anything that +wrongly looks like a zero-fill rejection causes the runner to trade on paper +while the operator believes it is live. + +## Decision + +1. Normalize every broker reply into exactly **four** outcomes — + `FILLED`, `PARTIAL`, `REJECTED`, `UNKNOWN` — carried by an `OrderResult` + that also holds the **filled quantity**. +2. Record every leg in `Dependencies/execution_ledger.py` **before** submission, + and apply broker fill reports as **deltas against cumulative totals**. +3. Never derive `REJECTED` from anything that could also be a transport error. + In the Dhan adapter: a `dict` `remarks` means the server refused; a `str` + means the outcome is indeterminate → `UNKNOWN`. +4. Treat `PARTIAL` and `UNKNOWN` identically at the call site: exposure may + exist — freeze new live entries, keep exits available, reconcile. + +## Options considered + +### Option A: Typed outcomes + quantity-bearing ledger (chosen) + +**Pros:** callers cannot mistake an ack for a fill — there is no boolean to +misread; `UNKNOWN` is representable, so ambiguity is handled instead of +collapsed; cumulative-delta application survives late, repeated and out-of-order +reports; the ledger's pre-submission record survives a lost response. +**Cons:** four states for every caller to handle; `UNKNOWN` ultimately needs an +operator; more code than `if result: ...`. + +### Option B: Boolean success + separate quantity lookup + +**Pros:** simplest call sites. +**Cons:** `False` conflates "refused" with "unknown", which are opposite +instructions — one means fall back to paper, the other means assume exposure. +Collapsing them is precisely the bug this ADR exists to prevent. + +### Option C: Trust order status polling alone, no local ledger + +**Pros:** the broker is the source of truth, so no duplicate state. +**Cons:** a lost placement response leaves nothing to poll *for*. Without a +pre-submission record there is no order ID and no evidence the attempt happened. +Dhan's `correlationId` recovery exists exactly because this case is real. + +## Trade-off analysis + +Option B is materially simpler and is wrong in the one scenario that costs the +most. The extra states in Option A are not incidental complexity — they are the +problem's actual shape. Every simplification here works by discarding the +distinction between "refused" and "unknown", and those demand opposite actions. + +The ledger (Option A over C) costs duplicated state, and duplicated state can +drift. That is accepted because the alternative — having no record of an +attempt whose response was lost — has no recovery path at all. + +## Consequences + +**Easier:** answering "how much am I holding?" at any moment; recovering an +order whose response was lost (Dhan `correlationId`); reasoning about partial +fills; auditing the live path. + +**Harder:** every call site must handle four states; `UNKNOWN` blocks new live +entries until reconciled, which will occasionally stop a session that was +actually fine; local state must be kept consistent with the broker. + +**To revisit when:** a broker offers a reliable streaming order-update channel +that removes the lost-response case — the ledger would still be wanted, but +`UNKNOWN` could become rarer. + +## Action items + +- [x] `Dependencies/broker_contract.py` — four outcomes, quantity carried. +- [x] `Dependencies/execution_ledger.py` — pre-submission record, cumulative deltas. +- [x] Dhan adapter never derives `REJECTED` from the placement envelope. +- [x] Dhan `order_tag` sent as `correlationId` for lost-response recovery. +- [x] 90% branch-coverage budget on both modules. +- [x] Paper fallback only on `REJECTED` **with zero filled quantity**. diff --git a/docs/adr/0004-paper-by-default-double-gate.md b/docs/adr/0004-paper-by-default-double-gate.md new file mode 100644 index 0000000..250c7ae --- /dev/null +++ b/docs/adr/0004-paper-by-default-double-gate.md @@ -0,0 +1,109 @@ +# ADR-0004: Paper by default, behind a two-flag live gate + +**Status:** Accepted +**Date:** 2026-08-10 (retrospective) +**Deciders:** repository owner + +## Context + +The same process runs strategies that are being evaluated on paper and +strategies that are trading real money, at the same time, in the same session. +The operator routinely edits `.env` between sessions. + +The failure to prevent: **a strategy trading live that the operator did not +intend to be live.** Causes include a leftover flag from yesterday, a typo, a +copy-pasted block, or a new strategy inheriting a global setting. + +## Decision + +A strategy places a real order only when **both** flags are true: + +``` +LIVE_TRADING_ENABLED = true # process-wide kill-switch, default FALSE +_LIVE_TRADING = true # per strategy, default FALSE +``` + +Plus a recognised `LIVE_BROKER`; an unknown value disables live entirely +([ADR-0002](0002-broker-agnostic-execution-contract.md)). + +The paper gate is deliberately **asymmetric**: + +| | Global switch | Per-strategy switch | Default | +|---|---|---|---| +| **Live** | required | required | off | +| **Virtual (paper)** | *none, by design* | `_VIRTUAL_TRADING` | on | + +`_VIRTUAL_TRADING=false` stops the worker thread from starting at all, +so the strategy does neither paper nor live. + +## Options considered + +### Option A: Two flags for live, one for paper, no global paper switch (chosen) + +**Pros:** no single edit can put a strategy live; the global switch is a genuine +kill-switch that stops everything in one line; a new strategy is paper by +default because both flags default false; the safe default differs correctly per +mode — "run everything" for paper, "run nothing" for live. +**Cons:** two places to change to go live; it is possible to believe you are +live when only one flag is set. + +### Option B: One global `LIVE_TRADING_ENABLED` only + +**Pros:** one switch; nothing to forget. +**Cons:** flipping it puts **every** strategy live simultaneously, including +ones added since the last review. Unacceptable — going live must be per +strategy, deliberately. + +### Option C: Per-strategy flag only, no global switch + +**Pros:** precise; no redundancy. +**Cons:** no single action stops all live trading. During an incident the +operator would edit ~27 lines. A kill-switch that takes 27 edits is not a +kill-switch. + +### Option D: A `--live` CLI flag instead of config + +**Pros:** explicit at launch; cannot be left over from yesterday. +**Cons:** conflicts with [ADR-0008](0008-single-env-as-config-source.md) (one +config source) and makes live-ness invisible to `check-env`. It also moves the +decision to the moment of *starting*, when the operator is least likely to be +reviewing per-strategy risk. + +## Trade-off analysis + +Option A's redundancy is the point. Options B and C each optimise away one flag +and lose a distinct property — B loses per-strategy control, C loses the +kill-switch. Both properties are needed, so both flags stay. + +The asymmetry with paper is worth stating plainly because it looks inconsistent +until you name the safe default: for paper, running is safe and the risk is +*forgetting to enable* a strategy you meant to evaluate; for live, not running +is safe and the risk is *enabling* one you did not mean to. Defaults follow the +safe direction in each case, so they point in opposite directions. + +The residual risk Option A accepts — thinking you are live when only one flag is +set — is a *missed trade*, not a *lost trade*. That is the correct direction for +the error to fall. + +## Consequences + +**Easier:** going live one strategy at a time; stopping all live trading in one +edit; adding a strategy safely (paper by default); paper-validating a change +before it risks money. + +**Harder:** going live (two edits, deliberately); noticing that a strategy is +*not* live when you meant it to be. + +**To revisit when:** the operator wants live trading on by default for a proven +core set — that would be a real change of posture and deserves its own ADR, not +a quiet default flip. + +## Action items + +- [x] Both flags default false; `_configure_startup_live_trading` enforces the + conjunction. +- [x] `_live_config_errors` blocks a strategy from live on malformed size config + (paper falls back to 1) — see [ADR-0006](0006-per-strategy-size-multiplier.md). +- [x] Startup exposure audit must be clean before live workers start. +- [x] Telegram and the EOD sheet label rows PAPER / LIVE / MIXED so a + mislabelled run is visible after the fact. diff --git a/docs/adr/0005-rest-vs-websocket-market-data.md b/docs/adr/0005-rest-vs-websocket-market-data.md new file mode 100644 index 0000000..accaebe --- /dev/null +++ b/docs/adr/0005-rest-vs-websocket-market-data.md @@ -0,0 +1,116 @@ +# ADR-0005: REST polling as the default, websocket as an opt-in producer + +**Status:** Accepted +**Date:** 2026-08-10 (retrospective — records PR #79) +**Deciders:** repository owner + +## Context + +The REST producer pulls the full OHLC window every 2–5 seconds, for NIFTY spot +plus every subscribed option leg. With multi-leg baskets (hedged pairs, the +Delta-0.2 four-leg spread, strangle legs, the SL Hunting BankNIFTY mirror) that +is a lot of repeated requests for data that mostly has not changed. + +Dhan offers a marketfeed websocket under its **paid Data API** subscription. + +Constraints: + +- Strategies act on completed candles, so this is **not** a latency problem. The + motivation is API load and quota, not speed. +- Every backtest was run on official exchange candles. Live bars must remain the + same bars, or results stop being comparable — and nothing would fail loudly if + they drifted. +- A trading session cannot be gambled on a socket that may drop. + +## Decision + +Support both producers behind one contract, selected by `MARKET_DATA_SOURCE`: + +- **`REST` (default)** — `CentralMarketDataFetcher`. Any unrecognised value also + yields REST (fail closed). +- **`WEBSOCKET` (opt-in)** — `WebSocketMarketDataFetcher`. Ticks build the + forming minute live; REST is retained for warmup history and reconnect + gap-backfill; and **once per minute the completed candles are trued up against + Dhan's official REST candles, with official always winning** and any divergence + logged. + +Both write the same shape into `SharedMarketDataStore`, so no strategy knows +which producer is running. Pure tick→bar logic lives in +`Dependencies/tick_bar_builder.py`. + +Rollback is `MARKET_DATA_SOURCE=REST` plus a restart. No state migration. + +## Options considered + +### Option A: REST only + +| Dimension | Assessment | +|---|---| +| Complexity | Low | +| Cost | No paid subscription | +| Correctness | Perfect — every bar is the official candle | + +**Pros:** no reconnect logic, no partial-bar state, nothing to true up. +**Cons:** one full-window pull every few seconds; LTPs only as fresh as the poll. + +### Option B: Websocket only + +| Dimension | Assessment | +|---|---| +| Complexity | High | +| Cost | Paid Data API | +| Correctness | Depends entirely on tick handling | + +**Pros:** lowest API load; real-time LTPs; the forming minute visible live. +**Cons:** a dropped socket has no fallback within the session; tick-built candles +can silently diverge from official ones; requires a paid subscription to run at +all — including in a paper session. + +### Option C: Both, REST default, websocket opt-in with a true-up (chosen) + +**Pros:** the load win is available without betting a session on the socket; +the true-up bounds divergence to at most one minute and always resolves in +favour of the official candle; rollback is a flag; unrecognised values fail +closed to the safe producer; the pure helpers are unit-testable without a feed. +**Cons:** two producers must be kept behaviourally identical — real, ongoing +maintenance; the websocket path has more moving parts (pump thread, supervisor, +warmup, backfill, true-up). + +## Trade-off analysis + +The API-load saving (from one full-window pull every 2–5 s to roughly one per +minute) is worth having, but not at the cost of the shared-candle invariant that +links backtests, paper and live. + +The true-up is what makes Option C safe rather than merely convenient: without +it, tick-built candles would slowly stop being the candles the strategies were +validated on, and **no individual test would fail**. Making "official always +wins" a blunt, unconditional rule removes any judgement from the reconciliation. + +Keeping REST as the default means the paid subscription is an optimisation, not +a dependency — the system runs without it. + +## Consequences + +**Easier:** staying inside Dhan's rate limits with many subscribed legs; +real-time LTPs for multi-leg baskets; adding a producer later (the contract +exists). + +**Harder:** two producers to maintain; the health gates needed a tick-aware +twist — a quiet-but-subscribed leg is fresh **only while the socket is +demonstrably alive**, because otherwise a dead socket looks exactly like a quiet +strike. + +**To revisit when:** the true-up starts logging routine divergence (that is a +signal the tick path is wrong, not that the true-up should be relaxed), or if +mid-session failover from websocket to REST becomes worth building. + +## Action items + +- [x] `_select_market_data_fetcher_class()` fails closed to REST. +- [x] Pure tick logic in `Dependencies/tick_bar_builder.py`, 90% coverage budget. +- [x] Per-minute true-up against official candles; divergence logged. +- [x] Tick-aware freshness for quiet subscribed legs. +- [x] Operator gate: ≥2 clean paper sessions before enabling live on the + websocket producer — CI never opens a real socket, so a green build proves + nothing about the transport. diff --git a/docs/adr/0006-per-strategy-size-multiplier.md b/docs/adr/0006-per-strategy-size-multiplier.md new file mode 100644 index 0000000..f72e61f --- /dev/null +++ b/docs/adr/0006-per-strategy-size-multiplier.md @@ -0,0 +1,115 @@ +# ADR-0006: A per-strategy size multiplier, deliberately not global + +**Status:** Accepted +**Date:** 2026-08-10 (retrospective) +**Deciders:** repository owner + +## Context + +Each strategy's size is described by four knobs that must stay consistent: +`_LOTS`, `_MAX_LOTS`, `_RISK_BUDGET`, and the absolute +`_MAX_LOSS`. + +As the account grows, size should grow. Editing four numbers per strategy by +hand is error-prone in a specific, expensive way: **raising the lot cap while +forgetting the budget** produces a position larger than the risk model allows, +and nothing complains. + +## Decision + +Add `_SIZE_MULTIPLIER` — one whole number per strategy, default `1`, +range 1–25 (capped by `MAX_SIZE_MULTIPLIER`) — that scales that strategy's whole +size/risk set together. + +Applied at **env-read time** via `_scaled_int` / `_scaled_float` / +`_strategy_size_multiplier`, so `Dependencies/risk_sizing.py` is untouched and +scaled values flow through sizing, the kill-switch, Telegram and the Sheet +unchanged. + +Three properties are deliberate: + +1. **Per strategy only. No global switch.** +2. **Applies to paper and live alike.** +3. **Malformed values (`0`, `2.5`, `30`, `"two"`) fall back to 1 for paper and + BLOCK that strategy from live** via `_live_config_errors`. + +Two knobs are **not** scaled, because their totals already inherit the +multiplier and scaling them would square it: + +- `_MAX_LOSS_PER_LOT` (Delta20) — multiplied by scaled lots downstream. +- `_STARTING_CAPITAL` / `_DAILY_MAX_LOSS_PCT` — their *product* carries it. + +## Options considered + +### Option A: Per-strategy multiplier applied at read time (chosen) + +**Pros:** one number per strategy instead of four kept consistent by hand; the +sizing module never learns about it; a typo can only affect one strategy; +scaling is visible in every downstream report because the scaled values *are* +the values. +**Cons:** the double-scaling trap is real and needs the two documented +exceptions; a reader of `.env` must remember the multiplier when interpreting +the other four numbers. + +### Option B: A global `SIZE_MULTIPLIER` + +**Pros:** one number scales the whole book; matches "the account grew". +**Cons:** **one typo enlarges every enabled strategy at once**, including ones +the operator has not looked at in weeks. The blast radius is the entire roster. +Rejected on that alone. + +### Option C: Apply the multiplier inside `risk_sizing.py` + +**Pros:** one place; sizing owns sizing. +**Cons:** the kill-switch, Telegram messages and the Sheet would keep showing +*unscaled* numbers, so reports would disagree with reality. It would also +require passing the multiplier through every call site. + +### Option D: Just edit the four numbers + +**Pros:** nothing to build; total transparency. +**Cons:** the exact failure described in Context — an inconsistent set that +allows a position larger than the risk model, with no error. + +## Trade-off analysis + +Option B is what most systems do and is the one clearly wrong answer here: a +global multiplier turns a single-character mistake into a book-wide risk change. +The cost of Option A over B — editing N numbers to scale the whole book — is +paid rarely and deliberately, which is exactly when the operator should be +thinking about risk. + +Applying it at read time (A over C) makes the scaled values the *only* values +anything downstream ever sees, which is what keeps the kill-switch, the alerts +and the P&L sheet honest. + +Paper-and-live parity matters because it lets an enlarged size be validated on +paper first — the same reason the whole system is paper-by-default. + +## Consequences + +**Easier:** growing position size with the account; paper-validating a size +change; keeping the four size knobs consistent. + +**Harder:** reading `.env` (the effective size is a product, not a literal); +reasoning about the SL Hunting basket, where the BankNIFTY mirror already +roughly doubles risk — with multiplier `M` the basket sits near **2 × M** times +the single-leg budget. + +Two known second-order effects, both documented rather than "fixed": + +- The scaled budget also loosens the "one lot exceeds the budget" skip. +- Because lots are floored, a 2× can land slightly above a pure doubling while + staying strictly inside the scaled budget. + +**To revisit when:** a global scale is genuinely wanted — it should then be an +explicit, separately-named control with its own confirmation, not a default. + +## Action items + +- [x] `_scaled_int` / `_scaled_float` / `_strategy_size_multiplier` at read time. +- [x] Malformed → 1 for paper, blocked from live. +- [x] The two deliberately unscaled knobs documented in code and in + [`../lld/risk-and-safety.md`](../lld/risk-and-safety.md) §4. +- [x] Drift-guard test: a new strategy reading a size knob with the raw `_env_*` + helpers fails the build. diff --git a/docs/adr/0007-llm-agents-as-opt-in-workers.md b/docs/adr/0007-llm-agents-as-opt-in-workers.md new file mode 100644 index 0000000..8125255 --- /dev/null +++ b/docs/adr/0007-llm-agents-as-opt-in-workers.md @@ -0,0 +1,129 @@ +# ADR-0007: LLM agents as opt-in workers with host-owned deterministic gates + +**Status:** Accepted +**Date:** 2026-08-10 (retrospective) +**Deciders:** repository owner + +## Context + +Two strategies are worth trading but resist being written as rules: the +discretionary "SL Hunting" price-action method, and a five-minute SRSI/VWAP +strategy whose regime read is a judgement call. Both were candidates for an LLM. + +The obvious risks: + +- A model is **non-deterministic** — the same bar can produce different answers. +- A model can be **confidently wrong**, and will explain itself persuasively. +- A model can **reason its way around an instruction** in its own prompt. +- An SDK can time out, error, or return something unparseable. +- Model calls cost money and time. + +None of that is acceptable near a live order path — unless the model is +prevented from touching the parts where being wrong is expensive. + +## Decision + +Run each agent as an ordinary worker thread, **off by default**, with a strict +division of labour: + +| The model decides | The host decides | +|---|---| +| Is this a real setup? | Position size | +| Is the premise still alive? | Stop and target | +| Direction (or HOLD) | Daily max loss | +| Regime (CPR AI) | Entry cutoff and square-off | +| | Post-exit cooldown | +| | Stale-data liquidation | +| | Paper vs live (the standard double gate) | +| | All execution | + +Plus four invariants: + +1. **Fail-soft.** Any agent or SDK error becomes a **HOLD**, and the separate + mechanical risk loop keeps checking stop, target, max-loss, stale data and + square-off regardless of whether the model ever answers. +2. **Same execution door.** Agents act through the same tested + `enter_position` / `exit_position` path as every other worker, so broker + selection, paper/live routing, max-loss, square-off and Telegram behave + identically. +3. **Lazy imports.** A missing optional dependency disables the strategy; it + never breaks the run. +4. **Schema-validated decisions.** A malformed decision is rejected, not coerced. + +The two agents differ in isolation mechanism: SL Hunting runs the +`claude-agent-sdk` in-process; CPR AI runs Codex in a **subprocess** with a +strict environment allowlist and exposes context through **four frozen +no-argument MCP tools**, so the model cannot request a different instrument, +strike or timeframe than the one the host froze. + +## Options considered + +### Option A: Opt-in worker, host owns every mechanical gate (chosen) + +**Pros:** the model can be wrong without being dangerous; it composes with the +existing safety machinery instead of bypassing it; disabling it is one flag; +non-determinism is confined to entry/exit *judgement*, never to size or risk. +**Cons:** two extra runtimes and subscriptions; prompt/knowledge maintenance; +decisions are not reproducible, which complicates post-hoc analysis (mitigated +by the journal and the decision log). + +### Option B: LLM proposes, human approves each trade + +**Pros:** maximum safety. +**Cons:** defeats the purpose — the method trades on 1-minute bars. A human in +the loop per bar is not a system. + +### Option C: LLM as an advisory overlay on existing strategies + +**Pros:** no new execution path. +**Cons:** an "advisor" that can veto or amplify existing strategies couples +everything to it, and a bad call would degrade strategies that were working. +Independent workers with independent P&L make the agent's contribution +measurable in isolation. + +### Option D: Give the model the risk controls too + +**Pros:** simpler code; the model could size to its own conviction. +**Cons:** disqualifying. Sizing and stops are exactly where a confident wrong +answer is most expensive, and where deterministic code is strictly better. + +## Trade-off analysis + +The design rests on one observation: the model's *comparative advantage* is +pattern judgement on a chart, and its *comparative disadvantage* is arithmetic +discipline under pressure. Deterministic code is the reverse. So the split +follows the strengths rather than the convenience of the API. + +The live journal supplied the strongest evidence for keeping mechanical rules in +code: SL Hunting's prompt already contained a judgement-based post-exit re-entry +gate, and the agent **talked past it twice** (23 and 27 Jul 2026) by relabelling +the same price structure as a fresh setup. The rule now lives in the host, where +it cannot be reasoned away. That is the general principle — *a rule the model +must not be able to argue with does not belong in the prompt.* + +## Consequences + +**Easier:** trading a discretionary method without hand-coding it; disabling an +agent instantly; comparing an agent's P&L against deterministic strategies. + +**Harder:** reproducing a decision; testing (the SDK is never called in tests — +decisions are injected); cost and latency management (mitigated: SL Hunting +skips the call entirely when flat past its 10:30 cutoff). + +**To revisit when:** a rule keeps needing enforcement in code after being stated +in the prompt — that is a signal the boundary should move further toward the +host, not that the prompt needs rewording. + +## Action items + +- [x] Both agents default off; both live-disabled by default; both use the + standard double gate ([ADR-0004](0004-paper-by-default-double-gate.md)). +- [x] Fail-soft to HOLD; the mechanical risk loop is independent. +- [x] CPR AI: subprocess isolation, `safe_subprocess_environment` allowlist, + four frozen no-argument MCP tools, strict pydantic decisions. +- [x] SL Hunting: post-exit cooldown enforced in code, not the prompt; the + BankNIFTY mirror is mechanical, not a model choice. +- [x] Learning (SL Hunting) is **human-gated**: journal → off-loop coach → + operator approval → `lessons.json`, injected only when + `SL_HUNTING_LESSONS_ENABLED`. Automating promotion would let the agent + rewrite its own instructions from its own losses. diff --git a/docs/adr/0008-single-env-as-config-source.md b/docs/adr/0008-single-env-as-config-source.md new file mode 100644 index 0000000..c538a03 --- /dev/null +++ b/docs/adr/0008-single-env-as-config-source.md @@ -0,0 +1,107 @@ +# ADR-0008: A single `.env` as the only configuration source + +**Status:** Accepted +**Date:** 2026-08-10 (retrospective) +**Deciders:** repository owner + +## Context + +The system has on the order of 300+ settings: per-strategy sizes, risk budgets, +time windows, poll intervals, broker credentials, feed selection, notifier +tokens, agent toggles. It runs on one machine, for one operator, one session a +day. + +The question is where those settings live, and how the operator knows what is in +force. + +## Decision + +**One gitignored `Dependencies/.env`, copied from the committed +`Dependencies/env.example` template, is the single source of truth.** Nothing is +hard-coded per run, and there are no CLI overrides, profiles, or +environment-specific layers. + +Values are read through `_env_str` / `_env_bool` / `_env_int` / `_env_float`, or +`_scaled_int` / `_scaled_float` for size-bearing knobs. **Never ad-hoc +`os.getenv`.** + +Two mechanisms keep the three places a setting can live from drifting apart: + +- `python algo.py check-env` — a read-only audit reporting keys missing from + `.env`, mistyped/stale keys, and knobs missing from the template. It prints + key **names only**, never values, so its output is safe to share. +- A CI gate (`test_every_env_setting_the_code_reads_is_documented_in_env_example`) + that fails the build when a new `_env_*` key lands without an `env.example` + entry. It imports the *same* helpers the operator command uses, so the two can + never disagree. + +## Options considered + +### Option A: One `.env` + template + drift audit (chosen) + +**Pros:** one place to look and one to audit; secrets stay out of git by +construction; trivially diffable; no framework; the same file describes paper +and live runs, so nothing changes shape when going live. +**Cons:** a large flat file; discovery depends entirely on `env.example` staying +current (hence the CI gate); no typing beyond the helpers; no per-environment +layering. + +### Option B: Layered config (YAML base + environment overrides) + +**Pros:** structure, typing, per-environment defaults. +**Cons:** "what is actually in force?" becomes a merge question. For a +single-machine system that is pure cost. It also fits secrets badly — they end +up in a separate mechanism anyway, so the count of configuration sources goes +*up*, not down. + +### Option C: CLI flags for the volatile settings + +**Pros:** explicit at launch; nothing left over from yesterday. +**Cons:** live-ness would become invisible to `check-env`, and the launch +command would become the real config — undocumented and unreviewable. It also +moves risk decisions to the moment of starting, when the operator is least +likely to be reviewing them. + +### Option D: A settings module in Python + +**Pros:** typed, IDE-navigable, no parsing. +**Cons:** secrets in a tracked file, or a second mechanism for secrets. Changing +a lot size would become a code change. + +## Trade-off analysis + +The decisive question is: **when something goes wrong mid-session, how many +places must be checked to know what the runner is doing?** + +Option A answers "one file, and one command that audits it". Every alternative +answers "two or more, plus a merge rule". For live-money code debugged under +time pressure by one person, that is worth more than typing or structure. + +The genuine weakness Option A accepts is the silent-default problem: a key +present in the code and the template but **missing from `.env` is not an error** +— the runner just uses the in-code default. Nothing complains, and an unseen +default ends up governing a live-money run. `check-env` exists for exactly that +failure mode, and it is the reason this ADR would be unsafe without it. + +## Consequences + +**Easier:** knowing what a run will do; diffing a config change; keeping secrets +out of git; auditing before a live session. + +**Harder:** discovering settings (mitigated by `env.example` + the CI gate); +running two different configurations simultaneously (not supported, and not +wanted); type errors are runtime, not authoring, errors. + +**To revisit when:** the system runs in more than one environment at once, or a +second operator needs a different configuration on the same machine. + +## Action items + +- [x] `env.example` is the only discovery surface; holds **blank placeholders**. +- [x] All reads through the `_env_*` / `_scaled_*` helpers. +- [x] `python algo.py check-env` — read-only, names-only output, non-zero exit on + findings so it can gate a pre-flight script. +- [x] CI gate: code → template, one direction only. The reverse would flag the + ~200 `_*` knobs built from f-strings, which the AST cannot see. +- [x] The gate asserts the AST walk finds >300 keys, so a renamed helper cannot + make it silently pass while checking nothing. diff --git a/docs/adr/0009-importlib-loading-for-spaced-filenames.md b/docs/adr/0009-importlib-loading-for-spaced-filenames.md new file mode 100644 index 0000000..6e8a7c9 --- /dev/null +++ b/docs/adr/0009-importlib-loading-for-spaced-filenames.md @@ -0,0 +1,116 @@ +# ADR-0009: Load spaced-name files with `importlib` instead of renaming them + +**Status:** Accepted +**Date:** 2026-08-10 (retrospective) +**Deciders:** repository owner + +## Context + +Many files in this repository have spaces in their names: + +``` +Nifty Multi Strategy Front Test - Master File.py +Signal Generators/Nifty Supertrend Signal Generator.py +Data Extractors/Nifty 1m 5Y Data Fetch Dhan.py +Signal Generators/SL Hunting AI Agent/ ← the folder, too +``` + +Python cannot import these with a normal dotted import: the filename is not a +valid module name, and neither is a directory with spaces. + +The names are not accidental — the author navigates the repository by them, and +they read as descriptions rather than identifiers. The live track record, +external references, and the operator's own muscle memory all point at these +paths. + +## Decision + +Keep the names. Load spaced-name files through +`load_module(module_name, file_path)`, a wrapper around +`importlib.util.spec_from_file_location`, and add explicit `sys.path` bootstraps +where sibling modules import each other by bare name. + +Accept the consequences deliberately: + +- **mypy scope is limited to identifier-named modules.** Files with spaces + cannot be mypy modules. They are covered by `compileall` (syntax) plus the + unittest suite (behaviour) instead. +- **Tests need explicit path bootstraps.** Each test folder for a spaced-name + source folder carries a `conftest.py` that puts the corresponding **source** + folder on `sys.path`. +- **The master's own test suite loads it via `importlib`** with `dhanhq` mocked. + +## Options considered + +### Option A: Keep the names, load via `importlib` (chosen) + +**Pros:** no rename churn across a live-money repository; paths in the README, +the CLI, external references and the operator's habits all keep working; the +loader is ~10 lines. +**Cons:** mypy cannot see the largest file in the repository; every test folder +needs a path bootstrap; imports do not look like normal Python; tooling +configuration (`pyproject.toml`, CI, Bandit excludes) has to name paths with +spaces, which means quoting everywhere. + +### Option B: Rename everything to snake_case + +**Pros:** ordinary imports; full mypy coverage including the master runner; no +conftest bootstraps; no quoting. +**Cons:** a very large, purely mechanical diff across live-money code, touching +every doc, every CLI example and every import site at once. The risk is not the +rename itself — it is that a rename this broad makes any *behavioural* change +hidden inside it invisible in review. Against a system running real money, that +is the wrong trade for a cosmetic gain. + +### Option C: Add a packaging layer (`__init__.py`, a `[project]` section) + +**Pros:** proper package semantics; imports resolve normally. +**Cons:** does not solve the problem — a package still cannot contain a module +whose *filename* has spaces. It would add packaging machinery on top of the +`importlib` loading that would still be required. Also, `__init__.py` files +would change pytest's `rootdir` insertion behaviour and break the bare-name +sibling imports the strategy folders rely on. + +## Trade-off analysis + +Option B is the technically correct end state and is the one that would be +chosen for a new repository. It is not chosen here because the *timing* is +wrong: the cost is a repository-wide diff against running live code, and the +benefit — mypy coverage of the master file — is partially available another way +(`compileall` plus a 487-case unittest suite). + +The honest summary: this is a decision to defer a cleanup, not a claim that +spaces are good. The cost is paid continuously in small amounts (quoting, +conftests, mypy scope) rather than once in a large amount. + +If the master runner is ever split into modules — the other significant piece of +debt named in the HLD — that is the natural moment to rename, because the files +are being rewritten anyway and the diff is already under review. + +## Consequences + +**Easier:** navigating by descriptive names; keeping every documented path, +README example and CLI invocation stable. + +**Harder:** +- mypy is scoped to identifier-named modules only (`pyproject.toml` lists them + explicitly, and a policy test asserts that every `cpr_ai_*.py` module is inside + that scope so a new one cannot silently escape). +- Every shell command and tooling exclude needs quoting. +- Test folders need `conftest.py` path bootstraps, and each one must insert + **only** the folder it needs — a repository-wide path would let tests pass + through imports production never uses and hide a missing dependency. +- Test file basenames must stay unique repository-wide, because pytest's + `prepend` import mode with no `__init__.py` files keys modules by basename. + +**To revisit when:** the master runner is split into modules. Do the rename +then, in the same review, not before. + +## Action items + +- [x] `load_module()` in the master runner (~L1141). +- [x] mypy `files` + `mypy_path` list every identifier-named module and root. +- [x] `compileall` covers every Python file, spaced names included. +- [x] Per-folder `conftest.py` bootstraps under `Tests/`, each inserting only + the source folder it needs ([ADR-0010](0010-tests-in-a-mirrored-tests-tree.md)). +- [x] Bandit and Ruff excludes name the vendored and reference-only paths. diff --git a/docs/adr/0010-tests-in-a-mirrored-tests-tree.md b/docs/adr/0010-tests-in-a-mirrored-tests-tree.md new file mode 100644 index 0000000..0fb54f2 --- /dev/null +++ b/docs/adr/0010-tests-in-a-mirrored-tests-tree.md @@ -0,0 +1,133 @@ +# ADR-0010: Consolidate tests into a mirrored top-level `Tests/` tree + +**Status:** Accepted +**Date:** 2026-08-10 +**Deciders:** repository owner + +## Context + +Test files were scattered across the repository, co-located with the code they +exercise: + +``` +test_nifty_multi_strategy_master.py (repo root, 504 KB) +test_market_data_health.py (repo root) +Dependencies/test_*.py (11 files) +Dependencies/Dhan API/test_dhan_execution.py +Dependencies/Flattrade API/test_flattrade_execution.py +Data Extractors/test_index_fetch_construction.py +Signal Generators/test_*.py (3 files) +Signal Generators/*/test_*.py and */tests/*.py (14 files) +Signal Generators/*/conftest.py (3 files) +``` + +Consequences of that layout: `ls Dependencies/` showed 11 test files against 15 +runtime modules; the two largest files in the repository root were test suites; +and "run the tests" required naming four different paths. + +Every one of these files is location-dependent. They anchor on `__file__` to +find the repo root or the module under test, or they rely on pytest's `prepend` +import mode inserting the test's own directory into `sys.path` so a bare +`from check_env_config import audit` resolves. + +## Decision + +Move every test file and every test `conftest.py` into a top-level `Tests/` +directory whose internal structure **mirrors the source tree**: + +``` +Tests/ + test_nifty_multi_strategy_master.py + test_market_data_health.py + Dependencies/ + test_broker_contract.py … + Dhan API/test_dhan_execution.py + Flattrade API/test_flattrade_execution.py + Data Extractors/test_index_fetch_construction.py + Signal Generators/ + test_renko_bounds.py … + CPR AI Agent/ conftest.py + tests + CPR Strategy/ + Regime Adaptive Strategy/ + SL Hunting AI Agent/ conftest.py + tests + Subhamoy Strategies/ +``` + +Each folder that mirrors a spaced-name source folder carries a `conftest.py` +that puts the corresponding **source** folder — not the test folder — on +`sys.path`, preserving the bare-name sibling imports the strategy modules use at +runtime. + +Path anchors are rewritten to point back at the source tree +(`Path(__file__).resolve().parents[N]`), and every runner path is updated: the CI +workflow (both jobs), `pyproject.toml` (Ruff per-file-ignores, coverage `omit`), +and the documented commands in `README.md`, `CLAUDE.md` and `AGENTS.md`. + +## Options considered + +### Option A: Mirrored `Tests/` tree (chosen) + +**Pros:** one obvious home for tests; a 1:1 mapping from a source path to its +test path; component folders show only runtime code; each conftest keeps the +narrow, deliberate `sys.path` scope the co-located ones had; test basenames stay +unique, which pytest's `prepend` import mode requires without `__init__.py`. +**Cons:** a test is no longer adjacent to its subject; the path anchors are one +level further from what they point at; the move itself touches every test file. + +### Option B: Flat `Tests/` folder + +**Pros:** simplest possible layout; one conftest. +**Cons:** ~28 files in one directory with no grouping, and a single conftest +would have to insert **every** source directory on `sys.path`. That is exactly +what the existing conftests deliberately avoid: a repository-wide path lets a +test pass through an import production never uses, hiding a missing dependency +or an accidental cross-strategy coupling. + +### Option C: Leave tests co-located + +**Pros:** zero risk; a test sits next to its subject. +**Cons:** the status quo being changed — no single place to look, component +folders dominated by test files, and the repository root led by two large test +suites. + +## Trade-off analysis + +Adjacency (Option C) is a real benefit and is what is being given up. It is +traded for discoverability and for a clean separation between runtime code and +test code in a repository where the runtime is the product. + +Option B was rejected on a safety argument rather than an aesthetic one: the +per-folder conftests are load-bearing. `Signal Generators/CPR AI Agent/conftest.py` +inserts *only* that agent's directory precisely so a test cannot resolve an +import that production would fail on. A single flat conftest would erase that +property for every strategy at once. + +The main risk of this change is silent test loss — a file that stops being +collected still looks like a green build. That is mitigated by comparing exact +counts before and after (see Action items), not by inspection. + +## Consequences + +**Easier:** finding tests; running everything (`python -m pytest Tests -q`); +reading a component folder; keeping runtime and test code visually separate. + +**Harder:** a test is no longer adjacent to its subject; path anchors are +indirect and must be right; **test file basenames must remain unique +repository-wide**, since pytest keys modules by basename with no `__init__.py` +present; and anyone adding a test must mirror the source path rather than drop +the file next to the code. + +**To revisit when:** `__init__.py` files are introduced (that would change +pytest's import behaviour and relax the basename constraint), or if the +mirroring proves to drift from the source tree in practice. + +## Action items + +- [x] Move all test files and test conftests with `git mv` so history follows. +- [x] Rewrite every `__file__`-relative anchor to point at the source tree. +- [x] One `conftest.py` per mirrored spaced-name folder, each inserting only the + source folder it needs. +- [x] Update the CI workflow (both jobs), `pyproject.toml`, `README.md`, + `CLAUDE.md`, `AGENTS.md` and the per-folder Readmes. +- [x] **Verify by exact count, not by "it passed":** 487 master + 26 + market-data-health + 1089 pytest = 1602 before; the same after. diff --git a/docs/adr/0011-committed-docs-untracked-superpowers.md b/docs/adr/0011-committed-docs-untracked-superpowers.md new file mode 100644 index 0000000..c0ff5b3 --- /dev/null +++ b/docs/adr/0011-committed-docs-untracked-superpowers.md @@ -0,0 +1,125 @@ +# ADR-0011: A committed `docs/` set, with the Superpowers workspace untracked + +**Status:** Accepted +**Date:** 2026-08-10 +**Deciders:** repository owner + +## Context + +`docs/` existed but held only `docs/superpowers/` — two files (a plan and a +design spec) written by an agent during one development session in July 2026: + +``` +docs/superpowers/plans/2026-07-27-mat-111-sl-hunting-cooldown.md +docs/superpowers/specs/2026-07-27-mat-111-sl-hunting-cooldown-design.md +``` + +Both were tracked in git. They describe *how one session's work was planned*, +not how the product works, and they were already stale relative to the shipped +cooldown implementation. + +Meanwhile the actual architecture was documented across `README.md` (operator +setup and a running changelog), `CLAUDE.md` / `AGENTS.md` (condensed rules for +coding agents), and six per-folder `Readme.md` files. There was no document a +new reader could open to understand the system as a system, and nothing recorded +*why* the significant decisions were made — only what they were. + +`.superpowers/` at the repository root was already gitignored for exactly this +reason; the `docs/superpowers/` subtree had simply been missed. + +## Decision + +Two parts. + +**1. Untrack `docs/superpowers/`.** Add it to `.gitignore` and `git rm --cached` +the two files. They stay on disk; they leave source control. This makes the rule +consistent with the existing `.superpowers/` entry. + +**2. Fill `docs/` with a committed architecture set:** + +``` +docs/ + README.md index and reading order + hld/ one high-level design for the whole repository + lld/ one low-level design per component (12) + adr/ decision records (11) +``` + +The division of labour between this and the existing documentation: + +| Document | Answers | Audience | +|---|---|---| +| root `README.md` | How do I set it up and run it? | operator | +| `CLAUDE.md` / `AGENTS.md` | What rules must I follow when changing code here? | coding agents | +| per-folder `Readme.md` | What is in this folder? | anyone in that folder | +| `docs/hld/` | How does the system fit together? | new reader | +| `docs/lld/` | How does this component work, and what breaks it? | anyone changing it | +| `docs/adr/` | Why is it like this? | anyone about to change it back | + +## Options considered + +### Option A: Committed HLD + LLDs + ADRs, Superpowers untracked (chosen) + +**Pros:** a new reader has one entry point; component internals are documented +where they can be reviewed in the same pull request as the code; the *reasons* +behind the safety model are recorded, which matters most for decisions that look +like over-engineering until you know the incident behind them. +**Cons:** documentation drifts unless maintained; ~24 files to keep current; +some overlap with the per-folder Readmes. + +### Option B: Keep everything in `README.md` + +**Pros:** one file; already the habit. +**Cons:** it is already 27 KB and mixes setup, a changelog and architecture. It +serves the operator well and would serve none of them well if the architecture +were added. + +### Option C: Commit the Superpowers plans as the design record + +**Pros:** they already exist. +**Cons:** they are session artefacts — planned rather than shipped, dated, and +already stale. Presenting them as the design record documents an intention, not +the system. + +## Trade-off analysis + +The cost of Option A is drift, and it is a real cost — stale architecture +documentation is worse than none, because it is believed. Two conventions are +adopted against it: + +1. **The LLD changes in the same commit as the component.** If a pull request + changes behaviour, its LLD is part of that pull request. +2. **Never restate a number the code owns.** Prefer naming the file that holds + the default to copying the value. + +ADRs are treated as append-only history: when a decision changes, a new ADR +supersedes the old one and the old one is marked `Superseded` rather than +edited. That keeps the record of *why the previous answer was chosen*, which is +the part that is expensive to reconstruct. + +## Consequences + +**Easier:** onboarding a reader (human or agent); reviewing a change against a +stated design; arguing against a decision from its actual reasoning rather than +from a guess about it. + +**Harder:** ~24 more files to keep current. The mitigation is convention, not +tooling. + +**To revisit when:** the docs are observed to be stale. The existing repository +already has precedent for enforcing documentation freshness in CI — +`test_repository_policy.py` fails the build on stale worker-roster claims in +`README.md`, `CLAUDE.md`, `AGENTS.md` and the master file. Extending that gate +to cover `docs/hld/` is a reasonable follow-up; it is deliberately **not** done +in this change, to keep the restructure surgical. + +## Action items + +- [x] `docs/superpowers/` added to `.gitignore`, files untracked with + `git rm --cached` (kept on disk). +- [x] `docs/README.md` index with reading order. +- [x] `docs/hld/system-overview.md`. +- [x] 12 LLDs under `docs/lld/`. +- [x] 11 ADRs under `docs/adr/`. +- [ ] **Follow-up:** consider adding `docs/hld/system-overview.md` to the + architecture-staleness gate in `Tests/Dependencies/test_repository_policy.py`. diff --git a/docs/hld/system-overview.md b/docs/hld/system-overview.md new file mode 100644 index 0000000..597929a --- /dev/null +++ b/docs/hld/system-overview.md @@ -0,0 +1,335 @@ +# HLD — NIFTY multi-strategy trading system + +**Status:** Current +**Scope:** the whole repository +**Audience:** anyone about to change the runtime, add a strategy, or add a broker + +--- + +## 1. What the system is + +A single-machine, single-process trading system for NIFTY index options. It runs +an approximately 27-strategy core roster plus two independently opt-in AI agents +concurrently against one shared market-data feed, decides entries and exits per +strategy, and executes those decisions either on paper (default) or through a +real broker (explicitly enabled, per strategy). + +It has been running live since May 2026. Every design decision in this document +is weighted by that: **the system is allowed to miss a trade; it is not allowed +to lose track of a position.** + +### The three phases + +``` + fetch backtest run + ----- -------- --- + DhanHQ REST ──► backtesting.py over multithreaded front test + 1-min OHLC CSV the same 1-min CSV (paper by default, live + (Data Extractors/) (My Backtest Files/) when explicitly enabled) +``` + +Only the third phase touches money. The first two exist so that a strategy's +behaviour is known before it is given capital, and both consume the *same* +1-minute candles the live runner consumes — which is why the websocket producer +trues its bars up against official REST candles (see §5.2). + +--- + +## 2. Requirements + +### 2.1 Functional + +| # | Requirement | +|---|---| +| F1 | Run many independent strategies concurrently against one market-data feed. | +| F2 | Each strategy independently decides entry/exit on NIFTY ATM options (some on multi-leg baskets). | +| F3 | Execute on paper by default; execute live only when explicitly enabled for both the process and the strategy. | +| F4 | Support more than one broker behind one interface, switchable by configuration. | +| F5 | Never lose track of live exposure — including across partial fills, lost responses, and restarts. | +| F6 | Report every entry/exit to Telegram, and per-strategy end-of-day P&L to a Google Sheet. | +| F7 | Be entirely configuration-driven from one file; nothing hard-coded per run. | + +### 2.2 Non-functional + +| Dimension | Target and why | +|---|---| +| **Latency** | Seconds, not milliseconds. Strategies act on *completed* 1-minute (or resampled 5-minute) candles, so a 2–5 second data lag is inside the decision granularity. This is the single most important non-functional fact about the system: **it is not a low-latency system**, and no design should be justified by latency. | +| **Throughput** | One index, ~30 workers, a handful of legs each. Trivial. Never the constraint. | +| **Availability** | One trading session per day, ~09:15–15:30 IST. A crash mid-session is a *safety* event (exposure may exist), not an availability event. | +| **Correctness** | Fail-closed everywhere. An unknown broker name disables live trading. An unreadable quote refuses a live entry. Ambiguous fill state freezes new live entries rather than guessing. | +| **Cost** | One DhanHQ subscription, one broker account, optionally one Claude and one Codex subscription. No cloud infrastructure. | +| **Operability** | One operator (the author). Everything must be diagnosable from one log file and one `.env` audit command. | + +### 2.3 Constraints that shaped the design + +- **One person maintains this.** Complexity that needs a team to keep alive is a + liability, not a feature. This rules out an event-bus, a database, or a + service mesh for a workload that fits in one process (see [ADR-0001](../adr/0001-single-process-thread-per-strategy.md)). +- **Broker APIs are the unreliable part.** Not the strategies, not the data + volume. The safety machinery is concentrated at that boundary. +- **Many source files have spaces in their names** (`Nifty Multi Strategy Front + Test - Master File.py`). They cannot be imported normally, which shapes module + loading, mypy scope and test layout (see [ADR-0009](../adr/0009-importlib-loading-for-spaced-filenames.md)). +- **The feed carries no volume.** Anything volume-derived (true VWAP, breadth) + is either a documented proxy or deliberately unimplemented. + +--- + +## 3. Context + +``` + ┌───────────────────────────────┐ + DhanHQ Data API ───►│ │ + (REST + optional │ Front-test master process │──► Telegram (alerts) + marketfeed WS) │ (one Python process) │ + │ │──► Google Sheets (EOD P&L) + Broker API ◄──►│ │ + (Kotak | Shoonya | │ │──► rotating log file + Flattrade | Dhan) └───────────────────────────────┘ + ▲ + Claude / Codex ◄─────────────────┘ (optional, only if the AI agents are enabled) + subscriptions +``` + +Note that DhanHQ appears on both sides of the boundary: it is the market-data +provider for every run, and it is *also* one of the four selectable execution +brokers. The two sessions are deliberately kept separate (`DhanBrokerClient` for +data, `dhan_execution_client` for orders) so a data-side token problem cannot +silently affect order placement, or vice versa. + +--- + +## 4. Component view + +``` + Nifty Multi Strategy Front Test - Master File.py + ┌──────────────────────────────────────────────────────────────────────────────┐ + │ │ + │ main() │ + │ ├─ config load + validation (_env_* / _scaled_* / _live_config_errors)│ + │ ├─ startup exposure audit (Dependencies/startup_exposure.py) │ + │ ├─ thread start + supervision (_start_and_supervise_runtime_threads) │ + │ └─ shutdown: flatten → confirm (Dependencies/trading_lifecycle.py) │ + │ │ + │ ┌─────────────────────┐ writes ┌──────────────────────────────┐ │ + │ │ Market data producer│ ───────────────► │ SharedMarketDataStore │ │ + │ │ (exactly one of): │ │ ─ lock-guarded │ │ + │ │ CentralMarketData │ │ ─ 1-min OHLC frames │ │ + │ │ Fetcher (REST) │ │ ─ LTP cache per leg │ │ + │ │ WebSocketMarketData│ │ ─ MarketDataHealth state │ │ + │ │ Fetcher (ticks) │ └──────────────┬───────────────┘ │ + │ └─────────────────────┘ reads │ │ + │ ▼ │ + │ ┌────────────────────────────────────────────────────────────────────────┐ │ + │ │ Strategy worker threads (one per enabled strategy) │ │ + │ │ │ │ + │ │ BasePaperStrategyWorker │ │ + │ │ └─ AtmSingleLegStrategyWorker ── Renko, EMA, HeikinAshi, │ │ + │ │ │ ProfitShooter, OpeningStrike, │ │ + │ │ │ CPR, CPRAlgo3, CPR AI, │ │ + │ │ │ + 14 factory-built ports │ │ + │ │ └─ NextOpenAtmStrategyWorker ── Goldmine, MoneyMachine │ │ + │ │ ├─ SupertrendBullishWorker / DonchianBearishWorker (hedged puts) │ │ + │ │ ├─ Delta20HedgedSpreadWorker (4-leg spread) │ │ + │ │ ├─ LongStrangleWorker (dual-leg OTM basket) │ │ + │ │ └─ SLHuntingAIWorker (optional, Claude) │ │ + │ └───────────────┬─────────────────────────────────┬──────────────────────┘ │ + │ │ enter_position/exit_position │ event dicts │ + │ ▼ ▼ │ + │ ┌────────────────────────────────┐ ┌────────────────────────────────┐ │ + │ │ ExecutionSafetyCoordinator │ │ queue.Queue │ │ + │ │ ─ one shared broker lock │ │ └─► TelegramMessageWorker │ │ + │ │ ─ rate limiting │ │ (best-effort, one thread)│ │ + │ │ ─ 10s deadline per call │ └────────────────────────────────┘ │ + │ │ ─ execution_ledger (qty) │ │ + │ └───────────────┬────────────────┘ │ + └──────────────────┼───────────────────────────────────────────────────────────┘ + ▼ + execution_client (broker-agnostic surface, ADR-0002) + ├─ Kotak API/kotak_execution.py + ├─ Shoonya API/shoonya_execution.py (+ vendored NorenApi.py) + ├─ Flattrade API/flattrade_execution.py + └─ Dhan API/dhan_execution.py +``` + +Shared, broker-neutral primitives live in `Dependencies/` and are deliberately +*small, pure and heavily tested* — they carry the highest coverage budgets in the +repository (90%; see [`../lld/testing-and-ci.md`](../lld/testing-and-ci.md)): + +| Module | Responsibility | +|---|---| +| `broker_contract.py` | The four normalized order outcomes and the adapter Protocol | +| `execution_ledger.py` | Quantity-bearing state per live leg; cumulative fills as deltas | +| `startup_exposure.py` | Read-only pre-flight audit of broker orders and positions | +| `trading_lifecycle.py` | Flatten-then-stop shutdown state machine | +| `market_data_health.py` | Candle validation and feed-freshness state | +| `tick_bar_builder.py` | Pure tick→bar helpers for the websocket producer | +| `risk_sizing.py` | Fail-closed lot sizing against a rupee budget | +| `next_open_entry.py` | One-bar lifetime and price rebasing for `NEXT_OPEN` signals | +| `order_splitting.py` | Split oversized orders into exchange-legal chunks | +| `secret_redaction.py` | Scrub credentials from every log record | +| `diagnostic_preflight.py` | Local checks shared by the four broker diagnostics | + +--- + +## 5. Data flow + +### 5.1 The decision loop (one strategy, one bar) + +``` + producer thread worker thread + ─────────────── ───────────── + poll/tick ─► validate ─► store + │ + ├─► worker wakes on its poll interval + │ read 1-min frame + LTP snapshot + │ health gate: is the feed fresh? ── stale ──► hold / liquidate + │ resample to the strategy's timeframe + │ signal logic (Signal Generators/…) ── none ───► sleep + │ resolve ATM strike + expiry + │ spread gate (_MAX_SPREAD_PCT) ─ too wide ► skip + │ size: risk_sizing.SizingDecision ─ over budget ► skip + │ double gate: LIVE_TRADING_ENABLED + │ && _LIVE_TRADING + ├─────► paper path: record fill at LTP + └─────► live path: ExecutionSafetyCoordinator + └─► execution_client.place_market_order + └─► poll status → typed OrderResult + └─► execution_ledger applies filled qty +``` + +The critical property of this loop is that **every gate fails closed**. A +missing quote, a malformed candle, a malformed size multiplier, an unknown +broker name — each one refuses the live action rather than proceeding on a +guess. Several of them (the spread gate, the size multiplier) deliberately +behave *differently* in paper and live: paper is allowed to proceed when the +failure is an API problem rather than a market fact, so an infrastructure blip +does not cost a paper data point. + +### 5.2 Market data, two producers, one contract + +Exactly one producer thread runs, selected by `MARKET_DATA_SOURCE` and failing +closed to REST on any unrecognised value. + +``` + REST (default) WEBSOCKET (opt-in, paid Data API) + ────────────── ───────────────────────────────── + poll full window every 2-5s ticks build the forming minute live + │ REST warmup at start + │ REST gap-backfill on reconnect + │ once per minute: true-up completed + │ candles against official REST candles + ▼ (official wins; divergence logged) + └──────────────► SharedMarketDataStore ◄──────────────┘ +``` + +Both write the *same* shape into the same store, so no strategy knows or cares +which producer is running. The true-up is what makes that claim safe: without +it, tick-built candles could drift from the candles the backtests used. See +[ADR-0005](../adr/0005-rest-vs-websocket-market-data.md). + +### 5.3 Reporting + +Trade events are published to a `queue.Queue` and drained by a single +`TelegramMessageWorker`. The queue exists so Telegram latency or downtime can +never block a trading decision — alerting is best-effort by design. + +End-of-day P&L takes a deliberately indirect route: on a clean shutdown, the +runner **parses its own append-mode log** for each strategy's realised P&L and +writes it to the Google Sheet. Parsing a log rather than reading in-memory +totals means a partially crashed session still reports what it actually did, and +the month can be backfilled from the same log. + +--- + +## 6. Concurrency model + +| Thread | Count | Role | +|---|---|---| +| Market data producer | 1 | Poll or stream; write the shared store | +| Strategy workers | ~27 core + up to 2 optional agents, minus disabled ones | Read the store, decide, execute | +| Telegram worker | 1 (if enabled) | Drain the event queue | +| Main | 1 | Start, supervise, shut down | + +**Shared mutable state is exactly two objects**, and both are lock-guarded: + +1. `SharedMarketDataStore` — many readers, one writer. +2. The broker session behind `ExecutionSafetyCoordinator` — many writers, serialized. + +Everything else a worker touches is its own. This is what makes a +thread-per-strategy model tractable for one maintainer: the concurrency review +surface is two objects, not thirty (see [ADR-0001](../adr/0001-single-process-thread-per-strategy.md)). + +The GIL is not a problem here because the workload is overwhelmingly I/O-bound +(HTTP calls and sleeps) and the CPU work per bar is small pandas operations on +frames of a few hundred rows. + +--- + +## 7. Failure model + +The system is designed around the failure modes that actually happen, in +descending order of how much they can cost: + +| Failure | Detection | Response | +|---|---|---| +| **Order acknowledged but fill unknown** | `OrderStatus.UNKNOWN` / `PARTIAL` from the adapter | Exposure may exist. Freeze new live entries, keep exits available, reconcile. **Never** treat an ack or an order ID as proof of fill. ([ADR-0003](../adr/0003-acknowledgement-is-not-a-fill.md)) | +| **Live exit rejected** | typed `REJECTED` on an exit | The position stays open. The runner does not pretend it closed. | +| **Live entry rejected, zero fill** | typed `REJECTED` **and** zero filled quantity | Only then does the entry fall back to paper. | +| **Broker call hangs** | 10-second deadline that *includes* the shared-lock and rate-limit wait | Abort the call. One hung HTTP request must not stall the shared lock and with it every other worker. | +| **Feed goes stale** | `MarketDataHealth` (10s LTP / 150s bar / 30s liquidation thresholds) | Refuse new entries; liquidate open positions past the liquidation threshold. | +| **Malformed candle** | `validate_ohlc_frame` | Reject the snapshot; do not publish it. Strategies see the last good data, never a bad one. | +| **Exposure present at startup** | `startup_exposure` read-only audit of orders *and* positions | Do not start live workers into a book this process did not create. | +| **Credentials in a log** | `install_redaction_filter` on the root logger | Scrubbed before the record reaches the console or file, including exception tracebacks and lazy `%s` args. | +| **AI agent errors** | any SDK/agent exception | Becomes a safe HOLD. The separate mechanical risk loop keeps checking stop, target, max-loss, stale data and square-off regardless. | +| **Config typo** | `_live_config_errors` at startup, `algo.py check-env` on demand | Malformed size knobs block that strategy from live (paper falls back to 1). | + +The recurring shape: **an ambiguous state is treated as the dangerous state.** + +--- + +## 8. Key trade-offs + +| Decision | What we gain | What we pay | ADR | +|---|---|---|---| +| One process, thread per strategy | Two lock-guarded objects to reason about; trivial deployment; one log | No horizontal scale; one crash stops everything; GIL caps CPU-bound work | [0001](../adr/0001-single-process-thread-per-strategy.md) | +| Broker-agnostic contract | Swap brokers by config; one execution path to test | Every adapter must contain its broker's quirks rather than leak them | [0002](../adr/0002-broker-agnostic-execution-contract.md) | +| Typed outcomes + quantity ledger | Cannot mistake an ack for a fill | More states for callers to handle; `UNKNOWN` needs an operator | [0003](../adr/0003-acknowledgement-is-not-a-fill.md) | +| Paper by default, two flags for live | A single flag can never be enough to risk money | Two places to change; easy to think you are live when you are not | [0004](../adr/0004-paper-by-default-double-gate.md) | +| REST default, websocket opt-in | Lower API load without betting the session on a socket | Two producers to keep behaviourally identical | [0005](../adr/0005-rest-vs-websocket-market-data.md) | +| Per-strategy size multiplier, no global | One typo cannot enlarge every strategy | Scaling the whole book means editing every strategy | [0006](../adr/0006-per-strategy-size-multiplier.md) | +| LLM agents opt-in, host owns gates | A model can be wrong without being dangerous | Two runtimes and two subscriptions to maintain | [0007](../adr/0007-llm-agents-as-opt-in-workers.md) | +| One `.env` | One place to look; one place to audit | A large flat file; discovery depends on `env.example` staying current | [0008](../adr/0008-single-env-as-config-source.md) | +| Spaced filenames + `importlib` | No churn renaming files the author navigates by name | mypy cannot see the master; tests need explicit path bootstraps | [0009](../adr/0009-importlib-loading-for-spaced-filenames.md) | + +--- + +## 9. Scale and what to revisit + +The system is nowhere near a resource limit, so "scale" here means *scope*, not +load. The things that would actually force a redesign: + +| If this changes | What breaks first | Likely response | +|---|---|---| +| **A second index traded as a first-class product** (not just BankNIFTY confirmation/mirror) | `SharedMarketDataStore` and the ATM resolution path both assume NIFTY is the primary underlying | Parameterize the underlying; one producer per index | +| **A strategy needs sub-second reaction** | The whole completed-candle model, and the 2–5s REST cadence | Not an increment — a different system. Do not stretch this one into it. | +| **More than ~50 workers** | Thread count is still fine; the shared broker lock becomes the queue | Per-broker connection pool, or batch order submission | +| **Multi-account or multi-operator** | `execution_ledger` and the startup audit assume one account, one process | Account-scoped ledger keys; a real store instead of in-process state | +| **The runner must survive a mid-session restart** | Ledger state is in-process; recovery today is operator-driven reconciliation | Persist the ledger (SQLite) and rebuild from the broker book on start | +| **Someone else joins the project** | Nothing technical — but the "small enough to hold in one head" premise behind ADR-0001 | Revisit ADR-0001 honestly rather than growing the master file | + +The one piece of debt worth naming explicitly: the master runner is a single +17k-line file. It is coherent and heavily tested, but it is the main reason a +change here is slower than it should be. Splitting it is a real option; it has +not been done because the test suite loads it as one module via `importlib` and +the split would have to be done without a single behavioural change to +live-money code. + +--- + +## 10. Where to go next + +- Component internals: [`../lld/`](../lld/) +- Why a decision was made: [`../adr/`](../adr/) +- How to run and configure it: root [`README.md`](../../README.md) +- Live-safety rules in condensed form: [`CLAUDE.md`](../../CLAUDE.md) / [`AGENTS.md`](../../AGENTS.md) diff --git a/docs/lld/configuration.md b/docs/lld/configuration.md new file mode 100644 index 0000000..35457d1 --- /dev/null +++ b/docs/lld/configuration.md @@ -0,0 +1,132 @@ +# LLD — Configuration and drift detection + +**Owns:** `Dependencies/env.example`, `Dependencies/.env` (gitignored), +`Dependencies/check_env_config.py`, and the `_env_*` / `_scaled_*` helpers in +the master file (~L396–510) +**Related ADR:** [0008 — a single `.env` as the only config source](../adr/0008-single-env-as-config-source.md) + +--- + +## 1. Responsibility + +One file decides everything about a run. No per-run flags, no profiles, no +environment-specific overrides. `python algo.py run` behaves entirely according +to `Dependencies/.env`. + +--- + +## 2. The three places a setting lives + +This is the source of every configuration bug in this repo, so it is worth +drawing: + +``` + Dependencies/env.example committed template ── the ONLY discovery surface + Dependencies/.env gitignored, real ── what actually runs + in-code default in _env_*(…) the fallback ── silently governs when .env omits the key +``` + +A key present in the code and the template but **missing from `.env` is not an +error** — the runner just uses the in-code default. That is exactly what makes +it dangerous: an unseen default ends up governing a live-money run, and nothing +complains. + +`python algo.py check-env` exists for that one failure mode. + +--- + +## 3. Reading configuration + +| Helper | Use for | +|---|---| +| `_env_str` / `_env_bool` / `_env_int` / `_env_float` | Ordinary knobs | +| `_scaled_int` / `_scaled_float` | **Size-bearing** knobs (`_LOTS`, `_MAX_LOTS`, `_RISK_BUDGET`, `_MAX_LOSS`) | +| `_strategy_size_multiplier(prefix)` | Resolve and validate `_SIZE_MULTIPLIER` | + +**Never use ad-hoc `os.getenv`.** Two reasons: the `check-env` audit and the CI +drift gate both find keys by walking the AST for `_env_*` calls, so a raw +`os.getenv` is invisible to both; and size knobs read without `_scaled_*` silently +ignore the multiplier. A drift-guard test fails if a new strategy reads a size +knob with the raw helpers. + +Per-strategy knobs are namespaced `_*`; the name→prefix map is +`STRATEGY_ENV_PREFIX`. + +--- + +## 4. `algo.py check-env` + +Read-only audit implemented in `Dependencies/check_env_config.py`. It reports: + +- settings the code reads that are **missing from `.env`** (an unseen in-code + default is in force) +- **mistyped or stale keys** — a typo means the setting you intended is not being + applied at all +- knobs **missing from the template** + +It exits non-zero on findings so it can gate a pre-flight script, and it prints +key **names only — never a value out of `.env`** — so its output is safe to paste +into an issue or share with a reviewer. + +--- + +## 5. The CI drift gate + +`Tests/Dependencies/test_repository_policy.py` imports the *same* helpers +(`audit`, `env_keys_read_by`, `source_files`) that the operator command uses, so +the gate and the tool can never disagree about what "documented" means. + +`test_every_env_setting_the_code_reads_is_documented_in_env_example` fails the +build when a new `_env_*` key lands without an `env.example` entry. + +**One direction only (code → template).** The reverse would flag the ~200 +per-strategy `_*` knobs that `_signal_gen_ops` builds from f-strings, +which are real settings the AST cannot see. The test also asserts that the AST +walk finds >300 keys — a sanity check, so that a renamed helper cannot make the +gate silently pass while checking nothing. + +--- + +## 6. Secrets + +- `.env` is gitignored. `env.example` holds **blank placeholders only**. +- Also gitignored: `shoonyakey.txt`, `Dependencies/gsheet_oauth_*.json`, + `Dependencies/all_instrument*.csv`. +- Every `.env` value whose key looks sensitive and is ≥8 characters is fed to the + root-logger redaction filter at startup — see + [`risk-and-safety.md`](risk-and-safety.md) §8. + +**Never commit a secret.** If one is committed, rotating it at the broker is the +fix; removing the commit is not. + +--- + +## 7. Config-driven safety + +Configuration is not just tuning here; several safety behaviours are decided at +read time: + +| Setting | Fail-closed behaviour | +|---|---| +| `LIVE_BROKER` | Unknown value → live disabled entirely | +| `MARKET_DATA_SOURCE` | Anything but `WEBSOCKET` → REST | +| `_SIZE_MULTIPLIER` | Malformed (`0`, `2.5`, `30`, `"two"`) → 1 for paper, **blocked from live** | +| `_VIRTUAL_TRADING` | false → the thread never starts | +| `LIVE_TRADING_ENABLED` | Absent → false → paper | +| `GSHEET_ID`, `TELEGRAM_*` | Absent → safe no-op | + +`_live_config_errors(...)` collects these at startup so a misconfiguration +surfaces before the first bar, not at the first order. + +--- + +## 8. Adding a setting — checklist + +1. Read it through the right helper (`_env_*`, or `_scaled_*` if it is + size-bearing). +2. Add it to `Dependencies/env.example` with a blank/placeholder value and a + comment explaining what it does. **CI fails without this.** +3. If it can be malformed, decide the fail-closed direction and add it to + `_live_config_errors` if it affects live trading. +4. Run `python algo.py check-env` and confirm it is clean. +5. Document it in the relevant LLD if it changes a component's contract. diff --git a/docs/lld/cpr-codex-ai-agent.md b/docs/lld/cpr-codex-ai-agent.md new file mode 100644 index 0000000..59ea725 --- /dev/null +++ b/docs/lld/cpr-codex-ai-agent.md @@ -0,0 +1,186 @@ +# LLD — CPR Codex AI Agent (optional, Codex) + +**Owns:** `Signal Generators/CPR AI Agent/` · `CPRAIWorker`, `CPRAITradeState` (master file) +**Status:** optional, **disabled by default** (`CPR_AI_ENABLED=false`) and +**live-disabled by default** (`CPR_AI_LIVE_TRADING=false`) +**Related ADR:** [0007 — LLM agents as opt-in workers](../adr/0007-llm-agents-as-opt-in-workers.md) +**Operator detail:** the folder's own [`README.md`](../../Signal%20Generators/CPR%20AI%20Agent/README.md) + +--- + +## 1. Responsibility + +An independent five-minute SRSI/VWAP strategy in which Codex judges regime, +setup, and premise exits, while the host owns every deterministic entry and risk +gate plus all execution. + +It is *not* an arbiter over the other CPR strategies. Ordinary CPR, CPR Algo 3, +Regime Adaptive and CPR AI are **independent strategies that may run together +with independent positions and independent P&L**. + +--- + +## 2. Modules + +| File | Role | +|---|---| +| `cpr_ai_agent.py` | `CPRAgent`, `CPRHostPolicy`, `CPRAgentRunResult`, `CPRToolCallRecord` | +| `cpr_ai_context.py` | Builds the frozen per-bar context | +| `cpr_ai_signals.py` | `freeze_cpr_context` — the snapshot boundary | +| `cpr_ai_tools.py` | `FrozenCPRContextRegistry`, `EXPECTED_TOOL_NAMES` | +| `cpr_ai_mcp_server.py` | In-process MCP server exposing the four tools | +| `cpr_ai_schema.py` | `CPRAgentDecision`, `validate_position_state` (strict pydantic) | +| `cpr_ai_prompt.py` | Versioned system prompt (`CPR_AI_PROMPT_VERSION`) | +| `cpr_ai_codex_runner.py` | Thread config, `safe_subprocess_environment` | +| `cpr_ai_codex_subprocess.py` | The child process boundary | +| `cpr_ai_decision_log.py` | JSONL decision log | +| `cpr_ai_runner.py` | Standalone smoke runner (`--synthetic --fake` / `--authenticated`) | + +--- + +## 3. The frozen-context boundary + +This is the component's defining design choice. + +``` + completed 5-min bar + │ + ▼ + freeze_cpr_context(...) ← ONE snapshot, taken once, immutable + │ + ▼ + FrozenCPRContextRegistry + │ + ├── session_levels() ┐ + ├── momentum_vwap() │ FOUR no-argument MCP tools + ├── market_structure() │ read-only, no parameters + └── position_state() ┘ + │ + ▼ + Codex ──► CPRAgentDecision (strict pydantic) +``` + +Why **no-argument** tools: + +- The model cannot ask about a different instrument, strike, or timeframe than + the one the host froze. There is no parameter to smuggle a request through. +- Every tool answers from the *same* snapshot, so the model cannot see the + market move mid-reasoning and produce a decision based on two different states. +- The tool surface is a fixed set (`EXPECTED_TOOL_NAMES`), verified by tests, so + a new capability cannot appear without a code change and a review. + +--- + +## 4. Division of labour + +| Codex decides | The host decides | +|---|---| +| Regime | Entry geometry | +| Setup validity | Stop distance | +| Premise exits | Level validation | +| | Sizing (`CPR_AI_LOTS`, `CPR_AI_MAX_LOSS`, `CPR_AI_SIZE_MULTIPLIER`) | +| | Time cutoffs (start 09:30, entry cutoff 15:00, square-off 15:15) | +| | Lifecycle state (`CPRAITradeState`) | +| | All execution | + +Indicator settings are fixed and documented: RSI 14 / Stochastic 14 / K 3 / D 3, +zones 20 and 80; one equal-size add; 30-NIFTY-point and 2-NIFTY-point geometry +constants; 0.40 threshold. These live in `env.example` and are asserted by +`Tests/Dependencies/test_repository_policy.py` so the operator-facing +explanation cannot drift from the code. + +--- + +## 5. Process isolation + +Codex runs in a **subprocess** with `safe_subprocess_environment` — a strict +allowlist, so trading and API secrets are not inherited by the child. The child +boundary is `cpr_ai_codex_subprocess.py`; the parent side is +`cpr_ai_codex_runner.py`. + +`CPR_AI_SDK_TIMEOUT_SECONDS` (default 90) bounds the call. A timeout is a HOLD. + +--- + +## 6. Safety posture + +- Disabled by default; live-disabled by default. Real orders require **both** + `LIVE_TRADING_ENABLED=true` and `CPR_AI_LIVE_TRADING=true`, plus the normal + startup exposure audit and config validation. +- `_cpr_ai_startup_errors()` refuses to start a misconfigured agent. +- Decisions are strict-pydantic validated; a malformed decision is rejected, not + coerced. +- `CPR_AI_DECISION_LOGGING_ENABLED` (default true) writes every decision to + `Backtest Outputs/cpr_ai_decisions.jsonl` for after-the-fact review. +- Any SDK/agent failure is a HOLD; the mechanical risk loop is unaffected. + +--- + +## 7. Configuration + +Defaults live in `Dependencies/env.example` and are pinned by the policy test: + +| Key | Default | +|---|---| +| `CPR_AI_ENABLED` | false | +| `CPR_AI_VIRTUAL_TRADING` | true | +| `CPR_AI_LIVE_TRADING` | false | +| `CPR_AI_MODEL` | `gpt-5.6-terra` | +| `CPR_AI_REASONING_EFFORT` | medium | +| `CPR_AI_SDK_TIMEOUT_SECONDS` | 90 | +| `CPR_AI_LOTS` / `CPR_AI_MAX_LOSS` / `CPR_AI_SIZE_MULTIPLIER` | 1 / 5500 / 1 | +| `CPR_AI_POLL_SECONDS` | 5 | +| `CPR_AI_TRADING_START_HOUR` / `_MINUTE` | 09:30 | +| `CPR_AI_ENTRY_CUTOFF_HOUR` / `_MINUTE` | 15:00 | +| `CPR_AI_SQUARE_OFF_HOUR` / `_MINUTE` | 15:15 | +| `CPR_AI_DECISION_LOGGING_ENABLED` / `_LOG_PATH` | true / `Backtest Outputs/cpr_ai_decisions.jsonl` | + +Install the exact optional set from `requirements-codex-ai.txt`. Both AI agents +run inside the same process, so they must agree on one MCP package version — +`mcp==1.29.0` appears in both `requirements-ai.txt` and `requirements-codex-ai.txt`, +and the policy test asserts it. + +--- + +## 8. Verification without spending money + +Two zero-order smoke commands: + +```bash +python "Signal Generators/CPR AI Agent/cpr_ai_runner.py" --synthetic --fake +``` + +```bash +python "Signal Generators/CPR AI Agent/cpr_ai_runner.py" --synthetic --authenticated +``` + +`--fake` makes **no billed/model/broker call** at all. CI runs only the +unauthenticated path — the authenticated smoke is an operator action. + +--- + +## 9. Testing + +`Tests/Signal Generators/CPR AI Agent/` — context freezing, tool registry, +schema validation, runtime/subprocess behaviour, and master integration. Its +`conftest.py` puts the **source** agent folder on `sys.path`, and deliberately +only that folder: adding a repository-wide path would let tests pass through +imports production never uses and could hide a missing dependency or an +accidental legacy-CPR coupling. + +`Tests/Dependencies/test_repository_policy.py` additionally asserts that every +`cpr_ai_*.py` module is inside mypy's scope, so a new module cannot silently +escape type checking. + +--- + +## 10. Contrast with SL Hunting + +| | CPR Codex AI | SL Hunting | +|---|---|---| +| Provider | Codex (subprocess + MCP) | Claude (`claude-agent-sdk`, in-process) | +| Timeframe | completed 5-min bars | completed 1-min bars | +| Tool surface | four frozen no-argument MCP tools | prompt context + order tool | +| Instruments | NIFTY only | NIFTY + mechanical BankNIFTY mirror | +| Learning loop | none — decision log only | journal → coach → human-gated `lessons.json` | +| Shared | opt-in, off by default, host-owned gates, fail-soft to HOLD, same double gate | ← identical | diff --git a/docs/lld/data-and-backtesting.md b/docs/lld/data-and-backtesting.md new file mode 100644 index 0000000..14d7eab --- /dev/null +++ b/docs/lld/data-and-backtesting.md @@ -0,0 +1,133 @@ +# LLD — Data extraction and backtesting + +**Owns:** `Data Extractors/` · `My Backtest Files (For Reference)/` · the +`fetch-data` and `backtest` commands in `algo.py` + +--- + +## 1. Responsibility + +Produce the historical 1-minute OHLC that strategies are validated against, and +run those strategies over it before they are given capital. + +Neither of these touches money or a broker. That is why they live outside the +runner's safety machinery entirely. + +--- + +## 2. Data extraction + +``` + Data Extractors/ + index_1m_5y_data_fetch_dhan_common.py ← the shared engine + Nifty 1m 5Y Data Fetch Dhan.py ┐ + Banknifty 1m 5Y Data Fetch Dhan.py ├ thin per-index wrappers + Finnifty 1m 5Y Data Fetch Dhan.py ┘ +``` + +One engine, three wrappers. The wrappers differ only in the instrument they +name; everything about paging, retry, epoch-unit handling and CSV writing lives +in the common module, which is also the only part under mypy (the wrappers have +spaces in their names). + +Output lands in `Backtest Outputs/`, which is gitignored. + +```bash +python algo.py fetch-data --index nifty --interval 5 --lookback 5y +``` + +Any flag beyond the selector passes straight through to the underlying script, +and each script still runs standalone. + +### 2.1 Epoch-unit inference + +The Dhan intraday response has been observed with different epoch units. +`_infer_epoch_unit` / `_validate_single_epoch_unit` (in the master, mirrored in +the extractor engine) infer the unit and then **assert it is consistent across +the response** — a frame with mixed units would otherwise produce candles +decades apart with no obvious symptom. + +--- + +## 3. Backtesting + +``` + My Backtest Files (For Reference)/ + Nifty Renko Strategy Backtest.py + Nifty EMA Trend Strategy Backtest.py + Nifty Heiken Ashi Futures 5Y Backtest.py + Nifty CPR Strategy Backtest.py + profit_shooter_backtest.py + Subhamoy Strategies/ + Nifty Goldmine Strategy Backtest.py + Nifty Money Machine Strategy Backtest.py + subhamoy_backtest_common.py +``` + +Built on [`backtesting.py`](https://pypi.org/project/backtesting/), run against +the CSVs from §2: + +```bash +python algo.py backtest --strategy renko --data "Backtest Outputs/nifty_renko_futures_5y_1min_data.csv" +``` + +These are explicitly **reference material**. They are excluded from Ruff's +default treatment (`E402` is allowed — they keep the deliberate +`sys.path`-before-import pattern), excluded from Bandit, and excluded from +coverage. They are kept because they record how a strategy was evaluated, not +because they are part of the runtime. + +--- + +## 4. The shared-candle invariant + +The single most important property linking these three phases: + +> **Backtests, the REST producer, and the websocket producer must all be looking +> at the same candles.** + +That is why the websocket producer trues its tick-built bars up against official +REST candles once per minute, with official always winning (see +[`market-data.md`](market-data.md) §3.2). Without it, live bars would slowly +drift away from the bars every backtest result was computed on, and no +individual test would fail. + +Both phases also share the "candles are labelled by their START time" convention +and the completed-candle rule. + +--- + +## 5. Typical workflow + +``` +1. python algo.py fetch-data --index nifty → CSV in Backtest Outputs/ +2. python algo.py backtest --strategy renko --data "…csv" +3. python algo.py run → paper by default +4. _LIVE_TRADING=true + LIVE_TRADING_ENABLED=true → live, one strategy at a time +``` + +Step 3 should run for at least one full session before step 4 for any strategy +whose size or logic changed. + +--- + +## 6. Testing + +`Tests/Data Extractors/test_index_fetch_construction.py` covers the shared +engine's construction and request-building. Its module path anchor points back +at the **source** `Data Extractors/` folder. + +The backtests themselves are not tested — they are reference scripts, not +runtime code. `compileall` is their only gate. + +--- + +## 7. Limitations + +- **No survivorship or corporate-action handling.** Index data, so not usually a + concern, but it is not modelled. +- **Backtests do not model the spread gate, sizing rejections, or partial fills.** + A backtest is an idea filter, not a P&L forecast. The paper-trading phase is + what exercises the execution path. +- **No volume** in the source data, so volume-derived indicators are proxies here + too — the same limitation the live feed has. diff --git a/docs/lld/execution-and-brokers.md b/docs/lld/execution-and-brokers.md new file mode 100644 index 0000000..712a024 --- /dev/null +++ b/docs/lld/execution-and-brokers.md @@ -0,0 +1,238 @@ +# LLD — Execution and the broker layer + +**Owns:** `ExecutionSafetyCoordinator`, `OptionsContractResolver`, +`_select_execution_client` (master file) · `Dependencies/broker_contract.py` · +`Dependencies/execution_ledger.py` · `Dependencies/startup_exposure.py` · +`Dependencies/order_splitting.py` · the four adapters under `Dependencies/*/` +**Related ADRs:** [0002](../adr/0002-broker-agnostic-execution-contract.md), [0003](../adr/0003-acknowledgement-is-not-a-fill.md) + +> This is the most safety-critical component in the repository. Read +> [`risk-and-safety.md`](risk-and-safety.md) alongside it. + +--- + +## 1. Responsibility + +Turn "buy 3 lots of the NIFTY 24150 CE" into a real order at whichever broker is +configured, and return an answer the caller **cannot misread**. + +The second half is the hard part. Broker APIs answer with acknowledgements, order +IDs, truthy dicts, and — in at least one case — a transport error that is +byte-identical to a rejection. None of those are fills. + +--- + +## 2. Layering + +``` + worker.enter_position(...) + │ + ▼ + ExecutionSafetyCoordinator ── ONE shared, lock-guarded broker session + ├─ acquire the broker lock + ├─ rate limit + ├─ 10-second deadline ◄── INCLUDES the lock + rate-limit wait, not just the HTTP call + ├─ execution_ledger.record_attempt(...) ── BEFORE submission + ▼ + execution_client ── broker-agnostic surface (ADR-0002) + ├─ ensure_logged_in / is_logged_in / logout + ├─ preload_scrip_master / resolve_option_symbol + ├─ place_market_order / get_order_status / cancel_order + ├─ list_open_orders / list_open_positions + ├─ recover_after_reconciliation + └─ extract_order_id + │ + ├─ Dependencies/Kotak API/kotak_execution.py + ├─ Dependencies/Shoonya API/shoonya_execution.py (+ vendored NorenApi.py) + ├─ Dependencies/Flattrade API/flattrade_execution.py + └─ Dependencies/Dhan API/dhan_execution.py + │ + ▼ + typed OrderResult ──► execution_ledger applies cumulative filled qty as a DELTA +``` + +The runner only ever touches the generic `execution_client`. Adding a broker +means implementing that surface and adding a coverage row in +`scripts/check_coverage_thresholds.py` — the latter is part of adding a broker, +not an afterthought, because a broker without a row silently escapes the 80% +adapter budget. + +--- + +## 3. The contract — `Dependencies/broker_contract.py` + +Four normalized outcomes, and only four: + +| `OrderStatus` | Meaning | What the caller must do | +|---|---|---| +| `FILLED` | The requested quantity traded | Proceed | +| `PARTIAL` | Some traded, some did not | **Exposure exists.** Freeze new live entries; reconcile | +| `REJECTED` | The broker refused | Entry: fall back to paper **only if filled qty is zero**. Exit: the position stays open | +| `UNKNOWN` | The outcome could not be determined | Treat exactly like `PARTIAL` — exposure may exist | + +`OrderResult` carries the filled quantity, not just a status, because a status +without a quantity cannot answer "how much am I holding?". + +The module also pins itself into `sys.modules` under **both** +`broker_contract` and `Dependencies.broker_contract`. Adapters run both as repo +modules and as standalone diagnostic scripts that add `Dependencies/` to +`sys.path`; without this, the same class would be imported twice under two names +and `isinstance(result, OrderResult)` would be false at the live-order boundary. + +--- + +## 4. The ledger — `Dependencies/execution_ledger.py` + +State per live leg, thread-safe, quantity-bearing: + +- A leg is recorded **before** submission, so a lost response still leaves a + trace of what was attempted. +- Broker fill reports are **cumulative totals**, applied as deltas. Reports can + arrive late, out of order, or repeat; applying them as deltas means known + quantity can never silently disappear or double-count. +- The runner never infers exposure from a boolean. + +--- + +## 5. Startup and shutdown boundaries + +**Startup** — `Dependencies/startup_exposure.py` performs exactly two reads: +open orders and open index-option positions. It never adopts, cancels, recovers +or flattens. If either read shows exposure, live workers do not start. + +**Shutdown** — `Dependencies/trading_lifecycle.py` blocks new entries the moment +shutdown begins, then requires every tracked leg to be closed and the broker to +confirm flat before the session may be called clean. A failed close keeps the +process alive in reconciliation. + +Both boundaries are measured against **the runner's own ledger**, because the +operator trades manually in the same account. A non-flat *account* is advisory; +a non-flat *ledger* is an error. + +--- + +## 6. Contract resolution — `OptionsContractResolver` + +Maps (underlying, spot, offset, expiry rule) → a tradable option symbol. + +- Strike selection: ATM by default; some strategies take an ITM/OTM offset + (`CPR_ALGO3_ITM_OFFSET`, the SL Hunting mirror's near-expiry ITM steps). +- Expiry: the ATM family buys the **next-next** expiry. +- Dhan resolves contracts from the local `Dependencies/all_instrument .csv` + (refreshed by `_refresh_instrument_master_for_next_day()` at shutdown), not a + live download. +- Every adapter also exposes `resolve_option_symbol` for its own wire format. + +### 6.1 The BankNIFTY monthly exception + +BankNIFTY lists only monthly series, so the ATM family's "next-next" rule would +put an intraday mirror leg two months out. The SL Hunting mirror therefore +always trades the **nearest** monthly expiry and never rolls forward — Kotak +rejects MIS (intraday) orders on next-month contracts, which repeatedly killed +the live mirror leg (operator finding, 2026-07-23). Expiry week is handled on +the *strike* axis instead: inside the final week the mirror buys a deep-ITM +strike, which is mostly intrinsic value and tracks the index rather than +bleeding near-expiry time premium. + +--- + +## 7. Order splitting — `Dependencies/order_splitting.py` + +NSE caps how much of a contract may be sent in one order (the freeze quantity). +An order at or above it is rejected by the exchange. + +That is dangerous here specifically because a rejected live **entry** with zero +fill is deliberately treated as a paper fallback — so without splitting, an +oversized order would quietly trade on paper while the operator believed it was +live. `max_legal_chunk_units` / `split_order_quantity` keep every submission +legal. 90% branch-coverage budget. + +--- + +## 8. Broker-specific quirks the adapters exist to contain + +### Dhan + +- Its SDK returns `{'status': 'failure', 'remarks': str(exc)}` for **transport** + errors — shape-identical to a genuine rejection. So `REJECTED` is never + derived from the placement envelope: a `dict` `remarks` means the server + refused; a `str` means the outcome is indeterminate (`UNKNOWN`). +- `order_tag` is sent as Dhan's `correlationId`, so `get_order_by_correlationID` + can recover an order whose response was lost. +- Non-contract states are aliased adapter-locally: `EXPIRED`→`CANCELLED`, + `PART_TRADED`→`PARTIAL`. `TRANSIT` and `PENDING` stay unmapped so they remain + transient rather than being forced into a terminal state. +- The SDK ships a 60s default timeout; `_login_locked` overrides it down to 10s. + +### Shoonya + +- The `NorenApi` client is **vendored** under `Dependencies/Shoonya API/`. It is + never linted, never rewritten locally, and excluded from mypy and Bandit. +- Every HTTP call has an explicit timeout — a hung call would otherwise stall a + worker thread *and* the shared broker lock. + +### Kotak + +- The official v2 client comes from the `v2.0.1` Git tag and pins older + pandas/requests, so it lives in `requirements-brokers.txt` and is validated in + its own CI job rather than combined with the core set. + +### Flattrade + +- Pi v2 browser-token flow, exact NFO index scrip master, documented request + limits, market-order protection, and `SingleOrdHist` fill confirmation. + +### Dhan as both data and execution + +`DhanBrokerClient` (market data) and `dhan_execution_client` (orders) are +separate sessions on purpose, so a data-side token problem cannot silently +affect order placement. + +--- + +## 9. Timeouts + +Every broker network/SDK call has a **ten-second deadline that includes its +shared lock and rate-limit wait**. Native HTTP timeouts remain enabled for +Shoonya, Flattrade and Dhan on top of that. + +The "includes the wait" part is the design: a deadline that starts after the +lock is acquired does not protect the other 29 workers queued behind a hung +call. + +--- + +## 10. Testing + +| Suite | Covers | +|---|---| +| `Tests/Dependencies/test_broker_contract.py` | The four outcomes, the Protocol, the dual `sys.modules` registration | +| `Tests/Dependencies/test_execution_ledger.py` | Cumulative-delta application, thread safety | +| `Tests/Dependencies/test_startup_exposure.py` | Read-only audit semantics | +| `Tests/Dependencies/test_order_splitting.py` | Freeze-quantity arithmetic | +| `Tests/Dependencies/Dhan API/test_dhan_execution.py` | The transport-vs-rejection distinction, state aliasing | +| `Tests/Dependencies/Flattrade API/test_flattrade_execution.py` | Wire format, fill confirmation | +| `Tests/test_nifty_multi_strategy_master.py` | Paper/live routing, the fail-closed broker switch, coordinator behaviour | + +Coverage budgets: 90% for the safety modules, **80% for every broker adapter**, +enforced by `scripts/check_coverage_thresholds.py`. A module missing from the +coverage report is a **failure**, not a pass — otherwise renaming a file would +silently retire its budget. + +CI runs the broker contract and Flattrade adapter suites a second time in the +isolated `requirements-brokers.txt` environment. + +--- + +## 11. Adding a broker — checklist + +1. Implement the full `execution_client` surface (§2). No partial adapters. +2. Contain the broker's quirks inside the adapter; never leak a broker-specific + status into the runner. +3. Never derive `REJECTED` from anything that could also be a transport error. +4. Give every call an explicit HTTP timeout. +5. Add a diagnostic script (`diagnose__symbol.py`) using + `Dependencies/diagnostic_preflight.py`. +6. Add a row to `BROKER_THRESHOLDS` in `scripts/check_coverage_thresholds.py`. +7. Add the adapter to `mypy.files` in `pyproject.toml`. +8. Tests under `Tests/Dependencies/ API/`. diff --git a/docs/lld/market-data.md b/docs/lld/market-data.md new file mode 100644 index 0000000..8974338 --- /dev/null +++ b/docs/lld/market-data.md @@ -0,0 +1,182 @@ +# LLD — Market data: producers, shared store, health gates + +**Owns:** `CentralMarketDataFetcher`, `WebSocketMarketDataFetcher`, +`SharedMarketDataStore`, `MarketSnapshot`, `LTPSnapshot`, `OptionSubscription` +(master file) · `Dependencies/market_data_health.py` · +`Dependencies/tick_bar_builder.py` +**Consumed by:** every strategy worker +**Related ADR:** [0005 — REST vs websocket](../adr/0005-rest-vs-websocket-market-data.md) + +--- + +## 1. Responsibility + +Produce one authoritative view of the market that ~30 worker threads can read +concurrently, and make "is this data safe to trade on?" a question with exactly +one answer for all of them. + +The second half is the important half. If each strategy decided independently +whether the feed was stale, they would disagree, and some would trade on data +others had rejected. + +--- + +## 2. The shared store + +```python +SharedMarketDataStore # lock-guarded; one writer, many readers + ├─ 1-minute OHLC frames # per instrument (NIFTY spot, BankNIFTY, option legs) + ├─ LTP cache # per subscribed leg, with a timestamp + ├─ MarketDataHealth # freshness state, shared by all readers + └─ OptionSubscription set # which legs are currently subscribed +``` + +Readers take the lock, copy what they need, and release. Workers never hold the +lock across a decision, let alone across a broker call. + +`MarketSnapshot` and `LTPSnapshot` are the immutable value objects handed out — +a worker cannot accidentally mutate shared state by holding onto what it read. + +--- + +## 3. Two producers, one contract + +Exactly one producer thread runs per process. Selection is by +`MARKET_DATA_SOURCE`, and **any value other than `WEBSOCKET` yields REST** — +including typos. `_select_market_data_fetcher_class()` owns that decision. + +### 3.1 `CentralMarketDataFetcher` (REST, default) + +``` +loop: + sleep(poll interval, 2-5s) + GET intraday OHLC for the full window + normalize_dhan_intraday_response(resp) # epoch-unit inference, column mapping + validate_ohlc_frame(frame) # fail closed on bad geometry + build_last_row_signature(frame) # cheap change detection + store.publish(frame, ltps) +``` + +Simple, no state to recover, and every bar is by definition the official +exchange candle. Its cost is API load: one full-window pull every few seconds. + +### 3.2 `WebSocketMarketDataFetcher` (opt-in, needs the paid Data API) + +Two cooperating pieces: + +``` + pump thread supervisor + ─────────── ────────── + dhanhq.marketfeed packets ─► tick_bar_builder (pure helpers) + ├─ update the FORMING minute in real time + ├─ close the minute at the boundary + └─ update LTP per leg + once per minute: + REST fetch official candles + true-up completed bars ← official wins, + divergence logged + on connect / reconnect: + REST warmup + gap backfill +``` + +Legs are subscribed and unsubscribed dynamically as workers enter and exit +positions — including multi-leg baskets (hedged pairs, the Delta-0.2 four-leg +spread, strangle legs, the SL Hunting BankNIFTY mirror). + +**Why the true-up exists.** Tick-built candles and official candles can disagree +(missed packets, boundary handling, exchange corrections). Backtests were run on +official candles. Without the true-up, live bars would slowly stop being the +thing the strategies were validated against. The rule is deliberately blunt: +official always wins. + +**Why the tick logic is a separate pure module.** `Dependencies/tick_bar_builder.py` +holds no sockets, no threads and no clock of its own, so bar-boundary and +out-of-order-tick behaviour is unit-testable without a live feed. It carries a +90% branch-coverage budget for the same reason the REST validators do: it feeds +the same frames. + +**Rollback** is `MARKET_DATA_SOURCE=REST` plus a restart. No state migration. + +--- + +## 4. Validation and health gates + +`Dependencies/market_data_health.py` is the single authority. It is pure and +has no knowledge of threads or brokers. + +### 4.1 Candle validation — `validate_ohlc_frame` + +Rejects, rather than repairs: + +- non-finite or non-positive prices +- broken geometry (`high < low`, close outside `[low, high]`, …) +- duplicate or non-monotonic timestamps +- incomplete trailing minutes (`complete_minute_bucket_mask`, + `newest_completed_minute_timestamp`) + +A rejected frame is **not published**. Strategies keep seeing the last good +snapshot; they never see a bad one. Naive timestamps are treated as Asia/Kolkata. + +### 4.2 Freshness — `MarketDataHealth` + +Three independent thresholds, each answering a different question: + +| Gate | Question | Effect when breached | +|---|---|---| +| LTP staleness (~10s) | Is the price I would trade at current? | Refuse new entries | +| Bar staleness (~150s) | Is the candle stream alive? | Refuse new entries | +| Liquidation (~30s) | Has this gone on long enough that holding is worse than exiting blind? | Liquidate open positions | + +The websocket producer adds one twist: a quiet-but-subscribed leg (a real, +untraded option) is legitimately silent. Its LTP is treated as fresh **only +while the socket is demonstrably alive** — otherwise a dead socket would look +identical to a quiet strike. + +--- + +## 5. Resampling + +`resample_ohlc_from_1m(ohlc, timeframe_minutes)` turns the shared 1-minute +frames into whatever timeframe a strategy wants (5-minute for CPR, and so on). + +The invariant that matters: **candles are labelled by their START time**, and a +strategy acts only on *completed* candles. `Dependencies/next_open_entry.py` +encodes the consequence for `NEXT_OPEN` strategies — a signal born on a +completed candle gets exactly one bar of life and is rebased to the next +candle's open. + +--- + +## 6. Interfaces + +| Direction | Contract | +|---|---| +| Producer → store | validated frames + LTPs + health timestamps, under the lock | +| Store → worker | immutable `MarketSnapshot` / `LTPSnapshot` copies | +| Worker → store | subscribe/unsubscribe an `OptionSubscription` on entry/exit | +| Store → health | freshness timestamps; the health object answers the gates | + +--- + +## 7. Testing + +- `Tests/test_market_data_health.py` — validation and freshness rules (unittest). +- `Tests/Dependencies/test_tick_bar_builder.py` — pure tick→bar behaviour. +- `Tests/test_nifty_multi_strategy_master.py` — producer threads, store locking, + subscription lifecycle, and the fail-closed source selection. + +Both `market_data_health.py` and `tick_bar_builder.py` sit in the 90% +branch-coverage tier enforced by `scripts/check_coverage_thresholds.py`. + +--- + +## 8. Known limitations + +- **No volume in the feed.** Anything volume-weighted is a documented proxy + (Regime Adaptive's VWAP is equal-weight — see + [`regime-adaptive.md`](regime-adaptive.md)) or unimplemented. +- **NIFTY is assumed primary.** BankNIFTY is fetched per bar for confirmation + and mirroring, not as a co-equal underlying. Making a second index + first-class means parameterizing this component. +- **One producer per process.** There is no failover from websocket to REST + mid-session; recovery is restart with the flag flipped. diff --git a/docs/lld/master-runner.md b/docs/lld/master-runner.md new file mode 100644 index 0000000..30600fd --- /dev/null +++ b/docs/lld/master-runner.md @@ -0,0 +1,170 @@ +# LLD — Master runner: process lifecycle and thread supervision + +**Owns:** `Nifty Multi Strategy Front Test - Master File.py` (`main()` and the +startup/shutdown helpers around it) +**Depends on:** every other component +**Read first:** [`../hld/system-overview.md`](../hld/system-overview.md) §6 (concurrency), §7 (failure model) + +--- + +## 1. Responsibility + +`main()` is the only place that knows the *order* in which the system must come +up and go down. Its contract: + +1. Nothing that can place an order starts before configuration is validated and + the broker book is proven clean. +2. Nothing reports a clean shutdown until every tracked leg is closed and the + broker agrees the account is flat. + +Everything between those two statements is supervision. + +--- + +## 2. Startup sequence + +``` +main() + │ + 1. setup_logging() + │ └─ install_redaction_filter(environment_secrets(os.environ)) on the ROOT logger + │ ── so every record from here on is scrubbed, including tracebacks + │ + 2. read + validate configuration + │ ├─ _env_str / _env_bool / _env_int / _env_float (plain knobs) + │ ├─ _scaled_int / _scaled_float (size-bearing knobs) + │ └─ _live_config_errors(...) ── malformed size knobs BLOCK that + │ strategy from live; paper falls back to 1 + │ + 3. _configure_startup_live_trading(...) + │ ├─ LIVE_TRADING_ENABLED false ──► everything paper, skip the rest + │ ├─ _select_execution_client(LIVE_BROKER) + │ │ └─ unknown name ──► FAIL CLOSED: live disabled, paper only + │ └─ _cpr_ai_startup_errors() etc. for optional agents + │ + 4. startup exposure audit (Dependencies/startup_exposure.py) + │ ├─ read open ORDERS ─┐ + │ └─ read open POSITIONS┴─► both must be clean for live workers to start + │ (read-only: never adopts, cancels, recovers or flattens) + │ └─ _enqueue_startup_exposure_alert(...) tells the operator either way + │ + 5. build the worker list + │ └─ filter by _strategy_virtual_trading_enabled(name) + │ ── _VIRTUAL_TRADING=false means the thread never starts, + │ so the strategy does neither paper nor live + │ + 6. _start_and_supervise_runtime_threads(...) + ├─ market data producer thread (exactly one, see market-data.md) + ├─ TelegramMessageWorker (if enabled) + └─ N strategy worker threads +``` + +### 2.1 Why the exposure audit is read-only + +The temptation is to have the runner "clean up" a stale position it finds at +startup. It deliberately does not. A position in the account may belong to the +operator's own manual trading in the same account — the runner has no way to +tell whose it is, and flattening someone else's position is worse than refusing +to start. The audit therefore reports and refuses; a human decides. + +This is also why end-of-day gates are measured against **the runner's own +ledger**, not the account books. + +--- + +## 3. Supervision + +`_start_and_supervise_runtime_threads` keeps the process alive while threads +run, and watches for the conditions that should end the session. Worker threads +are cooperative: they check a shutdown flag on their own poll cadence rather +than being interrupted, because interrupting a thread mid-order is exactly the +state the whole safety model exists to avoid. + +`_request_worker_shutdown` sets the flag **per worker**. The scope is deliberate: +one strategy hitting its daily max-loss stops that strategy, not the session. + +--- + +## 4. Shutdown sequence + +``` +shutdown requested (end of day, max loss, operator, or fatal supervision event) + │ + 1. TradingLifecycle → SHUTTING_DOWN + │ └─ new entries blocked process-wide from this instant + │ + 2. each worker closes its own tracked legs + │ └─ a REJECTED exit does NOT count as closed + │ + 3. _wait_for_shutdown_account_flat(...) + │ └─ ask the broker whether the account is flat + │ ├─ flat ──► continue + │ └─ not flat ──► stay alive in RECONCILIATION; do NOT report clean + │ + 4. _warn_if_account_not_flat / _advisory_account_audit + │ └─ advisory only: the operator may hold manual positions in the same + │ account, so a non-flat ACCOUNT is a warning, not an error, as long as + │ the runner's own ledger is flat + │ + 5. _finalize_flat_session(...) + ├─ _publish_eod_summary(...) → Telegram + ├─ _update_pnl_google_sheet() → per-strategy P&L row/column + └─ _refresh_instrument_master_for_next_day() +``` + +Step 3 is the one that matters. "Stopping the threads" and "safely stopping a +live trading process" are different things, and `Dependencies/trading_lifecycle.py` +exists to keep them separate: the process refuses to claim a clean exit while a +leg it opened is still open. + +--- + +## 5. Module loading + +Most strategy files have spaces in their names and cannot be imported normally. +`load_module(module_name, file_path)` wraps `importlib.util.spec_from_file_location` +so the runner can load them by path. Consequences worth knowing: + +- The master itself is loaded the same way by its test suite. +- mypy cannot type-check spaced-name files; they are covered by `compileall` + plus the unittest suite instead (see [`testing-and-ci.md`](testing-and-ci.md)). +- Anything imported this way is **not** on `sys.path` for its siblings, which is + why the agent folders carry their own path bootstraps. + +See [ADR-0009](../adr/0009-importlib-loading-for-spaced-filenames.md). + +--- + +## 6. Failure handling in this component + +| Condition | Behaviour | +|---|---| +| Unknown `LIVE_BROKER` | Live disabled entirely; the session runs paper. Never guesses a broker. | +| Malformed `_SIZE_MULTIPLIER` | Paper uses 1; that strategy is blocked from live by `_live_config_errors`. | +| Exposure found at startup | Live workers do not start. Operator is alerted. | +| A worker thread dies | Supervision notices; the session does not silently continue believing that strategy is running. | +| Broker not flat at shutdown | Process stays in reconciliation rather than exiting clean. | +| Google Sheet / Telegram unconfigured | Safe no-op. Reporting never blocks or fails shutdown. | + +--- + +## 7. Testing + +`Tests/test_nifty_multi_strategy_master.py` loads this file via `importlib` with +`dhanhq` mocked, then drives the startup and shutdown paths directly. The suite +is the primary gate for this component because mypy cannot see the file. + +Areas it specifically covers: env toggles, the fail-closed `LIVE_BROKER` switch, +paper/live routing, order fill-confirmation, symbol resolution, and the +shutdown-flatten sequence. + +--- + +## 8. Known debt + +- **One 17k-line file.** Cohesive but slow to change. A split is viable but must + be behaviour-preserving and would need the test suite's `importlib` loading + reworked in the same change. +- **In-process ledger.** A mid-session crash leaves recovery to operator-driven + reconciliation. Persisting the ledger is the natural next step if unattended + running is ever wanted. diff --git a/docs/lld/regime-adaptive.md b/docs/lld/regime-adaptive.md new file mode 100644 index 0000000..558543b --- /dev/null +++ b/docs/lld/regime-adaptive.md @@ -0,0 +1,102 @@ +# LLD — Regime Adaptive router + +**Owns:** `Signal Generators/Regime Adaptive Strategy/` +**Ported from:** [`workratananmol-hub/nifty-options-paper-trading-bot`](https://github.com/workratananmol-hub/nifty-options-paper-trading-bot) (MIT) +**Read before enabling live:** [`REGIME_PORTING_NOTES.md`](../../Signal%20Generators/Regime%20Adaptive%20Strategy/REGIME_PORTING_NOTES.md) + +--- + +## 1. Responsibility + +One worker that switches its trading **rule** based on the measured regime, +instead of running a single rule and hoping the market suits it. + +``` + read ADX on each completed bar + │ + ├─ trending ──► opening-range breakout, confirmed by VWAP + ├─ ranging ──► fade back to VWAP + └─ ADX missing ──► NO TRADE +``` + +The third branch is the interesting one: it **never guesses the regime**. A +missing ADX is not treated as "probably ranging" — it is treated as "I don't +know", and not knowing means not trading. + +--- + +## 2. Structure + +| File | Role | +|---|---| +| `Nifty Regime Adaptive Signal Generator.py` | The router: reads ADX, selects a candidate, returns the signal | +| `regime_candidates.py` | The two candidate rules, as **library code** | +| `regime_common.py` | Shared helpers; re-exports indicators from `misc_strategy_common` | +| `REGIME_PORTING_NOTES.md` | What was and was not ported, and why | + +### 2.1 Why the candidates have no worker of their own + +`regime_candidates.py` is deliberately library code with **no worker**. If the +breakout rule also ran as its own strategy, the router and that strategy could +take the *same* signal at the same moment — doubling size on one idea while the +roster appeared diversified. Keeping the candidates worker-less makes that +impossible by construction rather than by convention. + +If a candidate is ever wanted as a standalone strategy, the router must gain an +explicit exclusion; do not simply add a worker. + +--- + +## 3. Wiring + +Regime Adaptive is the fourteenth strategy built through +`_build_signal_gen_worker_class` (the same factory as the 13 TradingBot ports), +so it is an ordinary `AtmSingleLegStrategyWorker` from the runner's point of +view. Knobs are `REGIME_ADAPTIVE_*`. + +It is also the **first and only user of the shared bid/ask spread gate** at a +non-zero default: `_MAX_SPREAD_PCT` is `2.0` here and `0` (off) for +every other strategy, so introducing the gate changed nothing else. See +[`risk-and-safety.md`](risk-and-safety.md) §5. + +--- + +## 4. Porting gaps — read before enabling live + +These are the honest differences from the source project. They are recorded here +and in `REGIME_PORTING_NOTES.md` because a ported strategy that quietly differs +from its origin is a trap. + +| Source behaviour | Here | Why | +|---|---|---| +| Volume-weighted VWAP | **Equal-weight proxy** | This runner's feed carries no volume. | +| India VIX veto | **Not implemented** | Absent *by choice*, not for want of data — the source project also runs on Dhan. | +| Market-breadth veto | **Not implemented** | Same. | + +The VWAP proxy matters most to the ranging branch, which fades *to* VWAP. A +proxy VWAP is a different line from a true one, so the fade target is not +identical to the source's. + +--- + +## 5. Testing + +`Tests/Signal Generators/Regime Adaptive Strategy/test_regime_adaptive.py`, with +a `conftest.py` that puts both the **source** strategy folder and its parent +`Signal Generators/` on `sys.path` — the parent because `regime_common` +re-exports shared indicators from `misc_strategy_common` one level up. At runtime +`regime_common` bootstraps that itself; under pytest the import can arrive +through a different entry point, so it is done in the conftest as well. + +Coverage: the router's three branches, including the ADX-missing no-trade path, +are the cases that matter most. + +--- + +## 6. Known limitations + +- Equal-weight VWAP (§4) — the single biggest divergence from the source. +- Two vetoes unimplemented (§4). Adding them is a real improvement, not a port + fix; they would need a VIX feed and a breadth source wired in. +- The regime read is ADX-only. A second regime input would change the router's + contract and should be an ADR, not a quiet edit. diff --git a/docs/lld/reporting-and-observability.md b/docs/lld/reporting-and-observability.md new file mode 100644 index 0000000..01859e8 --- /dev/null +++ b/docs/lld/reporting-and-observability.md @@ -0,0 +1,137 @@ +# LLD — Reporting and observability + +**Owns:** `TelegramMessageWorker`, `format_trade_message`, `_publish_eod_summary`, +`_parse_eod_pnl_by_day`, `_compute_pnl_sheet_updates`, `_update_pnl_google_sheet`, +`setup_logging` (master file) + +--- + +## 1. Responsibility + +Tell the operator what happened — during the session (Telegram), after it +(Google Sheet), and forensically (the log) — **without ever being able to affect +trading**. + +Every design choice here follows from that last clause. Reporting is +best-effort, off the trading path, and a safe no-op when unconfigured. + +--- + +## 2. Telegram alerts + +``` + worker (any) queue.Queue TelegramMessageWorker + ──────────── ─────────── ───────────────────── + entry / exit ──► event dict ──► put_nowait() ──────────► get() → format → POST + (unbounded, non-blocking) │ + └─ failure: log and continue +``` + +The queue is the whole point. Telegram latency or downtime can never block a +trading decision, because the worker thread never waits on the network — it +hands off a dict and returns to its loop. + +Alerts carry the strategy, the exact option instrument(s), lot size, entry and +exit price, and P&L. Hedged spreads show both legs. `_format_inr` and +`_execution_mode_parts` / `_combined_execution_mode` keep the formatting +consistent, including the PAPER / LIVE / MIXED labelling. + +Disabled (`TELEGRAM_ENABLED=false`, the default) it is a cheap no-op — the +worker is not started and events are dropped. + +--- + +## 3. End-of-day P&L to Google Sheets + +On a clean end of day, the runner writes each strategy's realised P&L into a +tracker sheet: one row per strategy, one column per calendar day. + +``` + rotating log file (append mode) + │ + ▼ + _parse_eod_pnl_by_day(log_path, today) ← parses the RUN'S OWN LOG + │ _asctime_in_pnl_window filters to + │ the session; _normalize_pnl_strategy_name + │ maps log labels to sheet rows + ▼ + _compute_pnl_sheet_updates(values, pnl_by_day, today) + │ ├─ overwrite today's cell + │ └─ backfill BLANK earlier-this-month cells from the same log + ▼ + gspread (OAuth user token) ──► the sheet +``` + +### 3.1 Why it parses a log instead of reading memory + +Reading in-memory totals would be simpler and would also be wrong in the case +that matters: a session that ended messily. The append-mode log is the durable +record of what actually happened, so parsing it means a partial session still +reports honestly, and blank earlier-in-the-month cells can be backfilled without +re-running anything. + +### 3.2 Row labelling + +PAPER results use the existing row labels in column A (e.g. `Renko Strategy`). +LIVE and MIXED results use **separate** rows — `Renko Strategy [LIVE]`, +`Renko Strategy [MIXED]` — so real-money outcomes can never contaminate paper +history. Unmatched strategies are skipped with a warning rather than written to +a guessed row. + +Auth is OAuth user-token via `gspread` (`GSHEET_OAUTH_CLIENT_FILE`, +`GSHEET_OAUTH_TOKEN_FILE`, both gitignored). Leave `GSHEET_ID` blank to disable — +a safe no-op that never disturbs shutdown. + +--- + +## 4. Logging + +`setup_logging()` returns the root logger with two things attached: + +1. Console + append-mode file handlers. +2. **`install_redaction_filter(environment_secrets(os.environ))`** — see + [`risk-and-safety.md`](risk-and-safety.md) §8. Every record is scrubbed, + including lazy `%s` args and exception tracebacks. + +Conventions: + +- Library code uses a module-level `logging.getLogger(__name__)`, **never + `print()`**. +- Do not hand-redact new call sites; the root filter covers them. +- `*.log` is gitignored. + +The log is load-bearing, not decorative: the EOD sheet is derived from it (§3), +so a change to a trade log line's format is a change to the P&L pipeline. + +--- + +## 5. Failure behaviour + +| Failure | Effect on trading | +|---|---| +| Telegram down / token wrong | None. Logged, event dropped. | +| Google Sheet unreachable | None. Shutdown completes; the day can be backfilled on a later run. | +| Sheet row label unmatched | That strategy is skipped with a warning; nothing is written to a guessed row. | +| Log file unwritable | Console logging continues; the EOD sheet loses its source for that session. | + +--- + +## 6. Testing + +- `Tests/test_nifty_multi_strategy_master.py` covers `format_trade_message`, + the queue worker, `_parse_eod_pnl_by_day`, `_compute_pnl_sheet_updates` and + the PAPER/LIVE/MIXED labelling. +- `Tests/Dependencies/test_secret_redaction.py` covers the redaction filter. + +Neither Telegram nor Google Sheets is contacted in tests. + +--- + +## 7. Gaps + +- **No metrics or health endpoint.** Observability is a log file plus Telegram. + For a single-operator, single-session system that is proportionate; it would + not be if the runner ever ran unattended. +- **P&L is derived from log parsing**, so log-line format is an implicit + contract. If the runner ever grows a structured event store, this is the first + thing that should move to it. diff --git a/docs/lld/risk-and-safety.md b/docs/lld/risk-and-safety.md new file mode 100644 index 0000000..43619d6 --- /dev/null +++ b/docs/lld/risk-and-safety.md @@ -0,0 +1,220 @@ +# LLD — Risk, sizing, and the live-trading safety model + +**Owns:** `Dependencies/risk_sizing.py`, `Dependencies/next_open_entry.py`, +`Dependencies/secret_redaction.py`, `_strategy_size_multiplier` / `_scaled_*` / +`_live_config_errors` / `_spread_gate_allows_entry` (master file) +**Related ADRs:** [0003](../adr/0003-acknowledgement-is-not-a-fill.md), [0004](../adr/0004-paper-by-default-double-gate.md), [0006](../adr/0006-per-strategy-size-multiplier.md) + +> **This is live-money code.** Every rule below exists because the alternative +> costs real money. When in doubt, the safe direction is to refuse the trade. + +--- + +## 1. The one principle + +**Ambiguity is treated as the dangerous case.** Not the neutral case, not the +optimistic case. Concretely: + +- An unknown broker name does not mean "pick the default" — it means live + trading is off for the session. +- An unreadable bid/ask quote does not mean "the spread is probably fine" — it + means the live entry is refused. +- An acknowledged order does not mean "it filled" — it means the fill is unknown + until proven, and unknown means exposure may exist. +- A malformed size multiplier does not mean "assume 1 and carry on live" — paper + gets 1, live gets blocked. + +--- + +## 2. The live-trading double gate + +A strategy places a real order only when **both** are true: + +``` +LIVE_TRADING_ENABLED = true # global kill-switch, default FALSE +_LIVE_TRADING = true # per strategy, default FALSE +``` + +Plus `LIVE_BROKER` ∈ {`KOTAK`, `SHOONYA`, `FLATTRADE`, `DHAN`}. **An unknown +value fails closed** — live disabled, paper only. + +The mirror-image gate for paper is deliberately *asymmetric*: + +| | Global switch | Per-strategy | Default | +|---|---|---|---| +| **Live** | `LIVE_TRADING_ENABLED` (required) | `_LIVE_TRADING` (required) | off | +| **Virtual/paper** | *none by design* | `_VIRTUAL_TRADING` | on | + +There is no global paper switch because "run everything" is the safe default for +paper and "run nothing live" is the safe default for live. One flag can never be +enough to risk money; one flag is enough to silence a strategy. + +See [ADR-0004](../adr/0004-paper-by-default-double-gate.md). + +--- + +## 3. Position sizing — `Dependencies/risk_sizing.py` + +`SizingDecision` is the single authority for both the master runner and the +standalone SL Hunting executor. + +``` +lots = floor(risk_budget / (stop_distance × lot_size)) + capped at _MAX_LOTS +``` + +Three deliberate behaviours: + +1. **Floor, never ceil, and never force a minimum of one lot.** The older + per-strategy helpers used `ceil` and forced at least one lot; both can exceed + the configured budget. +2. **One-lot-over-budget is an explicit rejection**, not a rounded-down trade. + If the smallest legal size costs more than the budget allows, the setup is + skipped. +3. **Invalid inputs are rejections**, not defaults. + +--- + +## 4. Per-strategy size multiplier + +`_SIZE_MULTIPLIER` (default 1, whole numbers 1–25, capped by +`MAX_SIZE_MULTIPLIER`) scales that strategy's whole size/risk set *together*: +`_LOTS`, `_MAX_LOTS`, `_RISK_BUDGET` and the absolute `_MAX_LOSS`. + +Applied at **env-read time** through `_scaled_int` / `_scaled_float` / +`_strategy_size_multiplier`, so `risk_sizing.py` is untouched and scaled values +flow through sizing, the kill-switch, Telegram and the Sheet unchanged. + +| Property | Reason | +|---|---| +| Per-strategy only, **no global switch** | One typo must not enlarge every enabled strategy. | +| Applies to **paper and live alike** | An enlarged size can be paper-validated before it risks money. | +| Malformed → 1 for paper, **blocked from live** | Guessing a size is worse than refusing. | + +**Two knobs are deliberately NOT scaled**, because their totals already inherit +the multiplier and scaling them would square it: + +- `_MAX_LOSS_PER_LOT` (Delta20) — multiplied by scaled lots downstream. +- `_STARTING_CAPITAL` / `_DAILY_MAX_LOSS_PCT` — their *product* carries it. + +A drift-guard test fails if a new strategy reads a size-bearing knob with the +raw `_env_*` helpers instead of `_scaled_*`. + +Two consequences worth knowing: the scaled budget also loosens the +"one lot exceeds the budget" skip; and because lots are floored, a 2× can land +slightly above a pure doubling while staying strictly inside the scaled budget. + +--- + +## 5. The bid/ask spread gate + +Every strategy here **buys** options, so a 2%-of-mid spread is a 2% loss booked +at the instant of entry, before the idea has done anything. + +`_spread_gate_allows_entry` reads `top_bid_price` / `top_ask_price` off the +`/optionchain` response for the exact strike and expiry being bought and refuses +an entry quoted wider than `_MAX_SPREAD_PCT`. + +| Situation | Paper | Live | Why | +|---|---|---|---| +| Spread too wide | refuse | refuse | It is a market fact, so paper rows stay predictive. | +| Quote unreadable | allow (warn) | **refuse** | An API failure should not cost a paper data point — but it also should not spend real money on a check that did not run. | + +Workers share one 3-second cache (`_fetch_option_chain_cached`) because Dhan +allows a single option-chain request per 3 s per underlying/expiry. + +Default is `0` (off) for every strategy **except Regime Adaptive** (2.0), so +introducing the gate changed no existing strategy's behaviour. + +--- + +## 6. Kill-switches and time gates + +| Control | Scope | Effect | +|---|---|---| +| `_MAX_LOSS` (scaled) | one strategy, one day | That worker stops; the session continues. | +| `LIVE_TRADING_ENABLED=false` | process | No real orders at all. | +| Square-off time (15:15 by default) | one strategy | Close everything, including both legs of a basket. | +| Entry cutoff | one strategy | No **new** positions; exits and square-off still run. | +| Stale-data liquidation (~30 s) | one strategy | Close rather than hold blind. | +| Startup exposure audit | process | Live workers do not start into a dirty book. | +| Post-exit cooldown (SL Hunting) | one strategy | Blocks re-entry from the moment the **whole** basket is confirmed flat. | + +Note the difference between an entry cutoff and a square-off. SL Hunting's +10:30 `NO_NEW_ENTRY` is a cutoff: open positions, their stops and targets, and +the 15:15 square-off are all unaffected. When flat past the cutoff it skips the +LLM call entirely. + +--- + +## 7. `NEXT_OPEN` signal safety — `Dependencies/next_open_entry.py` + +Goldmine and Money Machine generate a setup on a *completed* candle but may only +enter at the **open of the following candle**. Two rules make that safe: + +1. **Exactly one bar of life.** Candles are labelled by their START time, so a + pending signal that is not taken on the very next bar expires. It cannot be + silently carried into a different market. +2. **Price rebasing.** The entry reference is the next bar's open, not the + signal bar's close. + +--- + +## 8. Credential-safe logging — `Dependencies/secret_redaction.py` + +`setup_logging()` installs `install_redaction_filter` on the **root** logger with +`environment_secrets(os.environ)` — every `.env` value whose KEY looks sensitive +and is ≥8 characters. Every record is scrubbed before it reaches the console or +the append-mode log, including lazy `%s` args and exception tracebacks. + +This is not theoretical: `dhanhq`'s marketfeed puts the live access token **in +its websocket URL**, so a connect error would otherwise write it verbatim into a +log operators routinely share. + +Short values (a 4-digit MPIN) are deliberately excluded from exact-match +replacement — they would blank strike prices and quantities — and are caught by +`redact_text`'s `name=value` pass instead. + +**Do not hand-redact new call sites.** The root-logger filter covers them; a +local redaction is one more thing to forget. + +--- + +## 9. Order-outcome rules (restated, because they are the ones that cost money) + +| Result | Entry | Exit | +|---|---|---| +| `FILLED` | Position open | Position closed | +| `REJECTED` **with zero fill** | Fall back to paper | Position **stays open** | +| `REJECTED` with non-zero fill | Exposure exists — treat as `PARTIAL` | Partially closed | +| `PARTIAL` | Freeze new live entries, keep exits available, reconcile | Partially closed | +| `UNKNOWN` | Same as `PARTIAL` | Same as `PARTIAL` | + +**Never** treat an acknowledgement, a truthy value, or an order ID as proof of +fill. + +--- + +## 10. Testing + +Every module named here carries a **90% branch-coverage budget** enforced by +`scripts/check_coverage_thresholds.py`: + +`risk_sizing.py`, `next_open_entry.py`, `order_splitting.py`, +`secret_redaction.py`, `broker_contract.py`, `execution_ledger.py`, +`startup_exposure.py`, `trading_lifecycle.py`, `market_data_health.py`, +`tick_bar_builder.py`. + +Suites: `Tests/Dependencies/test_risk_sizing.py`, +`test_next_open_entry.py`, `test_secret_redaction.py`, plus the double-gate, +spread-gate and multiplier cases in `Tests/test_nifty_multi_strategy_master.py`. + +--- + +## 11. When changing anything in this document + +1. Ask which direction the failure falls. If the answer is "it trades", stop. +2. Paper and live may differ **only** when the failure is an infrastructure + problem rather than a market fact. +3. Add the test before the change. +4. Paper-validate for at least one session before enabling live. diff --git a/docs/lld/sl-hunting-ai-agent.md b/docs/lld/sl-hunting-ai-agent.md new file mode 100644 index 0000000..2f9e24b --- /dev/null +++ b/docs/lld/sl-hunting-ai-agent.md @@ -0,0 +1,216 @@ +# LLD — SL Hunting AI Agent (optional, Claude) + +**Owns:** `Signal Generators/SL Hunting AI Agent/` · `SLHuntingAIWorker` (master file) +**Status:** optional, **off by default** (`SL_HUNTING_ENABLED`), paper unless +explicitly enabled +**Related ADR:** [0007 — LLM agents as opt-in workers](../adr/0007-llm-agents-as-opt-in-workers.md) +**Operator detail:** the folder's own [`README.md`](../../Signal%20Generators/SL%20Hunting%20AI%20Agent/README.md) + +--- + +## 1. Responsibility + +Trade the discretionary "SL Hunting" price-action method on NIFTY ATM options by +asking a Claude agent (via `claude-agent-sdk`, on a Claude subscription — **no +API key**) for one decision per completed 1-minute bar. + +The design question this component answers is: *how do you let a language model +trade real money without letting it be dangerous?* The answer is the division of +labour in §3 — the model judges **premise**, the host owns **everything +mechanical**. + +--- + +## 2. Modules + +| File | Role | +|---|---| +| `sl_hunting_agent.py` | The agent itself: prompt assembly, SDK call, `SLHuntingDecision` | +| `sl_hunting_knowledge.py` | The curated method knowledge injected into the prompt | +| `sl_hunting_tools.py` | The frozen tool surface the agent may call | +| `sl_hunting_executor.py` | `MasterWorkerExecutor` / `StandaloneExecutor` — routes agent actions into the host's `enter_position` / `exit_position` | +| `sl_hunting_indicators.py` | Indicator helpers incl. `cross_index_signal` (BankNIFTY confirmation) | +| `sl_hunting_journal.py` | Per-trade journal — the input to learning | +| `sl_hunting_coach.py` | Off-loop reflection coach that proposes lessons | +| `sl_hunting_lessons.py` | `lessons.json` load/validate; human-gated promotion | +| `sl_hunting_premarket.py` | Pre-open note (`premarket_note.json`) | +| `sl_hunting_ai_validation.py` | Schema/decision validation | +| `sl_hunting_runner.py` | Standalone runner for offline replay | + +Dependencies are **lazily imported**: a missing `claude-agent-sdk` simply +disables the strategy (`SLHuntingAIWorker` stays `None` and `main()` skips it). +It never breaks the run. + +--- + +## 3. Division of labour + +``` + ┌──────────────────────── the model decides ────────────────────────┐ + │ Is there a real level here? │ + │ Is this a genuine stop-hunt setup or noise? │ + │ Does BankNIFTY confirm the NIFTY read? │ + │ Is the premise for this position still alive? (per leg) │ + │ Direction: LONG / SHORT / HOLD │ + └───────────────────────────────────────────────────────────────────┘ + ┌──────────────────────── the HOST decides ─────────────────────────┐ + │ Position size risk_sizing.SizingDecision, floored lots, │ + │ never over SL_HUNTING_RISK_BUDGET, │ + │ capped at SL_HUNTING_MAX_LOTS (default 5) │ + │ Stop / target mechanical, checked every loop │ + │ Daily max loss scaled kill-switch │ + │ Entry cutoff 10:30 by default — no NEW positions │ + │ Square-off 15:15 — closes both legs │ + │ Post-exit cooldown time-based, enforced in code │ + │ Stale-data liquidation shared health gates │ + │ Paper vs live the standard double gate │ + │ The BankNIFTY mirror mechanical; the agent does not choose it │ + └───────────────────────────────────────────────────────────────────┘ +``` + +The agent **does not choose lots**. It supplies an underlying stop distance; the +host floors affordable whole lots from it. + +--- + +## 4. The decision loop + +``` +every completed 1-min bar: + if flat and past the 10:30 cutoff: skip the LLM call entirely ← saves tokens + build context: NIFTY 1-min frame + BankNIFTY (fetched per bar, + like CPR Algo 3) + open-position state + + knowledge + optionally lessons.json + ask the agent ──► SLHuntingDecision (schema-validated) + │ + ├─ ENTER (NIFTY only) ──► host gates ──► enter_position(NIFTY ATM) + │ └─► mechanical mirror: equal-lot + │ BankNIFTY ATM leg + ├─ EXIT ──► exit_leg selector: NIFTY | BNF | BOTH + └─ HOLD ──► nothing + │ + └─ ANY exception ──► safe HOLD + (the mechanical risk loop keeps running regardless) +``` + +**Fail-soft is the whole safety posture for the model half.** Any agent or SDK +error becomes a HOLD. The separate mechanical loop keeps checking stop, target, +max-loss, stale data and square-off whether or not the model ever answers. + +--- + +## 5. The BankNIFTY mirror + +`SL_HUNTING_BNF_MIRROR` (default **true**). Every NIFTY entry is mirrored with an +equal-lot BankNIFTY ATM leg. + +| Aspect | Behaviour | +|---|---| +| Entry | **NIFTY only.** The mirror copies it; the agent never enters BankNIFTY directly. | +| Hard risk | **Tied.** Stop, target, max-loss and the 15:15 square-off close both legs. | +| Premise | **Independent.** The agent evaluates each leg separately and can cut one alone via the EXIT `exit_leg` selector (`NIFTY` \| `BNF` \| `BOTH`). | +| Expiry | Always the **nearest monthly** — never rolls forward (see [`execution-and-brokers.md`](execution-and-brokers.md) §6.1). | +| Expiry week | Switches to a deep-ITM strike (`..._NEAR_EXPIRY_ITM_STEPS`) once fewer than `..._ROLLOVER_DAYS` remain. | +| Failure | Fail-soft: any mirror problem skips the mirror, never the NIFTY leg. | + +> ⚠️ **The mirror roughly DOUBLES the basket's rupee risk beyond +> `SL_HUNTING_RISK_BUDGET`.** This is operator-accepted, and the daily max-loss +> kill-switch still caps the day. With `SL_HUNTING_SIZE_MULTIPLIER=M`, the basket +> sits near **2 × M** times the single-leg budget. Anyone changing sizing here +> must account for that. + +--- + +## 6. Post-exit cooldown + +`SL_HUNTING_POST_EXIT_COOLDOWN_MINUTES` blocks a new entry after a target, stop, +or premise-invalidating exit. Three properties, each of which was a bug first: + +1. The timer starts only from the moment **the whole NIFTY/BankNIFTY basket is + confirmed flat**. A lone or partly closed leg does not run it down. +2. **Exits never consult it.** A cooldown must never prevent closing a position. +3. **Corrupt guard state rejects new LIVE entries** rather than defaulting to + "allowed". + +It exists in code because the prompt's judgement-based version was talked past +twice in the live journal (23 and 27 Jul 2026) by relabelling the same price +structure as a fresh setup. Rules the model must not reason its way around +belong in the host. + +--- + +## 7. Knowledge and learning + +**Knowledge** (`sl_hunting_knowledge.py`) is curated, versioned method content +injected into the prompt — including a `BNF_SPECIFIC` section (triple-index +BNF + NIFTY + Sensex read, BankNIFTY as the "major index", expiry-day priority, +round-number magnets). That section is **advisory context for the cross-index +read only — execution stays NIFTY-only.** Provenance is recorded in +`sl_hunting_doc.md`. + +**Learning** is a deliberately slow, human-gated loop: + +``` + live trade ──► sl_hunting_journal.py (per-trade record) + │ OFF the trading loop + ▼ + sl_hunting_coach.py tool-free, schema-validated reflection + │ proposes candidate lessons + ▼ + OPERATOR REVIEW digest-bound approval + │ + ▼ + lessons.json ──► injected into the prompt ONLY when + SL_HUNTING_LESSONS_ENABLED (default false) +``` + +No lesson reaches the prompt without a human promoting it. The coach runs off +the trading loop so reflection can never delay a decision. + +--- + +## 8. Configuration + +| Key | Default | Meaning | +|---|---|---| +| `SL_HUNTING_ENABLED` | false | Master switch for the whole agent | +| `SL_HUNTING_LIVE_TRADING` | false | Second half of the live double gate | +| `SL_HUNTING_RISK_BUDGET` | 2500 | Rupee risk for the **NIFTY leg** (mirror doubles the basket) | +| `SL_HUNTING_MAX_LOTS` | 5 | Hard cap | +| `SL_HUNTING_SIZE_MULTIPLIER` | 1 | Scales budget, lots, max-loss together | +| `SL_HUNTING_BNF_MIRROR` | true | Mechanical BankNIFTY mirror | +| `SL_HUNTING_BNF_MIRROR_ROLLOVER_DAYS` | 7 | Days-to-expiry at which the mirror switches to ITM strikes (the name is legacy — it no longer rolls) | +| `SL_HUNTING_BNF_MIRROR_NEAR_EXPIRY_ITM_STEPS` | 4 | How deep ITM in that window | +| `SL_HUNTING_NO_NEW_ENTRY_HOUR` / `_MINUTE` | 10:30 | Entry cutoff, **not** a square-off | +| `SL_HUNTING_POST_EXIT_COOLDOWN_MINUTES` | — | Re-entry block after a flat basket | +| `SL_HUNTING_LESSONS_ENABLED` | false | Inject `lessons.json` into the prompt | + +Setup: `pip install -r requirements-ai.txt`, then a one-time `claude setup-token`. +Keep `ANTHROPIC_API_KEY` **unset** so it bills the Claude plan rather than +per-token API usage. + +--- + +## 9. Testing + +`Tests/Signal Generators/SL Hunting AI Agent/` — agent, indicators, journal, +lessons, premarket, runner, schema, and the v2 behavioural suite. Its +`conftest.py` puts the **source** agent folder on `sys.path` (the folder name has +spaces and the modules import each other by bare name). + +The SDK is never called in tests; decisions are injected. Master-side wiring +(mirror, cooldown, cutoff, executor routing) is covered in +`Tests/test_nifty_multi_strategy_master.py`. + +--- + +## 10. Known limitations + +- **Non-deterministic.** The same bar can produce different decisions. The + mechanical half is what makes that acceptable; do not move a rule from the host + to the prompt. +- **Cost and latency** are bounded by skipping the call when flat past the cutoff. +- **Basket risk is ~2× the named budget** (§5). This surprises people; it is + documented in four places for that reason. +- **Learning is human-gated on purpose.** Automating lesson promotion would let + the agent rewrite its own instructions from its own losses. diff --git a/docs/lld/strategy-workers.md b/docs/lld/strategy-workers.md new file mode 100644 index 0000000..772b8e5 --- /dev/null +++ b/docs/lld/strategy-workers.md @@ -0,0 +1,189 @@ +# LLD — Strategy workers + +**Owns:** `BasePaperStrategyWorker`, `AtmSingleLegStrategyWorker`, +`NextOpenAtmStrategyWorker`, the hedged/spread/strangle workers, and +`_build_signal_gen_worker_class` (master file) · `Signal Generators/` +**Depends on:** [`market-data.md`](market-data.md), [`risk-and-safety.md`](risk-and-safety.md), [`execution-and-brokers.md`](execution-and-brokers.md) + +--- + +## 1. Responsibility + +One worker = one strategy = one thread. A worker owns its own positions, its own +P&L, its own risk knobs and its own poll cadence. Workers never talk to each +other; the only things they share are the market-data store and the broker lock. + +That isolation is the reason a strategy can be added, disabled or blown up +without touching another one. + +--- + +## 2. Class hierarchy + +``` +threading.Thread + └─ BasePaperStrategyWorker paper bookkeeping, P&L, max-loss, + │ square-off, Telegram events, the + │ enter_position / exit_position contract + │ + ├─ AtmSingleLegStrategyWorker buy ONE ATM option (CE or PE) + │ │ ── strike + expiry resolution + │ │ ── spread gate, sizing, stop/target + │ │ + │ ├─ RenkoStrategyWorker + │ ├─ EMATrendStrategyWorker + │ ├─ HeikinAshiStrategyWorker + │ ├─ ProfitShooterStrategyWorker + │ ├─ OpeningStrikePCRVWAPATRWorker + │ ├─ CPRStrategyWorker + │ ├─ CPRAlgo3StrategyWorker (multi-instrument: spot + ITM CE + ITM PE) + │ ├─ CPRAIWorker (optional — see cpr-codex-ai-agent.md) + │ ├─ 14 factory-built ports (_build_signal_gen_worker_class) + │ │ 13 TradingBot ports + Regime Adaptive + │ └─ NextOpenAtmStrategyWorker signal on a completed bar, ENTER at the + │ ├─ GoldmineStrategyWorker next bar's OPEN + │ └─ MoneyMachineStrategyWorker + │ + ├─ SupertrendBullishWorker hedged puts (main + protective leg) + ├─ DonchianBearishWorker hedged puts + ├─ Delta20HedgedSpreadWorker 4-leg hedged spread + ├─ LongStrangleWorker time-based dual-leg OTM1 CE+PE buy + └─ SLHuntingAIWorker optional — see sl-hunting-ai-agent.md +``` + +`SLHuntingAIWorker` is defined only when its optional dependencies imported +successfully; otherwise the name stays `None` and `main()` skips it. A missing +optional dependency disables a strategy — it never breaks the run. + +--- + +## 3. The worker loop + +``` +run(): + while not shutdown: + sleep(_POLL_SECONDS) + + # ---- always, position or not ------------------------------- + mechanical risk checks stop · target · daily max-loss + · stale-data liquidation · square-off time + + # ---- only when flat and inside the trading window ----------- + if flat and within window and not past entry cutoff: + frame = store.read() # immutable copy + if not health.entries_allowed(): continue + candles = resample_ohlc_from_1m(frame, timeframe) + signal = (candles) # Signal Generators/… + if signal: + strike, expiry = resolve_atm(...) + if not _spread_gate_allows_entry(...): continue + lots = SizingDecision(...) # fail-closed + enter_position(...) # paper or live +``` + +Two properties are load-bearing: + +1. **The risk checks run first and unconditionally.** They do not depend on the + signal logic succeeding, on the LLM agents answering, or on the feed being + fresh enough to enter. A worker that cannot decide anything can still stop + itself out. +2. **`enter_position` / `exit_position` are the only execution doors.** Every + worker — including the AI agents, which reach them through an executor + shim — goes through the same tested path, so paper/live routing, broker + selection, max-loss, square-off and Telegram behave identically everywhere. + +--- + +## 4. The signal-generator factory + +14 of the ATM strategies are not hand-written worker classes. `_signal_gen_ops(prefix)` +builds the env-knob accessors for a prefix, and `_build_signal_gen_worker_class(...)` +produces a worker class from a spec tuple. The spec list `_SIGNAL_GEN_WORKER_SPECS` +also feeds `STRATEGY_ENV_PREFIX`, so a ported strategy gets its name→prefix +mapping automatically. + +Adding a ported strategy is therefore: write the signal module in +`Signal Generators/`, add one spec row, add its `_*` keys to +`Dependencies/env.example`. The CI drift gate fails the build if the last step +is skipped. + +`STRATEGY_ENV_PREFIX` maps display name → env prefix for the hand-written +workers (`"Renko" → "RENKO"`, `"CPRAlgo3" → "CPR_ALGO3"`, …) and merges the +factory specs in. + +--- + +## 5. Per-strategy knobs + +Every strategy reads the same shape of configuration under its own prefix: + +| Knob | Meaning | +|---|---| +| `_VIRTUAL_TRADING` | Default **true**. False = the thread never starts. | +| `_LIVE_TRADING` | Default false. Live needs this **and** the global switch. | +| `_SIZE_MULTIPLIER` | Default 1. Scales `_LOTS`, `_MAX_LOTS`, `_RISK_BUDGET`, `_MAX_LOSS` together. | +| `_LOTS` / `_MAX_LOTS` | Size and hard cap. | +| `_RISK_BUDGET` | Rupee risk per trade, used by `SizingDecision`. | +| `_MAX_LOSS` | Daily kill-switch for this strategy. | +| `_MAX_SPREAD_PCT` | Bid/ask spread cap. Default 0 = off, except Regime Adaptive. | +| `_POLL_SECONDS` | Loop cadence. | +| `_*_HOUR` / `_MINUTE` | Trading window, entry cutoff, square-off. | + +There is **no global virtual-trading switch**: the default is that everything +runs and you silence strategies individually. Live trading is the opposite — +default off, and it needs two flags. The asymmetry is intentional (see +[ADR-0004](../adr/0004-paper-by-default-double-gate.md)). + +--- + +## 6. Multi-leg workers + +Three workers manage baskets rather than a single option: + +| Worker | Legs | Notes | +|---|---|---| +| `SupertrendBullishWorker`, `DonchianBearishWorker` | main + protective put | Hedge sized and closed with the main leg. | +| `Delta20HedgedSpreadWorker` | 4 | `_MAX_LOSS_PER_LOT` is deliberately **not** scaled by the size multiplier — the total already inherits it. | +| `LongStrangleWorker` | OTM1 CE + OTM1 PE | Time-based dual-leg BUY with momentum re-entry. | + +For a basket, "flat" means **every** leg is closed. This matters most for the SL +Hunting BankNIFTY mirror, whose post-exit cooldown only starts once the whole +basket is confirmed flat — a lone surviving leg must not run the timer down. + +--- + +## 7. Coexistence + +`CPR`, `CPRAlgo3`, `Regime Adaptive` and `CPR AI` are independent strategies +that may run together with independent positions and independent P&L. They are +not variants of one another and none disables another. + +Regime Adaptive's two candidate rules live in +`Signal Generators/Regime Adaptive Strategy/regime_candidates.py` as library +code with **no worker of their own**, specifically so the router and a candidate +can never take the same signal twice. See [`regime-adaptive.md`](regime-adaptive.md). + +--- + +## 8. Testing + +| Suite | Covers | +|---|---| +| `Tests/test_nifty_multi_strategy_master.py` | Worker construction, the loop, entry/exit routing, risk checks, factory output | +| `Tests/Signal Generators/test_deterministic_strategy_safety.py` | Shared safety properties across the deterministic strategies | +| `Tests/Signal Generators/test_trading_bot_ports.py` | The 13 ported signal modules | +| `Tests/Signal Generators/test_renko_bounds.py` | Renko brick bounds | +| `Tests/Signal Generators/CPR Strategy/`, `Subhamoy Strategies/` | Per-family signal logic | + +--- + +## 9. Adding a strategy — checklist + +1. Signal logic in `Signal Generators/` (pure function over a candle frame). +2. Either a spec row in `_SIGNAL_GEN_WORKER_SPECS`, or a worker subclass plus a + `STRATEGY_ENV_PREFIX` entry. +3. `_*` keys in `Dependencies/env.example` — CI enforces this. +4. Size knobs read through `_scaled_int` / `_scaled_float`, never the raw + `_env_*` helpers. A drift-guard test fails otherwise. +5. Tests under `Tests/`, mirroring where the code lives. +6. Update this document if the worker introduces a new *shape* of strategy. diff --git a/docs/lld/testing-and-ci.md b/docs/lld/testing-and-ci.md new file mode 100644 index 0000000..0bf7848 --- /dev/null +++ b/docs/lld/testing-and-ci.md @@ -0,0 +1,220 @@ +# LLD — Test architecture, coverage budgets, and CI + +**Owns:** `Tests/` · `.github/workflows/quality-and-security.yml` · +`scripts/check_coverage_thresholds.py` · the `[tool.*]` sections of `pyproject.toml` · +`.pre-commit-config.yaml` +**Related ADRs:** [0009](../adr/0009-importlib-loading-for-spaced-filenames.md), [0010](../adr/0010-tests-in-a-mirrored-tests-tree.md) + +--- + +## 1. Layout + +**Every test lives under `Tests/`, mirroring the source tree.** Runtime folders +contain only runtime code. + +``` +Tests/ +├── test_nifty_multi_strategy_master.py 487 tests — the master runner +├── test_market_data_health.py 26 tests — feed validation/freshness +├── Data Extractors/ +│ └── test_index_fetch_construction.py +├── Dependencies/ +│ ├── conftest.py puts SOURCE Dependencies/ on sys.path +│ ├── test_broker_contract.py test_check_env_config.py +│ ├── test_dhan_token_setup.py test_diagnostic_preflight.py +│ ├── test_execution_ledger.py test_next_open_entry.py +│ ├── test_order_splitting.py test_repository_policy.py +│ ├── test_risk_sizing.py test_secret_redaction.py +│ ├── test_startup_exposure.py test_tick_bar_builder.py +│ ├── test_trading_lifecycle.py +│ ├── Dhan API/test_dhan_execution.py +│ └── Flattrade API/test_flattrade_execution.py +└── Signal Generators/ + ├── test_deterministic_strategy_safety.py test_renko_bounds.py + ├── test_trading_bot_ports.py + ├── CPR AI Agent/ conftest.py + 4 suites + ├── CPR Strategy/ 1 suite + ├── Regime Adaptive Strategy/ conftest.py + 1 suite + ├── SL Hunting AI Agent/ conftest.py + 8 suites + └── Subhamoy Strategies/ 1 suite +``` + +**36 test files, 4 conftests, 1602 tests.** + +### 1.1 Two rules when adding a test + +1. **Put it at the mirrored path.** A test for `Dependencies/foo.py` goes to + `Tests/Dependencies/test_foo.py`. +2. **Give it a repository-unique filename.** There are no `__init__.py` files, so + pytest's `prepend` import mode keys modules by *basename*. Two files named + `test_utils.py` anywhere in the tree would collide. + +### 1.2 The conftests + +Four folders carry a `conftest.py`. Each one puts the **source** folder on +`sys.path` — never the test folder: + +| Conftest | Inserts | +|---|---| +| `Tests/Dependencies/` | `Dependencies/` | +| `Tests/Signal Generators/CPR AI Agent/` | `Signal Generators/CPR AI Agent/` | +| `Tests/Signal Generators/SL Hunting AI Agent/` | `Signal Generators/SL Hunting AI Agent/` | +| `Tests/Signal Generators/Regime Adaptive Strategy/` | that folder **and** `Signal Generators/` | + +They exist because those source folders have spaces in their names and their +modules import each other by bare name (`import sl_hunting_tools`, +`from check_env_config import audit`). Pointing at the source folder means the +tests exercise **the same import resolution production uses**. + +Each conftest inserts **only what it needs**. A repository-wide `sys.path` entry +would let a test resolve an import production never performs, hiding a missing +dependency or an accidental cross-strategy coupling — which is exactly what the +CPR AI conftest's comment warns about. + +The Regime Adaptive one adds the parent too, because `regime_common` re-exports +shared indicators from `misc_strategy_common` one level up. + +--- + +## 2. Running the suites + +```bash +python -m unittest Tests.test_nifty_multi_strategy_master +``` + +```bash +python -m unittest Tests.test_market_data_health +``` + +```bash +python -m pytest "Tests/Signal Generators" "Tests/Dependencies" "Tests/Data Extractors" -q +``` + +The two `unittest` suites are invoked as dotted module paths (`Tests` resolves as +a PEP 420 namespace package). They are kept separate from the pytest run so the +three counts stay independently verifiable — running `pytest Tests` would collect +the unittest `TestCase` classes as well and merge the numbers. + +--- + +## 3. Why the master suite is the master's only real gate + +mypy cannot see `Nifty Multi Strategy Front Test - Master File.py` — its filename +is not a valid module name ([ADR-0009](../adr/0009-importlib-loading-for-spaced-filenames.md)). +So the 17k-line runner is covered by: + +- `compileall` — syntax and consistency, +- `Tests/test_nifty_multi_strategy_master.py` — 487 cases loading it via + `importlib` with `dhanhq` mocked. + +That suite is therefore not optional detail; it is the type checker's stand-in. + +--- + +## 4. Coverage budgets + +Branch coverage is enabled globally (`[tool.coverage.run] branch = true`, +`concurrency = ["thread"]` because safety paths execute in worker threads, +`relative_files = true` so the JSON report is identical on Windows and Linux). + +Coverage.py has exactly **one** global `fail_under`, so the stricter per-module +budgets are enforced from `coverage.json` by +`scripts/check_coverage_thresholds.py`: + +| Tier | Floor | Modules | +|---|---|---| +| Repository baseline | **54.7%** | everything | +| Safety / data-safety | **90%** | `broker_contract`, `execution_ledger`, `startup_exposure`, `trading_lifecycle`, `market_data_health`, `tick_bar_builder`, `next_open_entry`, `risk_sizing`, `order_splitting`, `secret_redaction` | +| Broker adapters | **80%** | Kotak, Shoonya, Flattrade, Dhan | + +Two deliberate properties of the checker: + +- **A module missing from the coverage report is a FAILURE**, not a pass. + Otherwise renaming or deleting a file would silently retire its budget. +- **Every live adapter must have a `BROKER_THRESHOLDS` row.** A broker added + without one escapes the 80% policy — which is exactly what happened when the + Dhan adapter first landed. Adding the row is part of adding a broker. + +`omit` excludes `Tests/`, the reference backtests, and the vendored `NorenApi.py`. + +--- + +## 5. CI + +`.github/workflows/quality-and-security.yml`, on every push and pull request, +across **Python 3.12 and 3.13** (3.13 is the operator's live runtime; 3.12 catches +newer-only syntax). `permissions: contents: read` — the workflow only reads. + +**Job `verify`:** + +| Step | Purpose | +|---|---| +| install core + dev + ai + codex-ai, `pip check` | one resolvable environment | +| `pip_audit --local` | audits the clean resolved tree, not a developer's system Python | +| `pre_commit validate-config` | the hook config itself is valid | +| branch-enabled coverage over all three suites + threshold script | the budgets above | +| `compileall` | the syntax gate for spaced-name files | +| `ruff check .` | lint | +| `mypy` | scoped in `pyproject.toml` to identifier-named modules | +| `bandit -r .` | security; B101/B105/B110 skipped, vendored + reference code excluded | + +**Job `broker-dependencies`:** installs `requirements-brokers.txt` in its own +clean environment and re-runs the broker contract and Flattrade adapter suites. +It is separate because Kotak's official `v2.0.1` tag pins older pandas/requests +that conflict with the audited core set — asking pip to build that combined graph +is impossible, so the upstream environment is validated in isolation instead. + +CI never runs the authenticated CPR AI smoke command, and never opens a real +socket — **a green build proves nothing about the websocket transport**, which is +why enabling it live requires clean paper sessions +([ADR-0005](../adr/0005-rest-vs-websocket-market-data.md)). + +--- + +## 6. Policy tests + +`Tests/Dependencies/test_repository_policy.py` tests the *repository*, not the +runtime. It asserts, without contacting any network: + +- dependency sets are exact (`==` pins) and Kotak uses its official Git tag; +- CI runs the audit, branch coverage, and every dependency set; +- Dependabot updates pip and GitHub Actions weekly; +- coverage config stays branch-enabled at the 54.7% baseline; +- every `cpr_ai_*.py` module is inside mypy's scope; +- `env.example` documents every `_env_*` key the code reads (>300 found, as a + sanity check that the AST walk still works); +- `CLAUDE.md` and `AGENTS.md` share one identical runtime section; +- architecture docs make both optional agents visible and carry no stale + worker-roster claims. + +That last group is a **documentation staleness gate**. It currently covers +`README.md`, `Signal Generators/Readme.md`, `AGENTS.md`, `CLAUDE.md` and the +master file. Extending it to `docs/hld/` is an open follow-up recorded in +[ADR-0011](../adr/0011-committed-docs-untracked-superpowers.md). + +--- + +## 7. Local gates + +```bash +pip install -r requirements-dev.txt +``` + +Then the same commands CI runs (§2 and §5). `.pre-commit-config.yaml` wires +check-only hooks — ruff, merge-conflict, YAML, large-file, debug-statement. +**Policy: hooks check, never rewrite**, so a commit never changes content the +author did not review. Install once with `pre-commit install`. + +--- + +## 8. Testing conventions + +- **No network, ever.** Broker HTTP, browser flows, Telegram, Google Sheets and + both LLM SDKs are mocked or injected. `test_broker_contract.py` spawns + short-lived subprocesses to verify `sys.path` semantics, which is the one place + a subprocess is legitimate. +- **Optional dependencies skip, not fail.** Broker/SDK-specific cases skip when + the dependency is absent. +- **Fixtures are small and literal.** Safety tests assert on specific candles and + quantities rather than generated data, so a failure names the case. +- **Add the test before the change** for anything in the safety tier. diff --git a/docs/superpowers/plans/2026-07-27-mat-111-sl-hunting-cooldown.md b/docs/superpowers/plans/2026-07-27-mat-111-sl-hunting-cooldown.md deleted file mode 100644 index 0fd8f75..0000000 --- a/docs/superpowers/plans/2026-07-27-mat-111-sl-hunting-cooldown.md +++ /dev/null @@ -1,393 +0,0 @@ -# MAT-111 SL Hunting Post-Exit Guard Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make the SL Hunting cooldown begin exactly once when the complete NIFTY/BankNIFTY trade becomes flat, while keeping first entries and every exit path unblocked. - -**Architecture:** Keep the behavior inside the existing `SLHuntingAIWorker` and `MasterWorkerExecutor` seams. Replace wall-clock elapsed-time enforcement with one monotonic deadline, arm it only on a tracked trade's basket-flat transition, validate its output at the entry choke point, and extend the existing live-config validator rather than adding a new configuration system. - -**Tech Stack:** Python 3.12/3.13, `unittest`, `pytest`, existing environment helpers, Ruff, mypy, Coverage.py, Bandit, pre-commit, pip-audit. - -## Global Constraints - -- The exchange freeze-limit splitter and its metadata are intentional future groundwork and must not be modified. -- The cooldown starts only after a successfully opened SL Hunting trade becomes fully flat. -- A partial, unknown, or failed close that leaves either leg tracked must not start the cooldown. -- First entries and all exit paths remain unaffected. -- Live entry fails closed on corrupt cooldown state; paper and standalone entry remain fail-open. -- `SL_HUNTING_POST_EXIT_COOLDOWN_MINUTES=0` remains the supported disable value. -- The documented no-new-entry default remains 12:00 IST. -- Live flags remain off during development and verification. - ---- - -### Task 1: Arm the cooldown on the basket-flat transition - -**Files:** -- Modify: `Nifty Multi Strategy Front Test - Master File.py` -- Test: `test_nifty_multi_strategy_master.py` - -**Interfaces:** -- Consumes: `SLHuntingAIWorker.enter_position(...) -> bool`, `PaperPosition.active`, and `time.monotonic()`. -- Produces: `SLHuntingAIWorker._arm_post_exit_cooldown_if_flat() -> None` and `SLHuntingAIWorker.post_exit_cooldown_remaining_seconds() -> float`. - -- [ ] **Step 1: Write failing basket-transition tests** - -Add focused tests to `TestSLHuntingAIWorker` that use literal monotonic values: - -```python -def test_cooldown_waits_for_final_mirror_close(self): - worker, _ = self._make_worker() - with patch.object(master_file.time, "monotonic", return_value=100.0): - self.assertTrue(worker.enter_position("LONG", 24300.0, 24290.0, 24400.0)) - worker.exit_nifty_leg_only("NIFTY_PREMISE_INVALID") - self.assertIsNone(worker._post_exit_cooldown_deadline_monotonic) - - with patch.object(master_file.time, "monotonic", return_value=200.0): - worker.exit_bnf_mirror_only("BNF_PREMISE_INVALID") - self.assertEqual( - worker._post_exit_cooldown_deadline_monotonic, - 200.0 + master_file.SL_HUNTING_POST_EXIT_COOLDOWN_MINUTES * 60.0, - ) - - -def test_cooldown_waits_for_final_nifty_close(self): - worker, _ = self._make_worker() - self.assertTrue(worker.enter_position("LONG", 24300.0, 24290.0, 24400.0)) - worker.exit_bnf_mirror_only("BNF_PREMISE_INVALID") - self.assertIsNone(worker._post_exit_cooldown_deadline_monotonic) - - with patch.object(master_file.time, "monotonic", return_value=300.0): - worker.exit_position("NIFTY_PREMISE_INVALID") - self.assertEqual( - worker._post_exit_cooldown_deadline_monotonic, - 300.0 + master_file.SL_HUNTING_POST_EXIT_COOLDOWN_MINUTES * 60.0, - ) -``` - -Add a live partial-close regression using the existing scripted broker fake. It -must assert that the deadline remains `None` after the retained mirror position -and becomes a full new interval only after the retry confirms flat. Add an -idempotence assertion that a second no-op close does not change the deadline. - -- [ ] **Step 2: Run the new tests and verify the old implementation fails** - -Run: - -```powershell -python -m unittest ` - test_nifty_multi_strategy_master.TestSLHuntingAIWorker.test_cooldown_waits_for_final_mirror_close ` - test_nifty_multi_strategy_master.TestSLHuntingAIWorker.test_cooldown_waits_for_final_nifty_close -v -``` - -Expected: failure because `_post_exit_cooldown_deadline_monotonic` does not -exist and the old `_last_exit_at` is armed on the first NIFTY-leg close. - -- [ ] **Step 3: Implement the smallest basket-flat state machine** - -Initialize: - -```python -self._cooldown_trade_open = False -self._post_exit_cooldown_deadline_monotonic: float | None = None -``` - -After a successful NIFTY entry: - -```python -if ok: - self._cooldown_trade_open = True -``` - -Add: - -```python -def _arm_post_exit_cooldown_if_flat(self) -> None: - if not self._cooldown_trade_open: - return - if self.pos.active or self._mirror_pos.active: - return - self._cooldown_trade_open = False - if SL_HUNTING_POST_EXIT_COOLDOWN_MINUTES <= 0: - self._post_exit_cooldown_deadline_monotonic = None - return - self._post_exit_cooldown_deadline_monotonic = ( - time.monotonic() + SL_HUNTING_POST_EXIT_COOLDOWN_MINUTES * 60.0 - ) -``` - -Call the helper after a confirmed mirror close and at the end of -`after_exit()`. Replace the wall-clock calculation with: - -```python -deadline = self._post_exit_cooldown_deadline_monotonic -if SL_HUNTING_POST_EXIT_COOLDOWN_MINUTES <= 0 or deadline is None: - return 0.0 -return max(deadline - time.monotonic(), 0.0) -``` - -- [ ] **Step 4: Run focused and surrounding worker tests** - -Run: - -```powershell -python -m unittest test_nifty_multi_strategy_master.TestSLHuntingAIWorker -v -``` - -Expected: all SL Hunting worker tests pass. - -- [ ] **Step 5: Commit the basket transition** - -```powershell -git add -- "Nifty Multi Strategy Front Test - Master File.py" test_nifty_multi_strategy_master.py -git commit -m "fix: arm SL Hunting cooldown when basket is flat" ` - -m "Co-authored-by: Codex " -``` - -### Task 2: Validate the cooldown result at the entry boundary - -**Files:** -- Modify: `Signal Generators/SL Hunting AI Agent/sl_hunting_executor.py` -- Test: `Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_agent.py` - -**Interfaces:** -- Consumes: optional `worker.post_exit_cooldown_remaining_seconds()` and `worker.live_trading`. -- Produces: an entry rejection dictionary for corrupt live cooldown state. - -- [ ] **Step 1: Replace the fail-open-only test with explicit live and paper cases** - -Add a `live_trading` flag to the raising test worker and prove the two policies: - -```python -def test_broken_cooldown_hook_blocks_live_entry_but_not_paper(): - class _Boom(_FakeWorker): - def __init__(self, live_trading): - super().__init__() - self.live_trading = live_trading - - def post_exit_cooldown_remaining_seconds(self): - raise RuntimeError("clock unavailable") - - live = MasterWorkerExecutor(_Boom(True)) - rejected = live.enter("LONG", stop=1, target=2, reason="x", price=25000) - assert rejected["accepted"] is False - assert "cooldown safety check unavailable" in rejected["reason"].lower() - assert live._w.entries == [] - - paper = MasterWorkerExecutor(_Boom(False)) - assert paper.enter("LONG", stop=1, target=2, reason="x", price=25000)["accepted"] is True -``` - -Parametrize `float("nan")`, `float("inf")`, `float("-inf")`, and `-1.0` as -corrupt live values. Keep the existing test proving a worker without the hook -still enters and the existing test proving exits do not consult the hook. - -- [ ] **Step 2: Run the focused tests and verify failure** - -Run: - -```powershell -python -m pytest ` - "Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_agent.py" ` - -k "cooldown" -q -``` - -Expected: the live raising-hook and non-finite cases fail because the current -executor converts failures to zero and enters. - -- [ ] **Step 3: Implement finite, non-negative validation** - -Import `math`. In `MasterWorkerExecutor.enter()`, treat exceptions and invalid -values as a guard failure. Return the following only when -`bool(getattr(self._w, "live_trading", False))` is true: - -```python -{ - "accepted": False, - "reason": ( - "cooldown safety check unavailable; live entry rejected. " - "Exits remain available." - ), -} -``` - -Paper mode continues to zero the remaining interval on a guard failure. Valid -positive values keep the existing human-readable cooldown rejection. - -- [ ] **Step 4: Run the complete executor test module** - -Run: - -```powershell -python -m pytest ` - "Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_agent.py" -q -``` - -Expected: all tests in the module pass. - -- [ ] **Step 5: Commit the entry-boundary hardening** - -```powershell -git add -- ` - "Signal Generators/SL Hunting AI Agent/sl_hunting_executor.py" ` - "Signal Generators/SL Hunting AI Agent/tests/test_sl_hunting_agent.py" -git commit -m "fix: fail closed on corrupt live cooldown state" ` - -m "Co-authored-by: Codex " -``` - -### Task 3: Align and validate SL Hunting entry-control configuration - -**Files:** -- Modify: `Nifty Multi Strategy Front Test - Master File.py` -- Test: `test_nifty_multi_strategy_master.py` -- Modify: `AGENTS.md` -- Modify: `CLAUDE.md` -- Modify: `Signal Generators/SL Hunting AI Agent/README.md` -- Modify: `Signal Generators/SL Hunting AI Agent/sl_hunting_doc.md` - -**Interfaces:** -- Consumes: `_live_config_errors(worker, "SL_HUNTING")`. -- Produces: noon defaults and live-mode errors for malformed or invalid - `SL_HUNTING_POST_EXIT_COOLDOWN_MINUTES` and - `SL_HUNTING_NO_NEW_ENTRY_{HOUR,MINUTE}`. - -- [ ] **Step 1: Write failing configuration tests** - -In `TestStrictLiveConfiguration`, add: - -```python -def test_sl_hunting_entry_controls_are_strictly_validated_for_live(self): - worker = self._Worker("SL Hunting AI") - worker.no_new_entry_hour = 12 - worker.no_new_entry_minute = 0 - cases = { - "SL_HUNTING_POST_EXIT_COOLDOWN_MINUTES": "-1", - "SL_HUNTING_NO_NEW_ENTRY_HOUR": "24", - "SL_HUNTING_NO_NEW_ENTRY_MINUTE": "60", - } - for name, raw in cases.items(): - with self.subTest(name=name), patch.dict(os.environ, {name: raw}): - errors = master_file._live_config_errors(worker, "SL_HUNTING") - self.assertTrue(any(name in error for error in errors), errors) -``` - -Add resolved-value cases for a negative cooldown and invalid resolved cutoff. -Add a worker-level behavior test that deletes the two environment variables, -loads a fresh master module through the test's existing loader, and proves a -flat worker starts skipping new inference at 12:00 rather than 10:30. - -- [ ] **Step 2: Run the focused tests and verify failure** - -Run: - -```powershell -python -m unittest ` - test_nifty_multi_strategy_master.TestStrictLiveConfiguration.test_sl_hunting_entry_controls_are_strictly_validated_for_live -v -``` - -Expected: failure because the three raw settings are not currently part of the -live-config rule set. - -- [ ] **Step 3: Add the narrow validation rules and correct defaults** - -Change the worker defaults to: - -```python -no_new_entry_hour = _env_int("SL_HUNTING_NO_NEW_ENTRY_HOUR", 12) -no_new_entry_minute = _env_int("SL_HUNTING_NO_NEW_ENTRY_MINUTE", 0) -``` - -For `normalized_prefix == "SL_HUNTING"`, extend `raw_rules` with a -`nonnegative_integer` rule for the cooldown and existing `integer_range` rules -for the cutoff. Add the `nonnegative_integer` branch to the shared loop. Validate -the resolved module cooldown plus the worker's resolved no-new-entry fields. - -- [ ] **Step 4: Update operator documentation without changing splitter docs** - -Document that the cooldown starts when the whole basket is flat, uses the final -leg's close time, and rejects corrupt live entry state. Keep `AGENTS.md` and -`CLAUDE.md` byte-identical from “What this project is” downward. - -- [ ] **Step 5: Run focused configuration and documentation checks** - -Run: - -```powershell -python -m unittest ` - test_nifty_multi_strategy_master.TestStrictLiveConfiguration ` - test_nifty_multi_strategy_master.TestSLHuntingAIWorker -v -python algo.py check-env -python -c "from pathlib import Path; a=Path('AGENTS.md').read_text(encoding='utf-8').split('## What this project is',1)[1]; c=Path('CLAUDE.md').read_text(encoding='utf-8').split('## What this project is',1)[1]; assert a == c" -``` - -Expected: tests pass, `check-env` reports no repository/template drift, and the -documentation parity assertion exits zero. - -- [ ] **Step 6: Commit configuration and docs** - -```powershell -git add -- ` - "Nifty Multi Strategy Front Test - Master File.py" ` - test_nifty_multi_strategy_master.py ` - AGENTS.md CLAUDE.md ` - "Signal Generators/SL Hunting AI Agent/README.md" ` - "Signal Generators/SL Hunting AI Agent/sl_hunting_doc.md" -git commit -m "fix: validate SL Hunting entry controls for live mode" ` - -m "Co-authored-by: Codex " -``` - -### Task 4: Verify, security-scan, and publish MAT-111 - -**Files:** -- Modify only if a verification failure identifies a MAT-111 regression. -- Create ignored scan artifacts outside committed secret-bearing paths. - -**Interfaces:** -- Consumes: the complete `origin/main..HEAD` diff. -- Produces: a green local gate, a diff-scoped security conclusion, and one PR - from `codex/mat-111-sl-hunting-cooldown` to `main`. - -- [ ] **Step 1: Run all repository gates** - -Run the repository commands documented in `AGENTS.md`: both unittest suites, -all repository pytest suites, branch-enabled coverage plus -`scripts/check_coverage_thresholds.py`, pip-audit, Ruff, mypy, compileall, -Bandit, and pre-commit. - -- [ ] **Step 2: Review the exact diff** - -Run: - -```powershell -git diff --check origin/main...HEAD -git diff --stat origin/main...HEAD -git status --short -``` - -Confirm no freeze-limit splitter file or code path changed and no generated -test/coverage artifacts are tracked. - -- [ ] **Step 3: Run the Codex Security diff workflow** - -Use the `codex-security:security-diff-scan` workflow on `origin/main...HEAD`. -Complete its threat-model, discovery, validation, and attack-path phases for -every candidate it produces, and keep generated scan artifacts ignored. - -- [ ] **Step 4: Push and open the PR** - -```powershell -git push -u origin codex/mat-111-sl-hunting-cooldown -gh pr create --base main --head codex/mat-111-sl-hunting-cooldown ` - --title "MAT-111: harden SL Hunting post-exit cooldown" ` - --body-file .github-mat-111-pr-body.md -``` - -The PR body must include the basket-flat invariant, the first-entry/exit -non-regression, strict live-config behavior, full local verification evidence, -security conclusion, and Codex co-authorship. - -- [ ] **Step 5: Verify hosted checks and mergeability** - -Watch the PR checks to a terminal result. Fix only MAT-111 regressions, rerun the -relevant local gate, push normally, and confirm all required Python 3.12/3.13 -jobs pass and GitHub reports the PR mergeable. diff --git a/docs/superpowers/specs/2026-07-27-mat-111-sl-hunting-cooldown-design.md b/docs/superpowers/specs/2026-07-27-mat-111-sl-hunting-cooldown-design.md deleted file mode 100644 index 1659744..0000000 --- a/docs/superpowers/specs/2026-07-27-mat-111-sl-hunting-cooldown-design.md +++ /dev/null @@ -1,112 +0,0 @@ -# MAT-111 SL Hunting Post-Exit Guard Design - -## Problem - -SL Hunting trades a basket that can contain a NIFTY leg and a mechanically -mirrored BankNIFTY leg. The agent may close either leg independently when that -leg's premise is invalidated. - -The current cooldown starts in `after_exit()`, which runs when the NIFTY leg -closes. If the BankNIFTY leg remains open, the timer starts before the trade is -flat and can expire while exposure still exists. Closing the BankNIFTY leg later -can then be followed by an immediate re-entry, defeating the cooldown's purpose. -The same premature start can occur when a tied BankNIFTY close is partial or -indeterminate. - -Two adjacent live-entry controls also fail closed incompletely: - -- the code fallback for the documented noon cutoff is 10:30; -- malformed cooldown and no-new-entry values can fall back to defaults without - disabling live mode. - -The exchange freeze-limit splitter and its metadata are intentional future -groundwork for size multipliers up to 25. MAT-111 does not modify or wire that -code. - -## Required Behavior - -The cooldown applies only after an SL Hunting trade closes because of target, -stop-loss, or premise invalidation. Operational shutdown closes such as max-loss -and square-off may reach the same flat transition, but their lifecycle already -blocks re-entry. - -A trade is closed only when all locally tracked SL Hunting exposure is flat: - -- the NIFTY position is inactive; and -- the BankNIFTY mirror is inactive, including no partial or indeterminate live - quantity retained for reconciliation. - -The first successful entry marks a trade as open. Closing one leg while the -other remains open does not start the timer. The successful close of the final -leg starts the full configured interval. Repeated close/finalization calls for -the same trade do not restart or extend the timer. - -The timer blocks entry only. It is never consulted by any NIFTY or BankNIFTY -exit path. - -## State and Clock - -`SLHuntingAIWorker` will keep two private fields: - -- `_cooldown_trade_open: bool` records whether a successfully opened trade has - not yet produced its one basket-flat cooldown transition. -- `_post_exit_cooldown_deadline_monotonic: float | None` stores the enforcement - deadline. - -`enter_position()` sets `_cooldown_trade_open` only after the NIFTY entry -succeeds. `_arm_post_exit_cooldown_if_flat()` checks both legs, consumes the -open-trade marker exactly once, and records `time.monotonic() + minutes * 60`. -It is called after the NIFTY exit hook and after a confirmed BankNIFTY mirror -close so either leg can be the final leg. - -`post_exit_cooldown_remaining_seconds()` returns zero before the first completed -trade, when disabled, or after expiry. Otherwise it returns the non-negative -difference between the deadline and `time.monotonic()`. Wall-clock changes -therefore cannot shorten or extend an active cooldown. - -## Guard Failure Policy - -The master executor accepts a finite, non-negative remaining-seconds value. - -- In live mode, an exception, non-numeric value, NaN, infinity, or negative - value rejects the new entry with a stable safety reason. -- In paper mode and in the standalone paper runner, the same guard failure stays - fail-open so experiments are not turned into a trading outage. -- A worker without the optional cooldown hook continues to work as before. - -This policy affects entries only; exits do not call the cooldown hook. - -## Configuration - -The in-code defaults for `SL_HUNTING_NO_NEW_ENTRY_HOUR` and -`SL_HUNTING_NO_NEW_ENTRY_MINUTE` will be corrected to `12` and `0`, matching -`env.example`, README, AGENTS.md, and CLAUDE.md. - -For a live-enabled SL Hunting worker, `_live_config_errors()` will validate: - -- `SL_HUNTING_POST_EXIT_COOLDOWN_MINUTES` as a non-negative integer; -- `SL_HUNTING_NO_NEW_ENTRY_HOUR` as an integer from 0 through 23; -- `SL_HUNTING_NO_NEW_ENTRY_MINUTE` as an integer from 0 through 59; -- the resolved cooldown and resolved cutoff fields against the same bounds. - -Paper mode retains the forgiving environment helpers. - -## Verification - -Focused regressions will prove: - -- no cooldown exists before a completed trade; -- a NIFTY-only close does not start the timer while BankNIFTY remains open; -- a BankNIFTY-only close does not start the timer while NIFTY remains open; -- closing the final leg starts the full interval; -- a partial or indeterminate final close does not start the timer; -- repeated finalization does not extend the interval; -- the timer uses monotonic time; -- a broken or non-finite guard blocks live entries but not paper entries; -- exits remain available while the guard is active or broken; -- missing no-new-entry configuration resolves to noon; -- malformed or negative cooldown/cutoff configuration disables live mode. - -The complete repository test, coverage, dependency-audit, lint, type, compile, -Bandit, and pre-commit gates will run before publication. A diff-scoped Codex -Security scan will cover the final MAT-111 change. diff --git a/pyproject.toml b/pyproject.toml index d6614dc..ae4ab8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,8 +39,10 @@ external = ["BLE"] [tool.ruff.lint.per-file-ignores] # B017: tests intentionally assert on bare Exception for defensive paths. # RUF012: test fakes use mutable class attributes as fixtures. +# Every suite lives under Tests/ (mirroring the source tree); the glob covers +# them all, and the master suite is named explicitly for discoverability. "**/test_*.py" = ["B017", "RUF012"] -"test_nifty_multi_strategy_master.py" = ["B017", "RUF012"] +"Tests/test_nifty_multi_strategy_master.py" = ["B017", "RUF012"] # Reference-only backtests keep the deliberate sys.path-before-import pattern. "My Backtest Files (For Reference)/**" = ["E402"] @@ -164,6 +166,12 @@ relative_files = true concurrency = ["thread"] source = ["."] omit = [ + # Every suite now lives under Tests/ (mirroring the source tree). Both + # patterns are listed because Coverage.py's `*` does not cross a path + # separator: the first catches Tests/test_x.py, the second the nested + # Tests//... suites. + "Tests/*", + "Tests/**/*", "*/test_*.py", "*/tests/*", "Backtest Outputs/*", From 2c6a57115101f9047f6ecb3e75c793f58d7d62b9 Mon Sep 17 00:00:00 2001 From: DoRmAmMu1997 Date: Mon, 10 Aug 2026 19:28:53 +0530 Subject: [PATCH 2/5] chore(ci): raise the overall coverage floor from 54.7% to 70% The 54.7% floor was the branch-enabled baseline MAT-110 measured before the gate was installed. Coverage has grown to 71.4% since, so a floor 17 points below actual was no longer stopping erosion -- roughly 3,100 statements could have lost their tests without the build noticing. Raised to 70.0, which leaves ~1.4 points of real headroom against the measured 71.4%. The per-module budgets are unchanged: 90% for the safety/data-safety tier, 80% per broker adapter. On the local-vs-CI question: neo_api_client and pyotp are installed locally but are NOT in CI's verify job. That does not move the number -- CI already enforces and passes the 80% budget on kotak_execution.py without the Kotak SDK, because the adapters are tested through injected doubles rather than the real client. Updated everywhere the floor is stated so they cannot drift: pyproject, test_repository_policy's assertion, check_coverage_thresholds' docstring, README.md, CLAUDE.md, AGENTS.md and docs/lld/testing-and-ci.md. The two agent guides remain byte-identical after the shared marker. Documented the one-way rule next to the setting: this floor only ever moves UP, and only after a run shows headroom. Lowering it to make a red build pass defeats the gate. Verified locally: coverage report exits 0 at the new floor (71.4%), the per-module policy gate passes, ruff clean, and the policy suite passes with the updated assertion. Co-Authored-By: Claude Opus 5 --- AGENTS.md | 4 +++- CLAUDE.md | 4 +++- README.md | 2 +- Tests/Dependencies/test_repository_policy.py | 2 +- docs/lld/testing-and-ci.md | 4 ++-- pyproject.toml | 14 ++++++++++---- scripts/check_coverage_thresholds.py | 2 +- 7 files changed, 21 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 11bb9ff..6b07327 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -194,8 +194,10 @@ Backtest Outputs/ # generated CSVs/logs (gitign the branch-enabled Coverage.py run plus `scripts/check_coverage_thresholds.py`, pip-audit of committed pins locally plus the clean resolved CI environment, Ruff, mypy, compileall, - Bandit, and pre-commit. Coverage floors are 54.7% overall, 90% for new + Bandit, and pre-commit. Coverage floors are 70% overall, 90% for new execution/reconciliation/data-safety modules, and 80% per broker adapter. + The overall floor only ever moves UP, and only after a run shows headroom -- + never lower it to make a red build pass. - **Dependencies:** install core with `pip install -r requirements.txt`; add `requirements-ai.txt` for SL Hunting and `requirements-dev.txt` for local gates. `requirements-brokers.txt` is the isolated upstream compatibility diff --git a/CLAUDE.md b/CLAUDE.md index 6c34263..18510dd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -194,8 +194,10 @@ Backtest Outputs/ # generated CSVs/logs (gitign the branch-enabled Coverage.py run plus `scripts/check_coverage_thresholds.py`, pip-audit of committed pins locally plus the clean resolved CI environment, Ruff, mypy, compileall, - Bandit, and pre-commit. Coverage floors are 54.7% overall, 90% for new + Bandit, and pre-commit. Coverage floors are 70% overall, 90% for new execution/reconciliation/data-safety modules, and 80% per broker adapter. + The overall floor only ever moves UP, and only after a run shows headroom -- + never lower it to make a red build pass. - **Dependencies:** install core with `pip install -r requirements.txt`; add `requirements-ai.txt` for SL Hunting and `requirements-dev.txt` for local gates. `requirements-brokers.txt` is the isolated upstream compatibility diff --git a/README.md b/README.md index 5344336..5bc01ca 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,7 @@ python -m compileall -q . python -m ruff check . python -m mypy ``` -Coverage is branch-enabled: overall runtime coverage may not fall below 54.7%, new execution/reconciliation/data-safety modules require 90%, and every broker adapter requires 80%. The four local audit commands check committed direct pins; CI additionally audits the complete resolved dependency tree in a clean hosted environment. `pyproject.toml` holds the coverage, Ruff, and mypy config (mypy is scoped to the identifier-named modules — the spaced-name master file is covered by `compileall` + the unittest suite instead). `.pre-commit-config.yaml` wires the check-only hooks; install them once with `pre-commit install`. +Coverage is branch-enabled: overall runtime coverage may not fall below 70%, new execution/reconciliation/data-safety modules require 90%, and every broker adapter requires 80%. The four local audit commands check committed direct pins; CI additionally audits the complete resolved dependency tree in a clean hosted environment. `pyproject.toml` holds the coverage, Ruff, and mypy config (mypy is scoped to the identifier-named modules — the spaced-name master file is covered by `compileall` + the unittest suite instead). `.pre-commit-config.yaml` wires the check-only hooks; install them once with `pre-commit install`. # License Released under the MIT License — see [LICENSE](LICENSE). diff --git a/Tests/Dependencies/test_repository_policy.py b/Tests/Dependencies/test_repository_policy.py index 4a12721..7a8ad41 100644 --- a/Tests/Dependencies/test_repository_policy.py +++ b/Tests/Dependencies/test_repository_policy.py @@ -108,7 +108,7 @@ def test_coverage_config_is_branch_enabled_and_preserves_overall_baseline(): config = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) assert config["tool"]["coverage"]["run"]["branch"] is True - assert config["tool"]["coverage"]["report"]["fail_under"] == 54.7 + assert config["tool"]["coverage"]["report"]["fail_under"] == 70.0 def test_mypy_covers_the_complete_identifier_named_cpr_ai_runtime(): diff --git a/docs/lld/testing-and-ci.md b/docs/lld/testing-and-ci.md index 0bf7848..7cd9a51 100644 --- a/docs/lld/testing-and-ci.md +++ b/docs/lld/testing-and-ci.md @@ -123,7 +123,7 @@ budgets are enforced from `coverage.json` by | Tier | Floor | Modules | |---|---|---| -| Repository baseline | **54.7%** | everything | +| Repository floor | **70%** | everything (measured 71.4%; original MAT-110 baseline was 54.7%) | | Safety / data-safety | **90%** | `broker_contract`, `execution_ledger`, `startup_exposure`, `trading_lifecycle`, `market_data_health`, `tick_bar_builder`, `next_open_entry`, `risk_sizing`, `order_splitting`, `secret_redaction` | | Broker adapters | **80%** | Kotak, Shoonya, Flattrade, Dhan | @@ -179,7 +179,7 @@ runtime. It asserts, without contacting any network: - dependency sets are exact (`==` pins) and Kotak uses its official Git tag; - CI runs the audit, branch coverage, and every dependency set; - Dependabot updates pip and GitHub Actions weekly; -- coverage config stays branch-enabled at the 54.7% baseline; +- coverage config stays branch-enabled at the 70% floor; - every `cpr_ai_*.py` module is inside mypy's scope; - `env.example` documents every `_env_*` key the code reads (>300 found, as a sanity check that the AST walk still works); diff --git a/pyproject.toml b/pyproject.toml index ae4ab8b..b9d4e99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -180,10 +180,16 @@ omit = [ ] [tool.coverage.report] -# MAT-110 measured the branch-enabled runtime baseline before installing this -# gate. Per-module 90%/80% budgets are enforced from coverage.json by the small -# policy script because Coverage.py has only one global fail-under setting. -fail_under = 54.7 +# MAT-110 measured the branch-enabled runtime baseline (54.7%) before installing +# this gate. Coverage has grown well past it since, so the floor was raised to 70 +# against a measured 71.4% -- a floor 17 points below actual stops nothing. Per- +# module 90%/80% budgets are enforced from coverage.json by the small policy +# script because Coverage.py has only one global fail-under setting. +# +# Raising this is deliberate and one-way: only move it UP, and only after a run +# shows real headroom. Lowering it to make a red build pass defeats the gate -- +# fix the tests instead. +fail_under = 70.0 precision = 1 show_missing = true skip_covered = false diff --git a/scripts/check_coverage_thresholds.py b/scripts/check_coverage_thresholds.py index 48897d8..2865276 100644 --- a/scripts/check_coverage_thresholds.py +++ b/scripts/check_coverage_thresholds.py @@ -5,7 +5,7 @@ budgets to the live-money safety core and each broker adapter. Why this script exists at all: Coverage.py has exactly ONE global -``fail_under`` setting (pyproject pins it to the repository's 54.7% baseline), +``fail_under`` setting (pyproject pins it to the repository's 70% floor), so the stricter 90%/80% per-module budgets have to be enforced from the JSON report by hand. The split is deliberate -- the global floor stops overall erosion, while these budgets stop a specific safety module from quietly losing From 44611ddd2d94fc95d78c036582a29ee9ebdc4e53 Mon Sep 17 00:00:00 2001 From: DoRmAmMu1997 Date: Mon, 10 Aug 2026 19:34:01 +0530 Subject: [PATCH 3/5] fix(ci): set the coverage floor to 68%, measured from CI not from a local run The previous commit set the floor to 70.0 on a locally measured 71.4%. CI reports 69.1%, so the build failed by 0.9 points. The floor is now 68.0, which leaves ~1.1 points of headroom against the number that actually gates the build. My reasoning in the previous commit was wrong and is worth recording. I argued the missing broker SDKs could not move the total, because CI already enforces and passes the 80% budget on kotak_execution.py without neo_api_client. That holds for the ADAPTER, but not for everything the SDK gates: CI's verify job runs "1082 passed, 7 skipped" where a local run with neo_api_client and pyotp installed runs 1089 passed, 0 skipped. Those 7 tests are worth ~2.3 points of overall coverage. Enforcing a per-module budget says nothing about the modules outside that dict. So the rule is now written next to the setting and in both agent guides: judge this floor from CI, never from a local run, because a developer machine with the optional broker SDKs installed reads about 2 points high. CI deliberately installs core + dev + ai + codex-ai only -- Kotak's official tag pins older pandas/requests and cannot share the audited core environment -- so the hosted number is the honest one. Still a +13.3 point improvement on the 54.7% baseline, and the one-way rule stands: only move it UP, and only after a CI run shows headroom. Note the restructure itself was never in question in that failed run: CI collected 487 master + 26 market-data-health + 1089 (1082 passed, 7 skipped) across both Python 3.12 and 3.13, matching the pre-move counts exactly, and both broker-dependency jobs passed. Only fail_under was wrong. Co-Authored-By: Claude Opus 5 --- AGENTS.md | 8 ++++--- CLAUDE.md | 8 ++++--- README.md | 2 +- Tests/Dependencies/test_repository_policy.py | 2 +- docs/lld/testing-and-ci.md | 4 ++-- pyproject.toml | 23 +++++++++++++------- scripts/check_coverage_thresholds.py | 2 +- 7 files changed, 30 insertions(+), 19 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6b07327..74780f2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -194,10 +194,12 @@ Backtest Outputs/ # generated CSVs/logs (gitign the branch-enabled Coverage.py run plus `scripts/check_coverage_thresholds.py`, pip-audit of committed pins locally plus the clean resolved CI environment, Ruff, mypy, compileall, - Bandit, and pre-commit. Coverage floors are 70% overall, 90% for new + Bandit, and pre-commit. Coverage floors are 68% overall, 90% for new execution/reconciliation/data-safety modules, and 80% per broker adapter. - The overall floor only ever moves UP, and only after a run shows headroom -- - never lower it to make a red build pass. + Judge the overall floor from CI, never from a local run: a machine with the + optional broker SDKs installed runs 7 tests CI's verify job skips and reads + ~2 points high (CI measures 69.1%). The floor only ever moves UP, and only + after a CI run shows headroom -- never lower it to make a red build pass. - **Dependencies:** install core with `pip install -r requirements.txt`; add `requirements-ai.txt` for SL Hunting and `requirements-dev.txt` for local gates. `requirements-brokers.txt` is the isolated upstream compatibility diff --git a/CLAUDE.md b/CLAUDE.md index 18510dd..96e9c23 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -194,10 +194,12 @@ Backtest Outputs/ # generated CSVs/logs (gitign the branch-enabled Coverage.py run plus `scripts/check_coverage_thresholds.py`, pip-audit of committed pins locally plus the clean resolved CI environment, Ruff, mypy, compileall, - Bandit, and pre-commit. Coverage floors are 70% overall, 90% for new + Bandit, and pre-commit. Coverage floors are 68% overall, 90% for new execution/reconciliation/data-safety modules, and 80% per broker adapter. - The overall floor only ever moves UP, and only after a run shows headroom -- - never lower it to make a red build pass. + Judge the overall floor from CI, never from a local run: a machine with the + optional broker SDKs installed runs 7 tests CI's verify job skips and reads + ~2 points high (CI measures 69.1%). The floor only ever moves UP, and only + after a CI run shows headroom -- never lower it to make a red build pass. - **Dependencies:** install core with `pip install -r requirements.txt`; add `requirements-ai.txt` for SL Hunting and `requirements-dev.txt` for local gates. `requirements-brokers.txt` is the isolated upstream compatibility diff --git a/README.md b/README.md index 5bc01ca..84c0c63 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,7 @@ python -m compileall -q . python -m ruff check . python -m mypy ``` -Coverage is branch-enabled: overall runtime coverage may not fall below 70%, new execution/reconciliation/data-safety modules require 90%, and every broker adapter requires 80%. The four local audit commands check committed direct pins; CI additionally audits the complete resolved dependency tree in a clean hosted environment. `pyproject.toml` holds the coverage, Ruff, and mypy config (mypy is scoped to the identifier-named modules — the spaced-name master file is covered by `compileall` + the unittest suite instead). `.pre-commit-config.yaml` wires the check-only hooks; install them once with `pre-commit install`. +Coverage is branch-enabled: overall runtime coverage may not fall below 68% (CI measures 69.1%; a local run with the optional broker SDKs installed reads ~2 points higher because 7 SDK-gated tests skip in CI — trust the CI number), new execution/reconciliation/data-safety modules require 90%, and every broker adapter requires 80%. The four local audit commands check committed direct pins; CI additionally audits the complete resolved dependency tree in a clean hosted environment. `pyproject.toml` holds the coverage, Ruff, and mypy config (mypy is scoped to the identifier-named modules — the spaced-name master file is covered by `compileall` + the unittest suite instead). `.pre-commit-config.yaml` wires the check-only hooks; install them once with `pre-commit install`. # License Released under the MIT License — see [LICENSE](LICENSE). diff --git a/Tests/Dependencies/test_repository_policy.py b/Tests/Dependencies/test_repository_policy.py index 7a8ad41..39a3a15 100644 --- a/Tests/Dependencies/test_repository_policy.py +++ b/Tests/Dependencies/test_repository_policy.py @@ -108,7 +108,7 @@ def test_coverage_config_is_branch_enabled_and_preserves_overall_baseline(): config = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) assert config["tool"]["coverage"]["run"]["branch"] is True - assert config["tool"]["coverage"]["report"]["fail_under"] == 70.0 + assert config["tool"]["coverage"]["report"]["fail_under"] == 68.0 def test_mypy_covers_the_complete_identifier_named_cpr_ai_runtime(): diff --git a/docs/lld/testing-and-ci.md b/docs/lld/testing-and-ci.md index 7cd9a51..685f966 100644 --- a/docs/lld/testing-and-ci.md +++ b/docs/lld/testing-and-ci.md @@ -123,7 +123,7 @@ budgets are enforced from `coverage.json` by | Tier | Floor | Modules | |---|---|---| -| Repository floor | **70%** | everything (measured 71.4%; original MAT-110 baseline was 54.7%) | +| Repository floor | **68%** | everything (CI measures **69.1%**; original MAT-110 baseline was 54.7%) | | Safety / data-safety | **90%** | `broker_contract`, `execution_ledger`, `startup_exposure`, `trading_lifecycle`, `market_data_health`, `tick_bar_builder`, `next_open_entry`, `risk_sizing`, `order_splitting`, `secret_redaction` | | Broker adapters | **80%** | Kotak, Shoonya, Flattrade, Dhan | @@ -179,7 +179,7 @@ runtime. It asserts, without contacting any network: - dependency sets are exact (`==` pins) and Kotak uses its official Git tag; - CI runs the audit, branch coverage, and every dependency set; - Dependabot updates pip and GitHub Actions weekly; -- coverage config stays branch-enabled at the 70% floor; +- coverage config stays branch-enabled at the 68% floor; - every `cpr_ai_*.py` module is inside mypy's scope; - `env.example` documents every `_env_*` key the code reads (>300 found, as a sanity check that the AST walk still works); diff --git a/pyproject.toml b/pyproject.toml index b9d4e99..b5207fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -181,15 +181,22 @@ omit = [ [tool.coverage.report] # MAT-110 measured the branch-enabled runtime baseline (54.7%) before installing -# this gate. Coverage has grown well past it since, so the floor was raised to 70 -# against a measured 71.4% -- a floor 17 points below actual stops nothing. Per- -# module 90%/80% budgets are enforced from coverage.json by the small policy -# script because Coverage.py has only one global fail-under setting. +# this gate. Coverage has grown well past it since, so the floor was raised -- a +# floor 14 points below actual stops nothing. Per-module 90%/80% budgets are +# enforced from coverage.json by the small policy script because Coverage.py has +# only one global fail-under setting. # -# Raising this is deliberate and one-way: only move it UP, and only after a run -# shows real headroom. Lowering it to make a red build pass defeats the gate -- -# fix the tests instead. -fail_under = 70.0 +# 68.0 is set against CI's 69.1%. USE THE CI NUMBER, NOT A LOCAL ONE: a developer +# machine with the optional broker SDKs installed (neo_api_client, pyotp) runs 7 +# tests that CI's verify job skips, and reads ~71.4% -- about 2.3 points high. CI +# installs core + dev + ai + codex-ai only, deliberately (Kotak's official tag +# pins older pandas/requests and cannot share the audited core environment), so +# the hosted number is the honest one. +# +# Raising this is deliberate and one-way: only move it UP, and only after a CI +# run shows real headroom. Lowering it to make a red build pass defeats the gate +# -- fix the tests instead. +fail_under = 68.0 precision = 1 show_missing = true skip_covered = false diff --git a/scripts/check_coverage_thresholds.py b/scripts/check_coverage_thresholds.py index 2865276..ad0edd3 100644 --- a/scripts/check_coverage_thresholds.py +++ b/scripts/check_coverage_thresholds.py @@ -5,7 +5,7 @@ budgets to the live-money safety core and each broker adapter. Why this script exists at all: Coverage.py has exactly ONE global -``fail_under`` setting (pyproject pins it to the repository's 70% floor), +``fail_under`` setting (pyproject pins it to the repository's 68% floor), so the stricter 90%/80% per-module budgets have to be enforced from the JSON report by hand. The split is deliberate -- the global floor stops overall erosion, while these budgets stop a specific safety module from quietly losing From cdfa35978462ebc33fd08591b9f2d1f775cab45b Mon Sep 17 00:00:00 2001 From: DoRmAmMu1997 Date: Mon, 10 Aug 2026 19:55:56 +0530 Subject: [PATCH 4/5] docs: align architecture contracts with runtime Clarify restart limits, startup versus shutdown exposure scope, shared synchronization boundaries, and CPR code-owned invariants. Co-authored-by: Codex --- Dependencies/trading_lifecycle.py | 7 +++--- ...0001-single-process-thread-per-strategy.md | 18 ++++++++------ docs/hld/system-overview.md | 24 ++++++++++++------- docs/lld/cpr-codex-ai-agent.md | 8 ++++--- docs/lld/execution-and-brokers.md | 18 +++++++++----- docs/lld/master-runner.md | 8 ++++--- docs/lld/strategy-workers.md | 10 ++++---- 7 files changed, 59 insertions(+), 34 deletions(-) diff --git a/Dependencies/trading_lifecycle.py b/Dependencies/trading_lifecycle.py index 92f9c26..dea8f07 100644 --- a/Dependencies/trading_lifecycle.py +++ b/Dependencies/trading_lifecycle.py @@ -2,9 +2,10 @@ Stopping a worker thread is not the same as safely stopping a live trading process. Once shutdown begins, new entries must stay blocked while the runner -closes every tracked leg and asks the broker whether the account is flat. A -failed close keeps the process alive in reconciliation instead of allowing it -to report a clean shutdown. +closes every tracked leg and reconciles its own execution ledger to confirmed +flat. A failed close keeps the process alive in reconciliation instead of +allowing it to report a clean shutdown. The separate account-wide audit is +advisory because the operator may also hold manual positions in that account. This module deliberately performs no broker calls and never sleeps. The runner drives each transition and can inspect :meth:`retry_due` from its normal diff --git a/docs/adr/0001-single-process-thread-per-strategy.md b/docs/adr/0001-single-process-thread-per-strategy.md index 2b96de1..8539322 100644 --- a/docs/adr/0001-single-process-thread-per-strategy.md +++ b/docs/adr/0001-single-process-thread-per-strategy.md @@ -22,8 +22,10 @@ The workload's actual shape: Run everything in **one Python process** with **one `threading.Thread` per strategy**, one producer thread for market data, and one worker thread for -Telegram. Share exactly two mutable objects, both lock-guarded: -`SharedMarketDataStore` and the broker session behind `ExecutionSafetyCoordinator`. +Telegram. Keep cross-thread state behind explicit synchronization boundaries: +the `SharedMarketDataStore` aggregate, serialized broker access, lifecycle and +shutdown signals, and the optional Telegram queue. Strategy positions, P&L and +decision state remain worker-local. ## Options considered @@ -36,9 +38,9 @@ Telegram. Share exactly two mutable objects, both lock-guarded: | Scalability | Bounded by one machine — irrelevant at this size | | Team familiarity | Total | -**Pros:** the concurrency review surface is two objects; one broker session and -one lock make order serialization trivial; one log file; deployment is -`python algo.py run`; a shared in-memory feed needs no serialization. +**Pros:** the concurrency review surface is small and named; one serialized +broker boundary makes order coordination tractable; one log file; deployment is +`python algo.py run`; a shared in-memory feed needs no process serialization. **Cons:** no horizontal scale; one crash stops every strategy; the GIL caps CPU-bound work; a leaked exception in one thread can affect the process. @@ -89,8 +91,9 @@ that can disagree about "how much am I holding?" beats every other consideration ## Consequences -**Easier:** reasoning about concurrency (two objects); serializing orders; -deploying; debugging from one log; adding a strategy (add a thread). +**Easier:** reasoning about a small set of explicit synchronization boundaries; +serializing orders; deploying; debugging from one log; adding a strategy (add a +thread). **Harder:** surviving a crash (everything stops together); CPU-bound work (GIL); running strategies on separate machines (not possible without redesign). @@ -104,6 +107,7 @@ increment of this one). - [x] One lock-guarded `SharedMarketDataStore`. - [x] One lock-guarded broker session behind `ExecutionSafetyCoordinator`. +- [x] Thread-safe lifecycle, shutdown-event and Telegram-queue boundaries. - [x] Ten-second deadline **including** lock wait, so one hung call cannot stall the other workers ([ADR-0002](0002-broker-agnostic-execution-contract.md)). - [ ] Persist the execution ledger if unattended running is ever wanted. diff --git a/docs/hld/system-overview.md b/docs/hld/system-overview.md index 597929a..b1440ff 100644 --- a/docs/hld/system-overview.md +++ b/docs/hld/system-overview.md @@ -45,7 +45,7 @@ trues its bars up against official REST candles (see §5.2). | F2 | Each strategy independently decides entry/exit on NIFTY ATM options (some on multi-leg baskets). | | F3 | Execute on paper by default; execute live only when explicitly enabled for both the process and the strategy. | | F4 | Support more than one broker behind one interface, switchable by configuration. | -| F5 | Never lose track of live exposure — including across partial fills, lost responses, and restarts. | +| F5 | Track live exposure conservatively within a running process, including partial fills and lost responses. After a restart, refuse live startup when the broker account is not clean and require operator reconciliation. | | F6 | Report every entry/exit to Telegram, and per-strategy end-of-day P&L to a Google Sheet. | | F7 | Be entirely configuration-driven from one file; nothing hard-coded per run. | @@ -251,14 +251,22 @@ the month can be backfilled from the same log. | Telegram worker | 1 (if enabled) | Drain the event queue | | Main | 1 | Start, supervise, shut down | -**Shared mutable state is exactly two objects**, and both are lock-guarded: +Shared state is kept behind a small, explicit set of synchronization boundaries: -1. `SharedMarketDataStore` — many readers, one writer. -2. The broker session behind `ExecutionSafetyCoordinator` — many writers, serialized. +1. `SharedMarketDataStore` owns the lock-guarded OHLC, LTP and option-subscription + maps. It also aggregates the independently synchronized `MarketDataHealth`, + `ExecutionSafetyCoordinator` and `ExecutionLedger` safety components. +2. Broker-session access is serialized so workers cannot submit through the + shared client concurrently. +3. `TradingLifecycle` plus shutdown events coordinate process-wide and + worker-local stopping without interrupting a thread during an order. +4. The optional Telegram `queue.Queue` is the thread-safe hand-off between + strategy workers and the reporting worker. -Everything else a worker touches is its own. This is what makes a -thread-per-strategy model tractable for one maintainer: the concurrency review -surface is two objects, not thirty (see [ADR-0001](../adr/0001-single-process-thread-per-strategy.md)). +Positions, realised P&L and strategy decisions remain worker-local. The design +is tractable because every cross-thread path has a named owner and synchronization +rule—not because the process literally contains only two mutable objects (see +[ADR-0001](../adr/0001-single-process-thread-per-strategy.md)). The GIL is not a problem here because the workload is overwhelmingly I/O-bound (HTTP calls and sleeps) and the CPU work per bar is small pandas operations on @@ -292,7 +300,7 @@ The recurring shape: **an ambiguous state is treated as the dangerous state.** | Decision | What we gain | What we pay | ADR | |---|---|---|---| -| One process, thread per strategy | Two lock-guarded objects to reason about; trivial deployment; one log | No horizontal scale; one crash stops everything; GIL caps CPU-bound work | [0001](../adr/0001-single-process-thread-per-strategy.md) | +| One process, thread per strategy | A small set of explicit in-process synchronization boundaries; trivial deployment; one log | No horizontal scale; one crash stops everything; GIL caps CPU-bound work | [0001](../adr/0001-single-process-thread-per-strategy.md) | | Broker-agnostic contract | Swap brokers by config; one execution path to test | Every adapter must contain its broker's quirks rather than leak them | [0002](../adr/0002-broker-agnostic-execution-contract.md) | | Typed outcomes + quantity ledger | Cannot mistake an ack for a fill | More states for callers to handle; `UNKNOWN` needs an operator | [0003](../adr/0003-acknowledgement-is-not-a-fill.md) | | Paper by default, two flags for live | A single flag can never be enough to risk money | Two places to change; easy to think you are live when you are not | [0004](../adr/0004-paper-by-default-double-gate.md) | diff --git a/docs/lld/cpr-codex-ai-agent.md b/docs/lld/cpr-codex-ai-agent.md index 59ea725..e499404 100644 --- a/docs/lld/cpr-codex-ai-agent.md +++ b/docs/lld/cpr-codex-ai-agent.md @@ -85,9 +85,11 @@ Why **no-argument** tools: Indicator settings are fixed and documented: RSI 14 / Stochastic 14 / K 3 / D 3, zones 20 and 80; one equal-size add; 30-NIFTY-point and 2-NIFTY-point geometry -constants; 0.40 threshold. These live in `env.example` and are asserted by -`Tests/Dependencies/test_repository_policy.py` so the operator-facing -explanation cannot drift from the code. +constants; 0.40 threshold. These are deliberately **code-owned invariants**, not +`.env` knobs: indicator calculations live in `cpr_ai_context.py`, while the host +worker owns entry geometry and risk enforcement. The focused CPR AI context and +master-worker tests assert their behaviour. `env.example` contains only the +operator-selectable CPR AI settings listed in §7. --- diff --git a/docs/lld/execution-and-brokers.md b/docs/lld/execution-and-brokers.md index 712a024..84f5c9e 100644 --- a/docs/lld/execution-and-brokers.md +++ b/docs/lld/execution-and-brokers.md @@ -102,12 +102,18 @@ or flattens. If either read shows exposure, live workers do not start. **Shutdown** — `Dependencies/trading_lifecycle.py` blocks new entries the moment shutdown begins, then requires every tracked leg to be closed and the broker to -confirm flat before the session may be called clean. A failed close keeps the -process alive in reconciliation. - -Both boundaries are measured against **the runner's own ledger**, because the -operator trades manually in the same account. A non-flat *account* is advisory; -a non-flat *ledger* is an error. +reconcile the runner's ledger flat before the session may be called clean. A +failed close keeps the process alive in reconciliation. + +The two boundaries intentionally use different scopes: + +- Startup is **account-wide** because a fresh process has no trustworthy local + ledger yet. Any open order or relevant index-option position blocks live + startup and requires an operator decision. +- Shutdown is **runner-owned** because the in-process ledger identifies the legs + this run opened. Those legs must be broker-reconciled flat. Once they are, a + separate account-wide audit is advisory because remaining exposure may belong + to the operator's manual trading. --- diff --git a/docs/lld/master-runner.md b/docs/lld/master-runner.md index 30600fd..b0616de 100644 --- a/docs/lld/master-runner.md +++ b/docs/lld/master-runner.md @@ -15,7 +15,8 @@ up and go down. Its contract: 1. Nothing that can place an order starts before configuration is validated and the broker book is proven clean. 2. Nothing reports a clean shutdown until every tracked leg is closed and the - broker agrees the account is flat. + runner's execution ledger is broker-reconciled flat. A separate account-wide + audit then warns about manual or otherwise untracked exposure. Everything between those two statements is supervision. @@ -97,7 +98,7 @@ shutdown requested (end of day, max loss, operator, or fatal supervision event) │ └─ a REJECTED exit does NOT count as closed │ 3. _wait_for_shutdown_account_flat(...) - │ └─ ask the broker whether the account is flat + │ └─ despite the legacy function name, reconcile the RUNNER'S tracked ledger │ ├─ flat ──► continue │ └─ not flat ──► stay alive in RECONCILIATION; do NOT report clean │ @@ -115,7 +116,8 @@ shutdown requested (end of day, max loss, operator, or fatal supervision event) Step 3 is the one that matters. "Stopping the threads" and "safely stopping a live trading process" are different things, and `Dependencies/trading_lifecycle.py` exists to keep them separate: the process refuses to claim a clean exit while a -leg it opened is still open. +leg it opened is still open. Step 4 is deliberately advisory because an +account-wide position may belong to the operator rather than this runner. --- diff --git a/docs/lld/strategy-workers.md b/docs/lld/strategy-workers.md index 772b8e5..993b3db 100644 --- a/docs/lld/strategy-workers.md +++ b/docs/lld/strategy-workers.md @@ -10,11 +10,13 @@ ## 1. Responsibility One worker = one strategy = one thread. A worker owns its own positions, its own -P&L, its own risk knobs and its own poll cadence. Workers never talk to each -other; the only things they share are the market-data store and the broker lock. +P&L, its own risk knobs and its own poll cadence. Workers never call one another. +They share the market-data and execution-safety aggregate, lifecycle/shutdown +signals, and—when enabled—the Telegram event queue; each boundary is explicitly +synchronized. -That isolation is the reason a strategy can be added, disabled or blown up -without touching another one. +That separation is why a strategy can be added or disabled without coupling its +position and decision state to another worker. --- From 68ac534b8abe4d18ff053f2b9770482efa304cc3 Mon Sep 17 00:00:00 2001 From: DoRmAmMu1997 Date: Mon, 10 Aug 2026 16:28:44 +0530 Subject: [PATCH 5/5] fix(cpr-ai): align isolated Codex turn contract Co-authored-by: Codex --- .../CPR AI Agent/cpr_ai_agent.py | 5 +++- .../CPR AI Agent/cpr_ai_codex_subprocess.py | 23 +++++++++++++-- .../CPR AI Agent/cpr_ai_prompt.py | 29 ++++++++++++++----- .../CPR AI Agent/cpr_ai_schema.py | 6 ++-- .../CPR AI Agent/test_cpr_ai_context.py | 7 ++++- .../CPR AI Agent/test_cpr_ai_runtime.py | 11 +++++++ 6 files changed, 68 insertions(+), 13 deletions(-) diff --git a/Signal Generators/CPR AI Agent/cpr_ai_agent.py b/Signal Generators/CPR AI Agent/cpr_ai_agent.py index 251eaf1..ab167ae 100644 --- a/Signal Generators/CPR AI Agent/cpr_ai_agent.py +++ b/Signal Generators/CPR AI Agent/cpr_ai_agent.py @@ -448,7 +448,10 @@ def _run_turn(self, context: Mapping[str, Any], bar_signature: str) -> CPRAgentR from cpr_ai_schema import CPRAgentDecision return self.runner( - prompt=build_system_prompt(), + # Tell the prompt builder which configured model identifier must be + # echoed in the strict response. This remains advisory metadata; + # the host independently verifies it before accepting a decision. + prompt=build_system_prompt(model_used=self.model), context=context, bar_signature=bar_signature, model=self.model, diff --git a/Signal Generators/CPR AI Agent/cpr_ai_codex_subprocess.py b/Signal Generators/CPR AI Agent/cpr_ai_codex_subprocess.py index b2bb496..b8a7903 100644 --- a/Signal Generators/CPR AI Agent/cpr_ai_codex_subprocess.py +++ b/Signal Generators/CPR AI Agent/cpr_ai_codex_subprocess.py @@ -21,6 +21,21 @@ _REQUEST_KEYS = {"snapshot_path", "model", "reasoning_effort", "prompt", "output_schema"} _EXPECTED_TOOLS = ("session_levels", "momentum_vwap", "market_structure", "position_state") +# ``build_system_prompt()`` contains durable role, tool, and safety policy. The +# actual turn request stays short so those rules are stated once at the SDK's +# stronger developer-instruction layer instead of being repeated as user text. +_TURN_REQUEST = "Evaluate the current frozen CPR context and return one decision." +_ALLOWED_TURN_ITEM_TYPES = frozenset( + { + # The SDK records the prompt submitted through ``Thread.run`` as a + # completed user-message item. This is input bookkeeping, not a tool or + # capability used by Codex, so it is safe to ignore during action checks. + "userMessage", + "agentMessage", + "reasoning", + "mcpToolCall", + } +) def build_isolated_thread_config(snapshot_path: str, python_executable: str = sys.executable) -> dict[str, Any]: @@ -154,12 +169,16 @@ def _run_request(request: Mapping[str, Any]) -> dict[str, Any]: model=str(request["model"]), config=config, cwd=runtime_directory, + # The modular CPR prompt is policy for every turn, not merely the + # user's one-off task. Passing it through the SDK's dedicated + # field makes the intended instruction hierarchy explicit. + developer_instructions=str(request["prompt"]), ephemeral=True, sandbox=Sandbox.read_only, approval_mode=ApprovalMode.deny_all, ) result = thread.run( - str(request["prompt"]), + _TURN_REQUEST, approval_mode=ApprovalMode.deny_all, output_schema=request["output_schema"], effort=request["reasoning_effort"], @@ -174,7 +193,7 @@ def _run_request(request: Mapping[str, Any]) -> dict[str, Any]: "unexpected_actions": [ str(_item_value(_root_item(item), "type", "unknown")) for item in _item_value(result, "items", ()) - if _item_value(_root_item(item), "type", "") not in {"agentMessage", "reasoning", "mcpToolCall"} + if _item_value(_root_item(item), "type", "") not in _ALLOWED_TURN_ITEM_TYPES ], } diff --git a/Signal Generators/CPR AI Agent/cpr_ai_prompt.py b/Signal Generators/CPR AI Agent/cpr_ai_prompt.py index 1b18203..8003219 100644 --- a/Signal Generators/CPR AI Agent/cpr_ai_prompt.py +++ b/Signal Generators/CPR AI Agent/cpr_ai_prompt.py @@ -8,7 +8,7 @@ from __future__ import annotations -CPR_AI_PROMPT_VERSION = "cpr-srsi-vwap-context-v1" +CPR_AI_PROMPT_VERSION = "cpr-srsi-vwap-context-v2" _ROLE = """ROLE AND BOUNDARY You are an advisory CPR context analyst. Assess only complete five-minute bars. @@ -32,15 +32,30 @@ only when market_structure reports its eligible R1 candidate. Prefer HOLD/NONE whenever evidence conflicts or is incomplete.""" -_OUTPUT = f"""STRICT STRUCTURED OUTPUT +def _output_rules(model_used: str) -> str: + """Build output rules that echo the host's configured model exactly. + + ``model_used`` is dynamic configuration, so it cannot live in a fixed + module-level paragraph. Including it here keeps all prompt prose inside + the prompt builder instead of scattering instructions through SDK runtime + code. + """ + + return f"""STRICT STRUCTURED OUTPUT Return only CPRAgentDecision with exactly action, regime, setup, confidence, reasoning, model_used, and prompt_version. Valid actions are HOLD, ENTER_LONG, -ENTER_SHORT, EXIT, SCALE_IN. prompt_version must be {CPR_AI_PROMPT_VERSION}. -Never include entry, stop, target, trail, lots, quantity, symbol, expiry, broker, -venue, order, or any execution field.""" +ENTER_SHORT, EXIT, SCALE_IN. confidence must be an integer from 0 through 10. +model_used must exactly equal {model_used}. prompt_version must be +{CPR_AI_PROMPT_VERSION}. Never include entry, stop, target, trail, lots, +quantity, symbol, expiry, broker, venue, order, or any execution field.""" -def build_system_prompt(*, operator_approved_knowledge: str = "", discretionary_context: str = "") -> str: +def build_system_prompt( + *, + model_used: str = "gpt-5.6-terra", + operator_approved_knowledge: str = "", + discretionary_context: str = "", +) -> str: """Return one prompt while keeping future knowledge visibly separated. ``discretionary_context`` is a harmless compatibility alias while later @@ -57,7 +72,7 @@ def build_system_prompt(*, operator_approved_knowledge: str = "", discretionary_ # show exactly what changed without mixing it into permanent safety text. if knowledge: sections.append("FUTURE OPERATOR-APPROVED KNOWLEDGE\n" + knowledge) - sections.append(_OUTPUT) + sections.append(_output_rules(model_used)) return "\n".join(section.strip() for section in sections) + "\n" diff --git a/Signal Generators/CPR AI Agent/cpr_ai_schema.py b/Signal Generators/CPR AI Agent/cpr_ai_schema.py index 81f3874..bd99128 100644 --- a/Signal Generators/CPR AI Agent/cpr_ai_schema.py +++ b/Signal Generators/CPR AI Agent/cpr_ai_schema.py @@ -15,7 +15,7 @@ from collections.abc import Mapping from typing import Any, Literal -from pydantic import BaseModel, ConfigDict, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator CPRAction = Literal["HOLD", "ENTER_LONG", "ENTER_SHORT", "EXIT", "SCALE_IN"] CPRRegime = Literal["SIDEWAYS", "TRENDING", "UNDECIDED"] @@ -44,7 +44,9 @@ class CPRAgentDecision(BaseModel): action: CPRAction regime: CPRRegime setup: CPRSetup - confidence: int + # Field bounds become JSON-Schema minimum/maximum values. Codex therefore + # sees the same 0-10 contract that Pydantic enforces after the turn returns. + confidence: int = Field(ge=0, le=10) reasoning: str model_used: str prompt_version: str diff --git a/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_context.py b/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_context.py index 87dceb0..100e8bd 100644 --- a/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_context.py +++ b/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_context.py @@ -496,12 +496,17 @@ def test_mcp_server_exposes_exactly_four_no_argument_frozen_context_tools(tmp_pa def test_prompt_requires_tools_judgment_risk_boundary_and_future_knowledge_seam(): """The prompt must guide judgment while reserving execution for the host.""" - prompt = build_system_prompt(operator_approved_knowledge="Only after human approval.") + prompt = build_system_prompt( + model_used="configured-test-model", + operator_approved_knowledge="Only after human approval.", + ) assert all(name in prompt for name in EXPECTED_TOOL_NAMES) assert "SIDEWAYS" in prompt and "TRENDING" in prompt and "UNDECIDED" in prompt assert "breakout" in prompt.lower() and "breakdown" in prompt.lower() assert "SRSI" in prompt and "VWAP" in prompt and "PREMISE_EXIT" in prompt assert "host-owned" in prompt.lower() + assert "confidence" in prompt and "0 through 10" in prompt + assert "model_used" in prompt and "configured-test-model" in prompt assert "FUTURE OPERATOR-APPROVED KNOWLEDGE" in prompt assert CPR_AI_PROMPT_VERSION in prompt diff --git a/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_runtime.py b/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_runtime.py index 2fdf77e..fb0b5b9 100644 --- a/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_runtime.py +++ b/Tests/Signal Generators/CPR AI Agent/test_cpr_ai_runtime.py @@ -496,6 +496,10 @@ def runner(**kwargs): assert outcome.validation_code == "accepted_hold" assert observed["timeout_seconds"] == 17.5 + assert "model_used must exactly equal gpt-5.6-terra" in observed["prompt"] + confidence_schema = observed["output_schema"]["properties"]["confidence"] + assert confidence_schema["minimum"] == 0 + assert confidence_schema["maximum"] == 10 def test_timeout_returns_before_a_late_sdk_thread_finishes(): @@ -712,6 +716,10 @@ def run(self, prompt, *, approval_mode, output_schema, effort): {"approval_mode": approval_mode, "output_schema": output_schema, "effort": effort}, ) items = [ + # The real SDK includes the submitted turn input in + # ``TurnResult.items``. It is bookkeeping, not a capability + # used by Codex, so the host must not classify it as an action. + SimpleNamespace(root=SimpleNamespace(type="userMessage")), SimpleNamespace(root=SimpleNamespace(type="agentMessage")), SimpleNamespace(root=SimpleNamespace(type="reasoning")), *[ @@ -754,6 +762,8 @@ def thread_start(self, **kwargs): assert observed["start"]["config"] == codex_child.build_isolated_thread_config(str(tmp_path / "snapshot.json")) assert observed["start"]["approval_mode"] == "deny" + assert observed["start"]["developer_instructions"] == "prompt" + assert observed["run"][0] == "Evaluate the current frozen CPR context and return one decision." assert observed["run"][1] == {"approval_mode": "deny", "output_schema": {"type": "object"}, "effort": "medium"} assert [call["tool"] for call in response["tool_calls"]] == list(EXPECTED_TOOL_NAMES) assert [call["status"] for call in response["tool_calls"]] == ["completed"] * 4 @@ -763,6 +773,7 @@ def thread_start(self, **kwargs): "total_tokens": 5, "model_context_window": 128000, } + assert response["unexpected_actions"] == [] assert all(value is False for value in observed["start"]["config"]["features"].values()) assert set(observed["start"]["config"]["features"]) >= { "shell_tool",