diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 64c463b..f19d4c3 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -11,12 +11,16 @@ jobs: offline-validation: runs-on: ubuntu-latest timeout-minutes: 10 + defaults: + run: + working-directory: labs/lab-01-continuous-risk-identification steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: '3.12' cache: pip + cache-dependency-path: labs/lab-01-continuous-risk-identification/requirements.txt - run: python -m pip install --requirement requirements.txt - run: python -m unittest -v - run: python evaluation.py @@ -24,6 +28,7 @@ jobs: - run: python execute_notebook.py - name: Reject likely committed API keys shell: bash + working-directory: . run: | if grep -RIE --exclude-dir=.git --exclude='*.png' 'sk-(proj-)?[A-Za-z0-9_-]{24,}' .; then echo 'Potential API key detected.' >&2 diff --git a/.gitignore b/.gitignore index 97c6ea6..2b71c77 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ -.env* -runtime/ -__pycache__/ -.venv/ -.ipynb_checkpoints/ +**/.env* +**/runtime/ +**/data/ +**/__pycache__/ +**/.venv/ +**/.ipynb_checkpoints/ +**/*.zip diff --git a/README.md b/README.md index c55ca15..69fc599 100644 --- a/README.md +++ b/README.md @@ -1,164 +1,22 @@ -# Lab 01: Continuous risk identification +# risk.engineer labs -Build the first stage of an engineered cyber risk lifecycle: a repeatable data pipeline that turns changing system evidence into cited candidate risk statements. +[![validate labs](https://github.com/envokeME/risk-engineer-labs/actions/workflows/validate.yml/badge.svg?branch=main)](https://github.com/envokeME/risk-engineer-labs/actions/workflows/validate.yml) -```text -Okta + Zscaler mock records - ↓ -source registry + immutable landing receipt - ↓ -validation + quarantine - ↓ -OCSF 1.8 normalization → Parquet - ↓ -HR + service + objective enrichment - ↓ -DuckDB correlation and bounded clustering - ↓ -agent investigation + explicit dispositions - ↓ -candidate risk statements → human review -``` +Executable labs for practitioners building the cyber risk lifecycle as a data system. These labs show how operational evidence can become reviewable risk knowledge—not another spreadsheet that waits for an annual assessment. -![Pipeline reduction from source records to statements](assets/pipeline-funnel.png) +## Labs -The important number is not 49,200. It is **six**: code reduces each collection window to six bounded investigation clusters. The model must inspect all six; it is not told which are supported. +| Lab | Question | Status | +|---|---|---| +| [01 · Continuous risk identification](labs/lab-01-continuous-risk-identification/) | Can changing system evidence identify a risk condition and express it in business terms? | Learning prototype | -![Deterministic evidence joins and bounded interpretation](assets/evidence-to-statement.png) +## Roadmap -## Run locally +- **Lab 02 · Emerging risk discovery** — expand beyond the known offboarding scenario using temporal evidence graphs and weak-signal research. +- **Lab 03 · Risk analysis** — estimate likelihood and consequence only after a scenario has passed human review. +- **Lab 04 · Risk response** — connect approved treatment decisions to owners, controls and measurable work. +- **Lab 05 · Continuous monitoring** — detect material changes in accepted scenarios, assumptions and treatments. -Requires Python 3.12. - -```sh -python -m venv .venv -# Windows: .venv\Scripts\activate -# macOS/Linux: source .venv/bin/activate -python -m pip install -r requirements.txt -python -m unittest -v -python -m jupyterlab -``` - -Open `lab-01-continuous-risk-identification.ipynb` and run all cells. The first run deterministically generates: - -- 8,000 Okta System Log-shaped records; -- 40,000 Zscaler NSS-shaped web records; -- 1,000 HR worker records; -- 200 business-service context records. - -The notebook defaults to replaying saved output from actual bounded model runs over this mock evidence. No key or API charge is required. - -For a fresh model run, set `OPENAI_API_KEY` in your terminal environment, set `LIVE = True` in the notebook, and run all cells. Never paste a key into the notebook or repository. Completed identical runs are cached in `runtime/ocsf_history.sqlite`. - -## Run with Docker - -```sh -docker compose up --build -``` - -Open the loopback Jupyter URL printed in the container logs. Authentication remains enabled. Generated data and run history use named volumes; code comes from the image, so rebuilding does not leave an older notebook shadowing the new one. - -```sh -docker compose down # retain data and history -docker compose down -v # deliberately delete them -``` - -Docker image build, fourteen tests and full notebook replay were verified locally on 2026-09-06. Verification containers had networking disabled and no API key or host mounts. Compose browser startup and repository CI remain unverified. The local Python path has also been executed. - -## What the lab teaches - -1. Start with a business objective and target-profile outcome. -2. Register every supported source and its role, format, adapter, record ID and batch field. -3. Land source files by content hash and issue a validated ingestion receipt. -4. Quarantine malformed rows and normalize valid security events to OCSF. -5. Keep HR and business context outside the security-event schema. -6. Use DuckDB and Parquet to join and reduce large event volumes into bounded clusters. -7. Give the agent parameterized timeline/context tools—not raw telemetry or arbitrary SQL. -8. Require a disposition for every cluster: candidate, insufficient evidence, contradiction, or ambiguity. -9. Reject invented citations and assessment language in code. -10. Evaluate known synthetic cases and preserve stable scenario IDs across collection batches. - -## Evidence states - -- `supported`: the narrow retained-access condition has a complete evidence chain. -- `not-supported`: evidence contradicts that condition in this collection window. -- `unknown`: the join is ambiguous or incomplete. - -A supported condition is not proof of malicious use or harm. The model adds a plausible threat event and potential consequence as a labeled hypothesis. Every output remains `candidate-human-review-required`. - -## Boundaries - -- All identities, events, objectives, services, and hostnames are fictional. -- Source records imitate documented vendor shapes; this package is not an Okta or Zscaler connector. -- OCSF makes security events interoperable. It does not identify risk on its own. -- NIST CSF `PR.AA-05` is target-profile context, not a score or maturity rating. -- This lab performs identification only: no likelihood, severity, scoring, priority, treatment, acceptance, or automated risk-register write. -- “Continuous” means repeatable collection batches, stable scenario identity, durable history, and explicit changing evidence state. Scheduling and production pagination are future connector concerns. - -## Rebuild and verify - -### Risk-identification contracts - -`schemas/RiskEvidenceBundle.schema.json` defines scope, cited facts, business context and evidence limitations. The investigation tool returns this validated object without leaking the hidden benchmark disposition. -`schemas/CandidateRiskScenario.schema.json` defines the resulting condition, threat event, potential consequence, statement, citations, assumptions and human review status. Code validates it before persistence. -Both contracts are experimental version 1.0.0. OCSF remains the underlying event schema. HR is optional in the general contract; the currently implemented offboarding rule requires it. The model currently reasons within that selected scenario family. - -### Bring your own exports - -Prepare a folder containing `raw/okta_system_log.jsonl`, `raw/zscaler_nss_web.jsonl`, `raw/hr_workers.jsonl`, and `context/business_services.json`. Use the generated examples as the current adapter field contract. This is an explicit mapping step, not support for arbitrary vendor exports. Add `_collection.batch_id` to each security event. - -```sh -python import_exports.py /path/to/exports /path/to/new-workspace -``` - -The importer validates JSON, copies files into a new workspace, creates content-addressed landing copies, calculates checksums, counts collection batches and issues `data/landing/ingestion_receipt.json`. Existing destinations are refused and source files remain unchanged. Missing or duplicate source IDs are recorded as envelope warnings for later quarantine; they are not silently cleaned. Then, from Python launched in the lab directory: - -```python -from pipeline import correlate -result = correlate(1, root='/path/to/new-workspace') # local processing only -``` - -Inspect and minimize sensitive data before explicitly calling `engine.run(1, root=...)`, which sends evidence facts and context to the configured model provider. No model calls are made by the importer. Live connectors and general scenario discovery remain future work. - -```sh -python fixture_factory.py -python pipeline.py -python engine.py # live API calls; requires a private key -python evaluation.py # evaluate saved real model traces against the reference -python exercise.py # change source records in a temporary workspace; no API call -python build_notebook.py -python execute_notebook.py -python -m unittest -v -python package_lab.py -``` - -Key files: - -- `fixture_factory.py` — deterministic high-volume vendor-shaped mock sources. -- `config/source_registry.json` — supported source contracts and scenario requirements. -- `ingestion.py` — content-addressed landing, checksums, batch counts and receipt validation. -- `schemas/IngestionReceipt.schema.json` — machine-readable ingestion receipt contract. -- `pipeline.py` — validation, OCSF mapping, Parquet warehouse, DuckDB joins, investigation clusters. -- `engine.py` — LangGraph workflow, bounded investigation tools, abstention, output guardrails, SQLite replay. -- `evaluation.py` and `evals/` — twelve-case synthetic regression benchmark and scorecard. -- `visuals.py` — native Jupyter plots generated from pipeline results. -- `examples/` — saved actual model outputs and inspectable tool traces. -- `test_engine.py` — offline evidence-state, OCSF, guardrail, and replay tests. - -## References - -- [OCSF schema](https://github.com/ocsf/ocsf-schema) -- [Okta System Log API](https://developer.okta.com/docs/reference/system-log-query/) -- [Zscaler and Cribl deployment guide](https://help.zscaler.com/downloads/zscaler-technology-partners/operations/zscaler-and-cribl-deployment-guide/Zscaler-Cribl-Deployment-Guide-FINAL.pdf) -- [NIST CSF 2.0](https://nvlpubs.nist.gov/nistpubs/CSWP/NIST.CSWP.29.pdf) -- [NIST SP 800-30 Rev. 1](https://nvlpubs.nist.gov/nistpubs/legacy/sp/nistspecialpublication800-30r1.pdf) - -## What the current result proves - -The saved real model runs close twelve clusters across two batches and match all twelve authored labels. The deterministic reference also matches twelve. This is evidence of known-case agreement, not proof that the model discovers more risks. Its proposed contribution is inspectable investigation and business-language drafting. See SEMANTIC-REVIEW.md for specific wording issues that still require human judgment. - -The method is deliberately narrow: a successful authentication and allowed web activity must follow termination and any observed disable. A later disable contradicts current support only within that collection; it does not prove every session was revoked. Web activity is correlated by login and service, not by a shared session identifier. Missing evidence remains unresolved. Expected HR identity never substitutes for an observed identity. - -The ingestion receipt proves the identity and contents of a received file. It does not prove the source sent every event. Live polling, source watermarks, late data handling and broader scenario discovery are extension work. +The roadmap is directional, not a release commitment. Each lab will keep raw evidence, transformations, model output and human decisions visibly separate. Released under the [MIT License](LICENSE). diff --git a/.dockerignore b/labs/lab-01-continuous-risk-identification/.dockerignore similarity index 100% rename from .dockerignore rename to labs/lab-01-continuous-risk-identification/.dockerignore diff --git a/Dockerfile b/labs/lab-01-continuous-risk-identification/Dockerfile similarity index 100% rename from Dockerfile rename to labs/lab-01-continuous-risk-identification/Dockerfile diff --git a/labs/lab-01-continuous-risk-identification/README.md b/labs/lab-01-continuous-risk-identification/README.md new file mode 100644 index 0000000..5d39d30 --- /dev/null +++ b/labs/lab-01-continuous-risk-identification/README.md @@ -0,0 +1,225 @@ +# Lab 01 · Continuous risk identification + +This lab is for GRC engineers, security engineers, risk practitioners and technical leaders who want risk identification to operate from evidence—not wait for the next assessment. You will build a repeatable pipeline that joins changing system activity to business context and produces cited candidate risk statements for human review. + +The problem it solves: security tools continuously produce events, but a risk register usually records generalized risks by hand. This lab shows how repeated collection can reveal where a risk condition is present, absent or still uncertain as the evidence changes. + +## Detection or risk identification? + +A SIEM or IGA rule can fire on one event today. That is **detection**. + +This lab performs **risk identification** when it: + +1. observes a recurring pattern across collection windows; +2. connects the pattern to a business service and objective; +3. preserves the supporting and conflicting evidence; and +4. expresses the result as **condition → threat event → potential consequence**. + +That turns operational evidence into a candidate statement about a control deficiency and its business exposure. It does not prove harm, assess likelihood, assign severity or make a risk decision. + +“Continuous” means the same pipeline collects the next evidence window, repeats the joins, preserves a stable scenario identity and records whether support for that scenario changed. The lab uses repeatable batches rather than claiming a production streaming service. + +## A risk statement produced by the lab + +> **Candidate risk statement** +> +> If terminated user P0005 gains access through successful authentication and allowed web use post-termination, then unauthorized access or data exposure may occur, affecting financial-reporting objectives. + +| Field | Saved run output | +|---|---| +| Observed condition | A terminated worker’s matching identity authenticated and accessed the financial reporting service after termination. | +| Threat event | Unauthorized access through lingering credentials or access tokens. | +| Potential consequence | Unauthorized access or data exposure affecting the financial reporting workspace. | +| Business objective | Financial reporting | +| Cited evidence | `HR-0005`, `OKTA-P5`, `ZIA-P5` | +| Review state | `candidate-human-review-required` | + +This is the exact model-generated statement retained in `examples/batch-1.json`. The shorter observed-condition wording in the table is editorial explanation; the saved source remains unchanged. + +## What you will build + +```text +Okta + Zscaler-shaped evidence HR + business context + \ / + source registration + ↓ + content-addressed landing + ↓ + validation + quarantine + ↓ + OCSF normalization + ↓ + DuckDB correlation + clustering + ↓ + bounded LLM investigation + ↓ + cited candidate statements → human review +``` + +![Pipeline reduction from source records to statements](assets/pipeline-funnel.png) + +The important number is not 49,200 source records. It is **six investigation clusters per collection window**. Deterministic code performs the high-volume processing. The model receives bounded tools and must inspect and close every cluster. + +![Deterministic evidence joins and bounded interpretation](assets/evidence-to-statement.png) + +## Why the LLM is here + +The LLM is mandatory in the live demonstration because the lesson is the **governance harness around model reasoning**: + +- it cannot query arbitrary SQL or read raw telemetry; +- it must use parameterized timeline and business-context tools; +- it must inspect every cluster; +- it must choose one forced disposition; +- submitted statements must cite the exact evidence set; +- code rejects assessment language and unsupported citations; +- every result remains subject to human review. + +Lab 01 uses a familiar offboarding risk on purpose. The novelty is not inventing that risk. It is building the evidence pipeline and guardrails before expanding to open-ended discovery in Lab 02. + +### Saved model run + +- Provider interface: **OpenAI Responses API** (`POST /v1/responses`) +- Exact model snapshot: **`gpt-4.1-mini-2025-04-14`** +- Workflow revision: **`investigation-harness-3`** +- OCSF version: **1.8.0** + +Replay mode uses the saved responses and makes no API call. A live run uses the pinned model snapshot above and may vary because model output is not deterministic. + +## Run the lab + +### Local Python + +Requires Python 3.12. + +```sh +python -m venv .venv +# Windows: .venv\Scripts\activate +# macOS/Linux: source .venv/bin/activate +python -m pip install -r requirements.txt +python -m unittest -v +python -m jupyterlab +``` + +Open `lab-01-continuous-risk-identification.ipynb`, select **Restart Kernel and Run All Cells**, and confirm. The notebook defaults to replay mode, so no key or API charge is required. + +The first run creates fictional but realistic volumes: + +- 8,000 Okta System Log-shaped records; +- 40,000 Zscaler NSS-shaped web records; +- 1,000 HR worker records; +- 200 business-service records. + +### Docker + +From this lab directory: + +```sh +docker compose up --build +``` + +Open the loopback Jupyter URL printed in the container logs. Authentication remains enabled. To stop it: + +```sh +docker compose down # retain generated data and history +docker compose down -v # deliberately remove generated volumes +``` + +Docker image build and full offline notebook replay were verified on 2026-09-06. The current sixteen-test suite also passes locally, including two connector-boundary tests. + +## Connect a real API + +The included Okta System Log connector is the live ingestion entry point for this scenario. It collects paginated records into the same raw JSONL contract used by the mock source. It does **not** send those records to the LLM. + +```sh +export OKTA_DOMAIN="https://your-org.okta.com" +export OKTA_API_TOKEN="set-this-outside-the-repository" +python connectors/okta_system_log.py \ + --since 2026-09-01T00:00:00Z \ + --until 2026-09-02T00:00:00Z \ + --batch-id 1 \ + --output /path/to/new-workspace/raw/okta_system_log.jsonl +``` + +On PowerShell, use `$env:OKTA_DOMAIN` and `$env:OKTA_API_TOKEN`. The connector enforces HTTPS, follows only same-origin pagination links, caps pages and records, writes through a temporary file, and never logs the token. It is tested with a fake API transport; a live Okta tenant was not used for release validation. + +CrowdStrike, Zscaler and other sources should implement the same **collect → raw source contract → landing receipt** boundary. A connector only collects evidence. A scenario pack defines which sources and joins are sufficient to investigate a particular risk condition. CrowdStrike detections alone do not provide every source required by this offboarding scenario. + +## Follow the evidence pipeline + +1. **Register** each source, its role and its expected record identifiers. +2. **Collect or import** raw evidence without changing source values. +3. **Land** each file by content hash and issue an ingestion receipt. +4. **Validate** rows; quarantine malformed records instead of silently fixing them. +5. **Normalize** valid security events to OCSF while keeping HR and business context separate. +6. **Join and reduce** events into bounded investigation clusters with DuckDB and Parquet. +7. **Investigate** every cluster with constrained model tools. +8. **Validate and persist** candidate statements, abstentions and contradictions. +9. **Repeat** the collection and compare the stable scenario across time. + +## Evidence states + +- `supported` — the narrow evidence condition is complete in this window; +- `not-supported` — collected evidence contradicts that condition in this window; +- `unknown` — the identity join or activity evidence is incomplete or ambiguous. + +Absence of an event does not prove a risk was resolved. An allowed web event does not prove a download, malicious use or shared authenticated session. + +## Use your own exported files + +Prepare a new folder with `raw/okta_system_log.jsonl`, `raw/zscaler_nss_web.jsonl`, `raw/hr_workers.jsonl`, and `context/business_services.json`. The generated fixtures define the current field contract. + +```sh +python import_exports.py /path/to/exports /path/to/new-workspace +``` + +The importer preserves the source files, lands content-addressed copies, records checksums and produces `data/landing/ingestion_receipt.json`. Then run the deterministic pipeline: + +```python +from pipeline import correlate +result = correlate(1, root="/path/to/new-workspace") +``` + +Inspect and minimize sensitive data before calling `engine.run(...)`, because a live run sends the bounded evidence bundle and business context to OpenAI. + +## Verify and rebuild + +```sh +python fixture_factory.py +python pipeline.py +python engine.py # live API calls; requires OPENAI_API_KEY +python evaluation.py +python exercise.py # mutates temporary mock inputs; no API call +python build_notebook.py +python execute_notebook.py +python -m unittest -v +``` + +Key implementation files: + +- `connectors/` — live collection boundary and connector guidance; +- `config/source_registry.json` — registered evidence sources and scenario requirements; +- `ingestion.py` — content-addressed landing and receipts; +- `pipeline.py` — OCSF mapping, Parquet storage and DuckDB correlation; +- `engine.py` — bounded LangGraph investigation and output enforcement; +- `schemas/` — evidence, ingestion and candidate-statement contracts; +- `examples/` — saved model outputs and complete tool traces; +- `evals/` and `test_engine.py` — regression labels, scorecard and offline tests. + +## Boundaries + +- Every person, service, objective, hostname and event in replay mode is fictional. +- The live Okta connector is an extension point, not a certified vendor integration. +- File receipts prove what the pipeline received, not that a source supplied every event. +- This scenario uses a known offboarding-risk family; it does not perform open-ended risk discovery. +- Identification stops before likelihood, impact, scoring, prioritization, treatment or acceptance. +- The model matched all twelve authored dispositions, but the deterministic reference did too. The benchmark does not prove model superiority. +- Model wording still requires human judgment; see [SEMANTIC-REVIEW.md](SEMANTIC-REVIEW.md). + +## References + +- [OCSF schema](https://github.com/ocsf/ocsf-schema) +- [Okta System Log API](https://developer.okta.com/docs/reference/system-log-query/) +- [NIST CSF 2.0](https://nvlpubs.nist.gov/nistpubs/CSWP/NIST.CSWP.29.pdf) +- [NIST SP 800-30 Rev. 1](https://nvlpubs.nist.gov/nistpubs/legacy/sp/nistspecialpublication800-30r1.pdf) + +Released under the repository [MIT License](../../LICENSE). diff --git a/SEMANTIC-REVIEW.md b/labs/lab-01-continuous-risk-identification/SEMANTIC-REVIEW.md similarity index 100% rename from SEMANTIC-REVIEW.md rename to labs/lab-01-continuous-risk-identification/SEMANTIC-REVIEW.md diff --git a/SPEC.md b/labs/lab-01-continuous-risk-identification/SPEC.md similarity index 100% rename from SPEC.md rename to labs/lab-01-continuous-risk-identification/SPEC.md diff --git a/VALIDATION.md b/labs/lab-01-continuous-risk-identification/VALIDATION.md similarity index 76% rename from VALIDATION.md rename to labs/lab-01-continuous-risk-identification/VALIDATION.md index a36bb4a..ba24954 100644 --- a/VALIDATION.md +++ b/labs/lab-01-continuous-risk-identification/VALIDATION.md @@ -14,7 +14,7 @@ Engine revision: investigation-harness-3. Pinned model: gpt-4.1-mini-2025-04-14. - Twelve of twelve authored dispositions matched. Candidate precision and recall are 1.0 on these twelve cases; citation sets, ordered tool-use requirements and tool budgets passed. - The deterministic reference also gets twelve of twelve. There is no demonstrated detection improvement from the model on this benchmark. - The notebook executed in a fresh Python kernel in replay mode, including the source-mutation exercise. Saved trace versions and local evidence must match before replay. -- Fourteen offline tests cover OCSF projection, identity ambiguity and absence, temporal order, citations, assessment scope, ingestion integrity, cache behavior and scorecard logic. The test substitute cannot pass the live trace check. +- Sixteen offline tests cover OCSF projection, identity ambiguity and absence, temporal order, citations, assessment scope, ingestion integrity, cache behavior, scorecard logic, API pagination and cross-origin credential protection. The API connector tests use an injected fake transport; no live tenant is claimed. The model test substitute cannot pass the live trace check. The 86,288 total tokens above describe only the two retained successful runs. Earlier development attempts, including a failed investigation, incurred additional usage. These are API-reported token totals, not dollar costs or a throughput benchmark. @@ -40,9 +40,9 @@ The executed notebook produced the investigation board, source-volume figure, pi Public packaging uses an explicit file allowlist and a credential-pattern scan. Generated source data, private environment files and runtime history are excluded. The archive includes code that regenerates fictional inputs, saved real model traces and executed notebook outputs. -The archive was extracted into an empty temporary directory and tested with OPENAI_API_KEY removed from the subprocess environment: fourteen tests, saved-output evaluation and fresh-kernel notebook execution all passed. This checks clean files using the existing pinned Python environment; it is not an independent dependency installation or a Docker run. +The original single-lab archive was extracted into an empty temporary directory and tested with OPENAI_API_KEY removed from the subprocess environment: fourteen tests, saved-output evaluation and fresh-kernel notebook execution all passed. After the multi-lab repository restructure and connector addition, the current sixteen-test suite, saved-output evaluation and fresh-kernel notebook execution passed from the new lab directory. This uses the existing pinned Python environment. -Docker verification passed on 2026-09-06 after enabling the Windows WSL 2 prerequisites and rebooting. Docker Desktop 4.89.0 / Engine 29.7.2 built the image from python:3.12-slim with a fresh dependency installation. Fourteen tests and full fresh-kernel notebook execution passed in separate disposable containers using --network none, no API credentials and no host mounts. Image ID: sha256:59817beaa2a8fce9f6158f0e3f9abe3dadc134d1b8aa99995c9ee6f3a717084a. This verifies offline container replay, not live model calls, Compose browser startup or repository CI execution. +Docker verification passed on 2026-09-06 after enabling the Windows WSL 2 prerequisites and rebooting. Docker Desktop 4.89.0 / Engine 29.7.2 built the image from python:3.12-slim with a fresh dependency installation. The then-current fourteen tests and full fresh-kernel notebook execution passed in separate disposable containers using --network none, no API credentials and no host mounts. Image ID: sha256:59817beaa2a8fce9f6158f0e3f9abe3dadc134d1b8aa99995c9ee6f3a717084a. GitHub CI independently rebuilds the current code and executes the current suite. This local record does not claim a live model call, live Okta collection or Compose browser test. ## Deliberate boundaries diff --git a/assets/agent-investigation.png b/labs/lab-01-continuous-risk-identification/assets/agent-investigation.png similarity index 100% rename from assets/agent-investigation.png rename to labs/lab-01-continuous-risk-identification/assets/agent-investigation.png diff --git a/assets/candidate-risk-register.png b/labs/lab-01-continuous-risk-identification/assets/candidate-risk-register.png similarity index 100% rename from assets/candidate-risk-register.png rename to labs/lab-01-continuous-risk-identification/assets/candidate-risk-register.png diff --git a/assets/evidence-history.png b/labs/lab-01-continuous-risk-identification/assets/evidence-history.png similarity index 100% rename from assets/evidence-history.png rename to labs/lab-01-continuous-risk-identification/assets/evidence-history.png diff --git a/assets/evidence-to-statement.png b/labs/lab-01-continuous-risk-identification/assets/evidence-to-statement.png similarity index 100% rename from assets/evidence-to-statement.png rename to labs/lab-01-continuous-risk-identification/assets/evidence-to-statement.png diff --git a/assets/pipeline-funnel.png b/labs/lab-01-continuous-risk-identification/assets/pipeline-funnel.png similarity index 100% rename from assets/pipeline-funnel.png rename to labs/lab-01-continuous-risk-identification/assets/pipeline-funnel.png diff --git a/assets/source-volume.png b/labs/lab-01-continuous-risk-identification/assets/source-volume.png similarity index 100% rename from assets/source-volume.png rename to labs/lab-01-continuous-risk-identification/assets/source-volume.png diff --git a/build_notebook.py b/labs/lab-01-continuous-risk-identification/build_notebook.py similarity index 100% rename from build_notebook.py rename to labs/lab-01-continuous-risk-identification/build_notebook.py diff --git a/compose.yaml b/labs/lab-01-continuous-risk-identification/compose.yaml similarity index 100% rename from compose.yaml rename to labs/lab-01-continuous-risk-identification/compose.yaml diff --git a/config/source_registry.json b/labs/lab-01-continuous-risk-identification/config/source_registry.json similarity index 100% rename from config/source_registry.json rename to labs/lab-01-continuous-risk-identification/config/source_registry.json diff --git a/labs/lab-01-continuous-risk-identification/connectors/README.md b/labs/lab-01-continuous-risk-identification/connectors/README.md new file mode 100644 index 0000000..1b43cd7 --- /dev/null +++ b/labs/lab-01-continuous-risk-identification/connectors/README.md @@ -0,0 +1,7 @@ +# API connector boundary + +A connector has one job: collect source records and write the registered raw-file contract. It must not decide whether a record represents risk. + +The included Okta connector demonstrates pagination, time bounds, collection metadata, guardrails and atomic output. It directly supplies the Okta evidence used by Lab 01. Other vendors should follow this boundary, but each new source still needs an explicit field mapper and scenario requirement. + +Credentials belong in environment variables or an enterprise secret manager. Never place them in a notebook, connector configuration file, fixture or Git commit. diff --git a/labs/lab-01-continuous-risk-identification/connectors/okta_system_log.py b/labs/lab-01-continuous-risk-identification/connectors/okta_system_log.py new file mode 100644 index 0000000..8d8bb5e --- /dev/null +++ b/labs/lab-01-continuous-risk-identification/connectors/okta_system_log.py @@ -0,0 +1,113 @@ +"""Collect Okta System Log pages into Lab 01's registered raw JSONL contract. + +Release validation uses an injected fake transport. A live tenant is not required. +""" +from __future__ import annotations + +import argparse +import json +import os +import tempfile +import urllib.parse +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + + +def _next_link(value: str | None) -> str | None: + if not value: + return None + for part in value.split(","): + sections = [item.strip() for item in part.split(";")] + if len(sections) > 1 and sections[1] == 'rel="next"': + return sections[0].strip("<>") + return None + + +def _validate_origin(expected: str, candidate: str) -> None: + expected_url = urllib.parse.urlsplit(expected) + candidate_url = urllib.parse.urlsplit(candidate) + if expected_url.scheme != "https" or candidate_url.scheme != "https": + raise ValueError("Okta collection requires HTTPS") + if (expected_url.scheme, expected_url.netloc) != (candidate_url.scheme, candidate_url.netloc): + raise ValueError("Refusing to forward credentials to a different origin") + + +def _http_get(url: str, token: str) -> tuple[list[dict], dict[str, str]]: + request = urllib.request.Request( + url, + headers={"Authorization": f"SSWS {token}", "Accept": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=60) as response: + return json.loads(response.read()), dict(response.headers.items()) + + +def collect( + domain: str, + token: str, + since: str, + until: str, + batch_id: int, + output: Path | str, + *, + max_pages: int = 100, + max_records: int = 100_000, + transport=_http_get, +) -> dict: + """Collect bounded, paginated records and atomically publish one JSONL file.""" + domain = domain.rstrip("/") + query = urllib.parse.urlencode({"since": since, "until": until, "limit": 1000}) + url = f"{domain}/api/v1/logs?{query}" + _validate_origin(domain, url) + records: list[dict] = [] + pages = 0 + collected_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + while url: + if pages >= max_pages: + raise RuntimeError("Okta page limit reached before collection completed") + _validate_origin(domain, url) + page, headers = transport(url, token) + if not isinstance(page, list) or not all(isinstance(item, dict) for item in page): + raise ValueError("Okta response must be a JSON array of objects") + for record in page: + enriched = dict(record) + enriched["_collection"] = { + "source_id": "okta-system-log", + "batch_id": batch_id, + "collected_at": collected_at, + "window_start": since, + "window_end": until, + } + records.append(enriched) + if len(records) > max_records: + raise RuntimeError("Okta record limit reached before collection completed") + pages += 1 + link = next((value for key, value in headers.items() if key.lower() == "link"), None) + url = _next_link(link) + + destination = Path(output) + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile("w", encoding="utf-8", dir=destination.parent, delete=False) as handle: + temporary = Path(handle.name) + for record in records: + handle.write(json.dumps(record, separators=(",", ":")) + "\n") + temporary.replace(destination) + return {"source_id": "okta-system-log", "pages": pages, "records": len(records), "output": str(destination)} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Collect bounded Okta System Log evidence for Lab 01") + parser.add_argument("--since", required=True) + parser.add_argument("--until", required=True) + parser.add_argument("--batch-id", required=True, type=int) + parser.add_argument("--output", required=True) + parser.add_argument("--max-pages", type=int, default=100) + parser.add_argument("--max-records", type=int, default=100_000) + args = parser.parse_args() + domain = os.environ.get("OKTA_DOMAIN", "") + token = os.environ.get("OKTA_API_TOKEN", "") + if not domain or not token: + raise SystemExit("Set OKTA_DOMAIN and OKTA_API_TOKEN outside the repository") + print(collect(domain, token, args.since, args.until, args.batch_id, args.output, + max_pages=args.max_pages, max_records=args.max_records)) diff --git a/contracts.py b/labs/lab-01-continuous-risk-identification/contracts.py similarity index 100% rename from contracts.py rename to labs/lab-01-continuous-risk-identification/contracts.py diff --git a/engine.py b/labs/lab-01-continuous-risk-identification/engine.py similarity index 100% rename from engine.py rename to labs/lab-01-continuous-risk-identification/engine.py diff --git a/evals/ground_truth.json b/labs/lab-01-continuous-risk-identification/evals/ground_truth.json similarity index 100% rename from evals/ground_truth.json rename to labs/lab-01-continuous-risk-identification/evals/ground_truth.json diff --git a/evals/scorecard.json b/labs/lab-01-continuous-risk-identification/evals/scorecard.json similarity index 100% rename from evals/scorecard.json rename to labs/lab-01-continuous-risk-identification/evals/scorecard.json diff --git a/evaluation.py b/labs/lab-01-continuous-risk-identification/evaluation.py similarity index 100% rename from evaluation.py rename to labs/lab-01-continuous-risk-identification/evaluation.py diff --git a/examples/batch-1.json b/labs/lab-01-continuous-risk-identification/examples/batch-1.json similarity index 100% rename from examples/batch-1.json rename to labs/lab-01-continuous-risk-identification/examples/batch-1.json diff --git a/examples/batch-2.json b/labs/lab-01-continuous-risk-identification/examples/batch-2.json similarity index 100% rename from examples/batch-2.json rename to labs/lab-01-continuous-risk-identification/examples/batch-2.json diff --git a/execute_notebook.py b/labs/lab-01-continuous-risk-identification/execute_notebook.py similarity index 100% rename from execute_notebook.py rename to labs/lab-01-continuous-risk-identification/execute_notebook.py diff --git a/exercise.py b/labs/lab-01-continuous-risk-identification/exercise.py similarity index 100% rename from exercise.py rename to labs/lab-01-continuous-risk-identification/exercise.py diff --git a/fixture_factory.py b/labs/lab-01-continuous-risk-identification/fixture_factory.py similarity index 100% rename from fixture_factory.py rename to labs/lab-01-continuous-risk-identification/fixture_factory.py diff --git a/import_exports.py b/labs/lab-01-continuous-risk-identification/import_exports.py similarity index 100% rename from import_exports.py rename to labs/lab-01-continuous-risk-identification/import_exports.py diff --git a/ingestion.py b/labs/lab-01-continuous-risk-identification/ingestion.py similarity index 100% rename from ingestion.py rename to labs/lab-01-continuous-risk-identification/ingestion.py diff --git a/lab-01-continuous-risk-identification.ipynb b/labs/lab-01-continuous-risk-identification/lab-01-continuous-risk-identification.ipynb similarity index 100% rename from lab-01-continuous-risk-identification.ipynb rename to labs/lab-01-continuous-risk-identification/lab-01-continuous-risk-identification.ipynb diff --git a/methodology.py b/labs/lab-01-continuous-risk-identification/methodology.py similarity index 100% rename from methodology.py rename to labs/lab-01-continuous-risk-identification/methodology.py diff --git a/package_lab.py b/labs/lab-01-continuous-risk-identification/package_lab.py similarity index 74% rename from package_lab.py rename to labs/lab-01-continuous-risk-identification/package_lab.py index 663e418..4f7f0f0 100644 --- a/package_lab.py +++ b/labs/lab-01-continuous-risk-identification/package_lab.py @@ -4,10 +4,10 @@ import re root = Path(__file__).resolve().parent -names = ['contracts.py', 'ingestion.py', 'import_exports.py', 'config/source_registry.json', 'schemas/IngestionReceipt.schema.json', 'schemas/RiskEvidenceBundle.schema.json', 'schemas/CandidateRiskScenario.schema.json', 'fixture_factory.py', 'pipeline.py', 'methodology.py', 'exercise.py', 'engine.py', 'evaluation.py', 'evals/ground_truth.json', 'evals/scorecard.json', 'visuals.py', 'test_engine.py', 'build_notebook.py', 'execute_notebook.py', - 'package_lab.py', 'requirements.txt', 'Dockerfile', 'compose.yaml', '.dockerignore', '.gitignore', - 'README.md', 'LICENSE', 'SPEC.md', 'VALIDATION.md', 'SEMANTIC-REVIEW.md', - '.github/workflows/validate.yml', 'lab-01-continuous-risk-identification.ipynb', +names = ['contracts.py', 'ingestion.py', 'import_exports.py', 'config/source_registry.json', 'schemas/IngestionReceipt.schema.json', 'schemas/RiskEvidenceBundle.schema.json', 'schemas/CandidateRiskScenario.schema.json', 'fixture_factory.py', 'pipeline.py', 'methodology.py', 'exercise.py', 'engine.py', 'evaluation.py', 'evals/ground_truth.json', 'evals/scorecard.json', 'visuals.py', 'test_engine.py', 'test_connector.py', 'connectors/README.md', 'connectors/okta_system_log.py', 'build_notebook.py', 'execute_notebook.py', + 'package_lab.py', 'requirements.txt', 'Dockerfile', 'compose.yaml', '.dockerignore', + 'README.md', 'SPEC.md', 'VALIDATION.md', 'SEMANTIC-REVIEW.md', + 'lab-01-continuous-risk-identification.ipynb', 'examples/batch-1.json', 'examples/batch-2.json', 'assets/source-volume.png', 'assets/pipeline-funnel.png', 'assets/agent-investigation.png', 'assets/candidate-risk-register.png', 'assets/evidence-to-statement.png', 'assets/evidence-history.png'] for name in names: @@ -16,7 +16,8 @@ content = path.read_text(encoding='utf-8') if re.search(r'sk-(?:proj-)?[A-Za-z0-9_-]{24,}', content): raise RuntimeError('Potential credential pattern; packaging stopped') -with ZipFile(root.parent / 'risk-engineer-lab-01.zip', 'w', ZIP_DEFLATED) as archive: +with ZipFile(root / 'risk-engineer-lab-01.zip', 'w', ZIP_DEFLATED) as archive: for name in names: archive.write(root / name, 'risk-engineer-lab-01/' + name) + archive.write(root.parent.parent / 'LICENSE', 'risk-engineer-lab-01/LICENSE') print(f'Packaged {len(names)} allowlisted files; no runtime, environment files or credentials included') diff --git a/pipeline.py b/labs/lab-01-continuous-risk-identification/pipeline.py similarity index 100% rename from pipeline.py rename to labs/lab-01-continuous-risk-identification/pipeline.py diff --git a/requirements.txt b/labs/lab-01-continuous-risk-identification/requirements.txt similarity index 100% rename from requirements.txt rename to labs/lab-01-continuous-risk-identification/requirements.txt diff --git a/schemas/CandidateRiskScenario.schema.json b/labs/lab-01-continuous-risk-identification/schemas/CandidateRiskScenario.schema.json similarity index 100% rename from schemas/CandidateRiskScenario.schema.json rename to labs/lab-01-continuous-risk-identification/schemas/CandidateRiskScenario.schema.json diff --git a/schemas/IngestionReceipt.schema.json b/labs/lab-01-continuous-risk-identification/schemas/IngestionReceipt.schema.json similarity index 100% rename from schemas/IngestionReceipt.schema.json rename to labs/lab-01-continuous-risk-identification/schemas/IngestionReceipt.schema.json diff --git a/schemas/RiskEvidenceBundle.schema.json b/labs/lab-01-continuous-risk-identification/schemas/RiskEvidenceBundle.schema.json similarity index 100% rename from schemas/RiskEvidenceBundle.schema.json rename to labs/lab-01-continuous-risk-identification/schemas/RiskEvidenceBundle.schema.json diff --git a/labs/lab-01-continuous-risk-identification/test_connector.py b/labs/lab-01-continuous-risk-identification/test_connector.py new file mode 100644 index 0000000..6edbf3b --- /dev/null +++ b/labs/lab-01-continuous-risk-identification/test_connector.py @@ -0,0 +1,40 @@ +import json +import tempfile +import unittest +from pathlib import Path + +from connectors.okta_system_log import collect + + +class OktaConnectorTests(unittest.TestCase): + def test_collects_pages(self): + calls = [] + + def fake_transport(url, token): + calls.append((url, token)) + if len(calls) == 1: + return ([{"uuid": "one"}], {"Link": '; rel="next"'}) + return ([{"uuid": "two"}], {}) + + with tempfile.TemporaryDirectory() as folder: + output = Path(folder) / "raw" / "okta_system_log.jsonl" + result = collect("https://tenant.okta.com", "private-token", "2026-09-01T00:00:00Z", + "2026-09-02T00:00:00Z", 7, output, transport=fake_transport) + rows = [json.loads(line) for line in output.read_text(encoding="utf-8").splitlines()] + self.assertEqual(result["records"], 2) + self.assertEqual([row["uuid"] for row in rows], ["one", "two"]) + self.assertTrue(all(row["_collection"]["batch_id"] == 7 for row in rows)) + self.assertTrue(all(token == "private-token" for _, token in calls)) + + def test_rejects_cross_origin_pagination(self): + def fake_transport(url, token): + return ([{"uuid": "one"}], {"Link": '; rel="next"'}) + + with tempfile.TemporaryDirectory() as folder: + with self.assertRaisesRegex(ValueError, "different origin"): + collect("https://tenant.okta.com", "private-token", "a", "b", 1, + Path(folder) / "records.jsonl", transport=fake_transport) + + +if __name__ == "__main__": + unittest.main() diff --git a/test_engine.py b/labs/lab-01-continuous-risk-identification/test_engine.py similarity index 100% rename from test_engine.py rename to labs/lab-01-continuous-risk-identification/test_engine.py diff --git a/visuals.py b/labs/lab-01-continuous-risk-identification/visuals.py similarity index 100% rename from visuals.py rename to labs/lab-01-continuous-risk-identification/visuals.py