From 0ddd3ae37ec3f43d7da6fc5b03aa5bbcc187d090 Mon Sep 17 00:00:00 2001 From: Derek Chamorro Date: Sun, 25 Oct 2026 21:50:30 -0500 Subject: [PATCH] P7b-1: match customer threat-intel lists and rank matched findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Predictive security roadmap P7b, first slice (IOC matching) under the agreed defaults: AWS-native + customer-supplied lists, DynamoDB store, ranking factor, curated mapping and graph-only emulation deferred to P7b-2. - ThreatIndicators store (risk-engine): one row per indicator with a TTL — freshness is the TTL, so a dropped indicator expires by itself — batch writes and batched, bounded reads; list parsing (one indicator per line, '#' comments, IP v4/v6 and 12-digit accounts, dedupe, cap with counted invalid lines); and pathIndicators(), the fact join the runner uses (anomaly source_ip, trusted ExternalAccount account_id). - threat-intel-import (runtime command; CronJob suspended by default, rule-runner service account): imports plain S3 lists into the store. No internet: the lists are account-owned objects, and AWS's managed reputation lists need no import because GuardDuty applies them (their findings already feed RULE-018/RULE-029). - Ranking: a match adds a flat +10 threatIntelContribution to the risk score (additive like exploitability — a fact about the path, not a proportional strength), the issue records threatIntelMatches and the issue page shows a "Threat intelligence" card; refresh clears matches whose indicator is gone; lookup failures score without it and log. No auto-ticketing until /rules/quality shows measured precision. - Infra: Flash (Terraform) creates the ThreatIndicators table (TTL, deletion protection, CMK), grants the rule-runner role full access to it and s3:GetObject on threat_intel_list_bucket when set; the URIs, table name, and TTL flow through helm_values.config; chart gains threatIntelImport.*, config envs, schema entries, and the threat-intel-import egress component. Why this is safe: read-only for the engine (the importer only writes its own indicators table); no new outbound network path; matches are transparent (metadata + card) and cannot ticket; invalid/absent data degrades to "no match". --- ARCHITECTURE.md | 18 ++ CHANGELOG.md | 11 ++ OPERATIONAL.md | 30 ++- README.md | 10 +- deploy/helm/khalifa/templates/configmap.yaml | 4 + .../khalifa/templates/networkpolicies.yaml | 2 +- .../templates/threat-intel-import.yaml | 70 +++++++ deploy/helm/khalifa/values.schema.json | 11 ++ deploy/helm/khalifa/values.yaml | 28 +++ deploy/terraform/khalifa-data/dynamodb.tf | 31 +++ deploy/terraform/khalifa-data/iam.tf | 12 ++ deploy/terraform/khalifa-data/outputs.tf | 48 ++--- deploy/terraform/khalifa-data/variables.tf | 29 +++ deploy/testdata/eks-deployment-values.yaml | 5 + jest.config.js | 8 + lambdas/runtime/index.test.ts | 42 ++++ lambdas/runtime/index.ts | 27 +++ lambdas/threat-intel-import/index.test.ts | 88 +++++++++ lambdas/threat-intel-import/index.ts | 133 +++++++++++++ lambdas/threat-intel-import/jest.config.js | 9 + lambdas/threat-intel-import/package.json | 36 ++++ lambdas/threat-intel-import/tsconfig.json | 20 ++ package-lock.json | 37 ++++ packages/risk-engine/src/index.ts | 11 ++ packages/risk-engine/src/runner.test.ts | 88 +++++++++ packages/risk-engine/src/runner.ts | 98 ++++++++-- packages/risk-engine/src/scoring.test.ts | 25 +++ packages/risk-engine/src/scoring.ts | 14 +- .../src/threat-intel.store.test.ts | 69 +++++++ packages/risk-engine/src/threat-intel.test.ts | 86 ++++++++ packages/risk-engine/src/threat-intel.ts | 183 ++++++++++++++++++ packages/risk-engine/src/types.test.ts | 1 + packages/risk-engine/src/types.ts | 3 + ui/app/issues/[id]/page.test.tsx | 23 +++ ui/app/issues/[id]/page.tsx | 37 ++++ 35 files changed, 1299 insertions(+), 48 deletions(-) create mode 100644 deploy/helm/khalifa/templates/threat-intel-import.yaml create mode 100644 lambdas/threat-intel-import/index.test.ts create mode 100644 lambdas/threat-intel-import/index.ts create mode 100644 lambdas/threat-intel-import/jest.config.js create mode 100644 lambdas/threat-intel-import/package.json create mode 100644 lambdas/threat-intel-import/tsconfig.json create mode 100644 packages/risk-engine/src/threat-intel.store.test.ts create mode 100644 packages/risk-engine/src/threat-intel.test.ts create mode 100644 packages/risk-engine/src/threat-intel.ts diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 20735d02..6a887835 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -245,6 +245,24 @@ beside the configured ones. Everything is cost-gated: Terraform creates nothing without `flow_logs_bucket`, the CronJob ships suspended, and a deployment without the table simply has no observed traffic. +## Threat-intelligence matching + +Customer-supplied intelligence arrives as plain S3 lists (one indicator per +line): `threat-intel-import` validates them (IPs v4/v6, 12-digit account ids), +caps each list, and writes `ThreatIndicators` with a TTL — freshness is the +TTL, so an indicator dropped from a list drains itself. Nothing here calls the +internet: AWS's own managed reputation lists need no import because GuardDuty +applies them and the findings they drive already feed RULE-018/RULE-029. The +rule runner joins the store against the facts rules already match: an anomaly's +`source_ip` and a trusted `ExternalAccount`'s account id. A match adds a flat +**+10 threat-intelligence factor** to the risk score (a fact about the path, +not a proportional strength; recorded as `threatIntelMatches` on the issue and +labelled on the score card) and matches clear on refresh once the indicator is +gone. Matched issues do **not** auto-ticket until `/rules/quality` shows the +measured precision P6 requires; the local +`scripts/risk-plan` gate is unaffected (it has no store to read, so it never +claims a match). + ## Compliance reconciliation and the trust center The compliance job evaluates the CIS AWS Foundations v3.0.0 and Khalifa diff --git a/CHANGELOG.md b/CHANGELOG.md index f9c9f4a5..5f8a2698 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to Khalifa. +## [Unreleased] - Threat-intelligence matching (P7b-1) + +### Added + +- **Customer threat-intel matching**: `threat-intel-import` (runtime command; CronJob **suspended by default**, running as the rule-runner service account) imports plain S3 lists — one indicator per line, `#` comments, IPs (v4/v6) and 12-digit account ids — into a `ThreatIndicators` table with a TTL (`threat_intel_ttl_days`, default 90): freshness is the TTL, so a dropped indicator expires by itself, and invalid lines are skipped and counted in the log. Nothing calls the internet — the lists live in an account-owned bucket, and AWS's own managed reputation lists need no import because GuardDuty applies them (the findings they drive already feed RULE-018/RULE-029). Terraform creates the table, grants the rule-runner role read/write on it, and `s3:GetObject` on `threat_intel_list_bucket` when set; the list URIs, table name, and TTL flow through the chart config. +- **Matches change ranking**: the rule runner joins the store against the facts rules already match — an anomaly's `source_ip` and a trusted `ExternalAccount`'s account id — and a match adds a flat **+10 `threatIntelContribution`** to the risk score (additive like exploitability; a fact about the path, not a proportional strength). The issue records `threatIntelMatches` (type, value, source) and the issue page renders a "Threat intelligence" card; a refresh clears matches whose indicator is gone. **No auto-ticketing** for matched issues until `/rules/quality` shows the measured precision P6 requires, and the offline `risk:plan` gate is unaffected (no store to read, so it never claims a match). The lookup is failure-tolerant: a missing table or grant scores without it and logs a warning. + +### Notes + +- P7b-2 (TTP mapping to MITRE ATT&CK and graph-only adversary-emulation scenarios) remains open — it depends on the curated mapping table this phase deliberately left out; and the importer does not fetch external feeds (commercial/open IoC subscriptions would add the deployment's first outbound-internet component and licensing constraints on storing indicators). + ## [Unreleased] - Observed reachability (P7a) ### Added diff --git a/OPERATIONAL.md b/OPERATIONAL.md index 35fc89ae..15ae73ed 100644 --- a/OPERATIONAL.md +++ b/OPERATIONAL.md @@ -13,9 +13,10 @@ 7. **compliance** - Compliance CronJob 8. **cloudtrail-analyzer** - CloudTrail usage CronJob (Athena → AccessAnalyzerCache; optional anomaly nodes via graph-writer) 9. **flow-log-analyzer** - VPC flow-log CronJob (Athena → TrafficObservations; **suspended by default**, reuses the cloudtrail-analyzer IRSA role) -10. **dspm-scanner** - CronJob: secret scanning of S3 objects and Lambda environment variables (Macie primary, regex fallback; findings written via graph-writer) -11. **policy-evaluator / least-privilege-sweep** - Lambdas: effective permissions, escalation paths, rightsizing -12. **IRSA roles** - IAM roles for pod authentication to Neptune/DynamoDB/SQS +10. **threat-intel-import** - CronJob: imports customer threat-intel lists from S3 into ThreatIndicators (**suspended by default**, runs as the rule-runner service account) +11. **dspm-scanner** - CronJob: secret scanning of S3 objects and Lambda environment variables (Macie primary, regex fallback; findings written via graph-writer) +12. **policy-evaluator / least-privilege-sweep** - Lambdas: effective permissions, escalation paths, rightsizing +13. **IRSA roles** - IAM roles for pod authentication to Neptune/DynamoDB/SQS ### Network Flow @@ -201,6 +202,29 @@ kubectl create job --from=cronjob/khalifa-rule-runner rule-runner-manual -n secu the collector stops seeing fresh rows (a stale row still marks an instance until it expires — acceptable, traffic was real). +### Runbook: Importing customer threat-intel lists + +1. Put plain lists in an account-owned S3 bucket: one indicator per line, + `#` comments and blanks ignored. IPs (v4/v6) in one object, 12-digit + account ids in another. +2. In `khalifa-data`: set `threat_intel_list_bucket` (grants the rule-runner + role `s3:GetObject` on it) and `threat_intel_ip_list_s3_uri` / + `threat_intel_account_list_s3_uri` (flow into the chart's config), plus + `threat_intel_ttl_days` (default 90). Apply. +3. Set `threatIntelImport.image` (the rule-runner digest) and unsuspend or + run it once: + `kubectl create job --from=cronjob/khalifa-threat-intel-import threat-intel-manual -n security-graph`. + An unreadable list fails the job (exit non-zero) so the failure is visible; + invalid lines are skipped and counted in the log. +4. Verify: `ThreatIndicators` gains rows with a TTL; the next rule run scores + matching anomaly IPs / trusted external accounts with the +10 + threat-intelligence factor and records the match on the issue. +5. Freshness is the TTL: an indicator dropped from the list expires by itself + up to `threat_intel_ttl_days` after it was last imported. AWS's own managed + reputation lists need no import (GuardDuty applies them; RULE-018/RULE-029 + already consume the findings). Auto-ticketing stays off for matched issues + until `/rules/quality` shows measured precision. + ## I. Graph Query Profiling Neptune serves id lookups from the index; a property filter on a whole label diff --git a/README.md b/README.md index 249ac9d6..ca9d83e7 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ One member role. Evaluation stays in-account. | **CIEM** | IAM decomposition, effective permissions, escalation paths (assume chains and `iam:PassRole` to compute, RULE-034), CloudTrail-grounded rightsizing. | | **Least privilege** | Sweep of roles and users. Deployable policy recommendations. | | **Secrets** | Secrets Manager and SSM parameter inventory with who-can-read analysis from identity and resource policies, rotation and access age, sharing detection (RULE-010, RULE-032, RULE-033), and admin users holding active access keys (RULE-031). | -| **Risk** | Scheduled rules and attack-path traversals on the live graph (34 rules), with an exploitability factor that raises scores for paths carrying Amazon Inspector findings, an explainable likelihood on every path, a per-fix simulation that counts what each edge removal would clear, and a per-rule quality report (suppression vs resolution) that calibrates which rules to trust. | +| **Risk** | Scheduled rules and attack-path traversals on the live graph (34 rules), with an exploitability factor that raises scores for paths carrying Amazon Inspector findings, an explainable likelihood on every path, a per-fix simulation that counts what each edge removal would clear, matching against customer-supplied threat-intel lists (additive score bonus, TTL-fresh), and a per-rule quality report (suppression vs resolution) that calibrates which rules to trust. | | **Detection** | GuardDuty detector coverage per region (including `MISSING`), Security Hub findings joined to internet-exposed resources (RULE-017, RULE-018), CloudTrail anomaly signals: root activity, secret reads from IPs outside each principal's own 30-day baseline, AccessDenied deviations from each principal's own daily baseline (RULE-020 to RULE-022), canary credential tripwires (RULE-023), secret scanning of Lambda environment variables plus IMDSv1-with-role detection (RULE-024), and Amazon Inspector CVEs on internet-exposed workloads (RULE-003). | | **Exposure** | Public EKS control planes (RULE-019) plus public S3/RDS and world-open security groups, derived from collected endpoint and network config, with optional observed traffic from VPC flow logs confirming where traffic actually goes. | | **Compliance** | CIS AWS Foundations v3.0 and the Khalifa baseline, per-control evidence, mapped SOC 2 / ISO 27001 / PCI DSS / NIST 800-53 requirements, and CIS reconciliation against Security Hub before anything is publishable. | @@ -158,7 +158,11 @@ EKS uses [`deploy/helm/khalifa/values.yaml`](deploy/helm/khalifa/values.yaml). T | `NEPTUNE_ENDPOINT` (auth) | Graph access uses SigV4 from the execution role; no Neptune auth secret is read | — | | `ISSUES_TABLE` | DynamoDB issues table | `SecurityIssues` | | `ACCESS_ANALYZER_TABLE` | CloudTrail usage cache | `AccessAnalyzerCache` | -| `TRAFFIC_OBSERVATIONS_TABLE` | Observed-traffic cache for the EC2 collector (amber: empty disables the read) | `TrafficObservations` | +| `TRAFFIC_OBSERVATIONS_TABLE` | Observed-traffic cache for the EC2 collector (empty disables the read) | `TrafficObservations` | +| `THREAT_INDICATORS_TABLE` | Threat-intel indicators read by the rule runner | `ThreatIndicators` | +| `THREAT_INTEL_IP_LIST_S3_URI` | s3://bucket/key of the newline-separated IP indicator list (empty disables) | — | +| `THREAT_INTEL_ACCOUNT_LIST_S3_URI`| s3://bucket/key of the newline-separated account-id list (empty disables) | — | +| `THREAT_INTEL_TTL_DAYS` | Days an imported indicator stays valid (DynamoDB TTL) | `90` | | `ATHENA_DATABASE` | Glue database for CloudTrail | `khalifa_cloudtrail_db` | | `ATHENA_WORKGROUP` | Athena workgroup | `khalifa-cloudtrail-analysis` | | `CLOUDTRAIL_S3_LOCATION` | CloudTrail S3 prefix | `s3://cloudtrail-logs/AWSLogs/` | @@ -272,7 +276,7 @@ reported as certified. | Path | Contents | | ---------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | -| [`lambdas/`](lambdas) | Collector, incremental, graph-writer, policy-evaluator, CloudTrail, flow-log, least-privilege, DSPM, runtime | +| [`lambdas/`](lambdas) | Collector, incremental, graph-writer, policy-evaluator, CloudTrail, flow-log, threat-intel import, least-privilege, DSPM, runtime | | [`packages/risk-engine`](packages/risk-engine) | Rules, scoring, compliance | | [`api-service/`](api-service) · [`ui/`](ui) | REST API and Next.js UI | | [`deploy/`](deploy) | Helm chart, Terraform modules, fixtures | diff --git a/deploy/helm/khalifa/templates/configmap.yaml b/deploy/helm/khalifa/templates/configmap.yaml index 39a5a212..b51a8355 100644 --- a/deploy/helm/khalifa/templates/configmap.yaml +++ b/deploy/helm/khalifa/templates/configmap.yaml @@ -16,6 +16,10 @@ data: POSTURE_TRENDS_TABLE: {{ .Values.config.postureTrendsTable | quote }} ACCESS_ANALYZER_TABLE: {{ .Values.config.accessAnalyzerTable | quote }} TRAFFIC_OBSERVATIONS_TABLE: {{ .Values.config.trafficObservationsTable | quote }} + THREAT_INDICATORS_TABLE: {{ .Values.config.threatIndicatorsTable | quote }} + THREAT_INTEL_IP_LIST_S3_URI: {{ .Values.config.threatIntelIpListS3Uri | quote }} + THREAT_INTEL_ACCOUNT_LIST_S3_URI: {{ .Values.config.threatIntelAccountListS3Uri | quote }} + THREAT_INTEL_TTL_DAYS: {{ .Values.config.threatIntelTtlDays | quote }} LEAST_PRIVILEGE_TABLE: {{ .Values.config.leastPrivilegeTable | quote }} COGNITO_USER_POOL_ID: {{ .Values.config.cognitoUserPoolId | quote }} COGNITO_CLIENT_ID: {{ .Values.config.cognitoClientId | quote }} diff --git a/deploy/helm/khalifa/templates/networkpolicies.yaml b/deploy/helm/khalifa/templates/networkpolicies.yaml index eb98b093..8efcc44d 100644 --- a/deploy/helm/khalifa/templates/networkpolicies.yaml +++ b/deploy/helm/khalifa/templates/networkpolicies.yaml @@ -1,5 +1,5 @@ {{- if .Values.networkPolicies.enabled }} -{{- range $component := list "api" "ui" "rule-runner" "org-collector" "incremental" "graph-writer" "compliance" "cloudtrail-analyzer" "flow-log-analyzer" "dspm-scanner" "policy-evaluator" "least-privilege-sweep" }} +{{- range $component := list "api" "ui" "rule-runner" "org-collector" "incremental" "graph-writer" "compliance" "cloudtrail-analyzer" "flow-log-analyzer" "threat-intel-import" "dspm-scanner" "policy-evaluator" "least-privilege-sweep" }} apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: diff --git a/deploy/helm/khalifa/templates/threat-intel-import.yaml b/deploy/helm/khalifa/templates/threat-intel-import.yaml new file mode 100644 index 00000000..6c1ffc8f --- /dev/null +++ b/deploy/helm/khalifa/templates/threat-intel-import.yaml @@ -0,0 +1,70 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "khalifa.fullname" . }}-threat-intel-import + labels: + {{- include "khalifa.labels" . | nindent 4 }} + app.kubernetes.io/component: threat-intel-import +spec: + schedule: {{ .Values.threatIntelImport.schedule | quote }} + {{- with .Values.threatIntelImport.timeZone }} + timeZone: {{ . | quote }} + {{- end }} + suspend: {{ .Values.threatIntelImport.suspend }} + concurrencyPolicy: {{ .Values.threatIntelImport.concurrencyPolicy }} + successfulJobsHistoryLimit: {{ .Values.threatIntelImport.successfulJobsHistoryLimit }} + failedJobsHistoryLimit: {{ .Values.threatIntelImport.failedJobsHistoryLimit }} + startingDeadlineSeconds: {{ .Values.threatIntelImport.startingDeadlineSeconds }} + jobTemplate: + metadata: + labels: + {{- include "khalifa.selectorLabels" (dict "root" . "component" "threat-intel-import") | nindent 8 }} + spec: + activeDeadlineSeconds: {{ .Values.threatIntelImport.activeDeadlineSeconds }} + ttlSecondsAfterFinished: {{ .Values.threatIntelImport.ttlSecondsAfterFinished }} + backoffLimit: {{ .Values.threatIntelImport.backoffLimit }} + template: + metadata: + labels: + {{- include "khalifa.selectorLabels" (dict "root" . "component" "threat-intel-import") | nindent 12 }} + {{- with .Values.threatIntelImport.podLabels }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.threatIntelImport.podAnnotations }} + annotations: + {{- toYaml . | nindent 12 }} + {{- end }} + spec: + # The rule-runner service account owns the ThreatIndicators table: + # the runner reads it and this import writes it. + serviceAccountName: {{ include "khalifa.serviceAccountName" (dict "root" . "key" "ruleRunner" "suffix" "rule-runner") }} + automountServiceAccountToken: {{ .Values.serviceAccounts.ruleRunner.automountServiceAccountToken }} + restartPolicy: Never + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + {{- include "khalifa.podSecurityContext" . | nindent 12 }} + {{- if .Values.topologySpread.enabled }} + topologySpreadConstraints: + {{- include "khalifa.topologySpreadConstraints" (dict "root" . "component" "threat-intel-import") | nindent 12 }} + {{- end }} + containers: + - name: threat-intel-import + image: {{ required "threatIntelImport.image must be an immutable repository@sha256 digest" .Values.threatIntelImport.image | quote }} + imagePullPolicy: IfNotPresent + args: ["threat-intel-import"] + securityContext: + {{- include "khalifa.containerSecurityContext" . | nindent 16 }} + envFrom: + - configMapRef: + name: {{ include "khalifa.fullname" . }}-config + resources: + {{- toYaml .Values.threatIntelImport.resources | nindent 16 }} + volumeMounts: + - name: tmp + mountPath: /tmp + volumes: + - name: tmp + emptyDir: {} diff --git a/deploy/helm/khalifa/values.schema.json b/deploy/helm/khalifa/values.schema.json index 294aa3c5..c45f426b 100644 --- a/deploy/helm/khalifa/values.schema.json +++ b/deploy/helm/khalifa/values.schema.json @@ -16,6 +16,7 @@ "graphWriter", "compliance", "reconciliation", + "threatIntelImport", "flowLogAnalyzer", "cloudtrailAnalyzer", "dspmScanner", @@ -86,6 +87,10 @@ "postureTrendsTable", "accessAnalyzerTable", "trafficObservationsTable", + "threatIndicatorsTable", + "threatIntelIpListS3Uri", + "threatIntelAccountListS3Uri", + "threatIntelTtlDays", "leastPrivilegeTable", "cognitoUserPoolId", "cognitoClientId", @@ -110,6 +115,10 @@ "postureTrendsTable": { "type": "string", "minLength": 1 }, "accessAnalyzerTable": { "type": "string", "minLength": 1 }, "trafficObservationsTable": { "type": "string" }, + "threatIndicatorsTable": { "type": "string" }, + "threatIntelIpListS3Uri": { "type": "string", "pattern": "^(|s3://.+/.+)$" }, + "threatIntelAccountListS3Uri": { "type": "string", "pattern": "^(|s3://.+/.+)$" }, + "threatIntelTtlDays": { "type": "integer", "minimum": 1, "maximum": 365 }, "leastPrivilegeTable": { "type": "string", "minLength": 1 }, "cognitoUserPoolId": { "type": "string", @@ -189,6 +198,7 @@ "trustCenterSyncClientId": { "type": "string", "pattern": "^(|[0-9a-f]{32}\\.access)$" } } }, + "threatIntelImport": { "$ref": "#/definitions/cronJob" }, "flowLogAnalyzer": { "$ref": "#/definitions/cronJob" }, "cloudtrailAnalyzer": { "$ref": "#/definitions/cronJob" }, "dspmScanner": { "$ref": "#/definitions/cronJob" }, @@ -796,6 +806,7 @@ "compliance", "cloudtrail-analyzer", "flow-log-analyzer", + "threat-intel-import", "dspm-scanner", "policy-evaluator", "least-privilege-sweep" diff --git a/deploy/helm/khalifa/values.yaml b/deploy/helm/khalifa/values.yaml index 12d4b807..4782c965 100644 --- a/deploy/helm/khalifa/values.yaml +++ b/deploy/helm/khalifa/values.yaml @@ -81,6 +81,10 @@ config: postureTrendsTable: '' accessAnalyzerTable: '' trafficObservationsTable: '' + threatIndicatorsTable: '' + threatIntelIpListS3Uri: '' + threatIntelAccountListS3Uri: '' + threatIntelTtlDays: 90 leastPrivilegeTable: '' cognitoUserPoolId: '' cognitoClientId: '' @@ -306,6 +310,30 @@ compliance: cpu: 1000m memory: 1Gi +threatIntelImport: + image: '' + schedule: '0 5 * * *' + timeZone: '' + # Imports customer-supplied lists; nothing external is contacted. Runs as the + # rule-runner service account, which owns the ThreatIndicators table. + suspend: true + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + startingDeadlineSeconds: 600 + activeDeadlineSeconds: 900 + ttlSecondsAfterFinished: 86400 + backoffLimit: 1 + podAnnotations: {} + podLabels: {} + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi + flowLogAnalyzer: image: '' days: 7 diff --git a/deploy/terraform/khalifa-data/dynamodb.tf b/deploy/terraform/khalifa-data/dynamodb.tf index deff40db..5cab3d8f 100644 --- a/deploy/terraform/khalifa-data/dynamodb.tf +++ b/deploy/terraform/khalifa-data/dynamodb.tf @@ -284,3 +284,34 @@ resource "aws_dynamodb_table" "traffic_observations" { tags = merge(var.tags, { Name = "${local.name}-traffic-observations" }) } + +# Customer threat-intel indicators for the rule runner's matching and the +# threat-intel-import CronJob's writes; TTL is what makes an indicator stale. +resource "aws_dynamodb_table" "threat_indicators" { + name = "${local.name}-threat-indicators" + billing_mode = "PAY_PER_REQUEST" + hash_key = "indicator" + + attribute { + name = "indicator" + type = "S" + } + + ttl { + attribute_name = "expires_at" + enabled = true + } + + deletion_protection_enabled = true + + point_in_time_recovery { + enabled = true + } + + server_side_encryption { + enabled = true + kms_key_arn = aws_kms_key.this.arn + } + + tags = merge(var.tags, { Name = "${local.name}-threat-indicators" }) +} diff --git a/deploy/terraform/khalifa-data/iam.tf b/deploy/terraform/khalifa-data/iam.tf index 366900ae..ff016104 100644 --- a/deploy/terraform/khalifa-data/iam.tf +++ b/deploy/terraform/khalifa-data/iam.tf @@ -252,6 +252,12 @@ resource "aws_iam_role_policy" "api" { Action = local.dynamodb_read_actions Resource = concat(local.compliance_table_arns, [for arn in local.compliance_table_arns : "${arn}/index/*"]) }, + { + Sid = "ThreatIndicators" + Effect = "Allow" + Action = local.dynamodb_full_actions + Resource = [aws_dynamodb_table.threat_indicators.arn] + }, { Sid = "IssuesTopic" Effect = "Allow" @@ -301,6 +307,12 @@ resource "aws_iam_role_policy" "rule_runner" { ] Resource = [local.neptune_resource_arn] }, + { + Sid = "ThreatIntelLists" + Effect = "Allow" + Action = ["s3:GetObject"] + Resource = var.threat_intel_list_bucket != "" ? ["arn:aws:s3:::${var.threat_intel_list_bucket}/*"] : [] + }, ] }) } diff --git a/deploy/terraform/khalifa-data/outputs.tf b/deploy/terraform/khalifa-data/outputs.tf index 68153c04..33f65566 100644 --- a/deploy/terraform/khalifa-data/outputs.tf +++ b/deploy/terraform/khalifa-data/outputs.tf @@ -87,28 +87,32 @@ output "helm_values" { description = "Non-secret chart values derived from this module. Secrets stay in Secrets Manager." value = { config = { - awsRegion = data.aws_region.current.region - neptuneEndpoint = "wss://${aws_neptune_cluster.this.endpoint}:8182/gremlin" - issuesTable = aws_dynamodb_table.issues.name - evidenceTable = aws_dynamodb_table.evidence.name - reportsTable = aws_dynamodb_table.reports.name - postureTrendsTable = aws_dynamodb_table.posture_trends.name - accessAnalyzerTable = aws_dynamodb_table.access_analyzer.name - trafficObservationsTable = aws_dynamodb_table.traffic_observations.name - leastPrivilegeTable = aws_dynamodb_table.least_privilege.name - cognitoUserPoolId = aws_cognito_user_pool.this.id - cognitoClientId = aws_cognito_user_pool_client.this.id - cognitoRegion = data.aws_region.current.region - corsAllowedOrigins = [var.ui_base_url] - issueSinks = ["sns"] - issuesTopicArn = aws_sns_topic.issues.arn - uiBaseUrl = var.ui_base_url - athenaDatabase = local.cloudtrail_enabled ? aws_glue_catalog_database.cloudtrail[0].name : "" - athenaWorkgroup = local.cloudtrail_enabled ? aws_athena_workgroup.cloudtrail[0].name : "" - athenaResultsLocation = local.athena_results_location - analysisDays = var.analysis_days - dspmScanMode = var.dspm_scan_mode - macieEnabled = var.macie_enabled + awsRegion = data.aws_region.current.region + neptuneEndpoint = "wss://${aws_neptune_cluster.this.endpoint}:8182/gremlin" + issuesTable = aws_dynamodb_table.issues.name + evidenceTable = aws_dynamodb_table.evidence.name + reportsTable = aws_dynamodb_table.reports.name + postureTrendsTable = aws_dynamodb_table.posture_trends.name + accessAnalyzerTable = aws_dynamodb_table.access_analyzer.name + trafficObservationsTable = aws_dynamodb_table.traffic_observations.name + threatIndicatorsTable = aws_dynamodb_table.threat_indicators.name + threatIntelIpListS3Uri = var.threat_intel_ip_list_s3_uri + threatIntelAccountListS3Uri = var.threat_intel_account_list_s3_uri + threatIntelTtlDays = var.threat_intel_ttl_days + leastPrivilegeTable = aws_dynamodb_table.least_privilege.name + cognitoUserPoolId = aws_cognito_user_pool.this.id + cognitoClientId = aws_cognito_user_pool_client.this.id + cognitoRegion = data.aws_region.current.region + corsAllowedOrigins = [var.ui_base_url] + issueSinks = ["sns"] + issuesTopicArn = aws_sns_topic.issues.arn + uiBaseUrl = var.ui_base_url + athenaDatabase = local.cloudtrail_enabled ? aws_glue_catalog_database.cloudtrail[0].name : "" + athenaWorkgroup = local.cloudtrail_enabled ? aws_athena_workgroup.cloudtrail[0].name : "" + athenaResultsLocation = local.athena_results_location + analysisDays = var.analysis_days + dspmScanMode = var.dspm_scan_mode + macieEnabled = var.macie_enabled } collection = { homeAccountId = var.home_account_id diff --git a/deploy/terraform/khalifa-data/variables.tf b/deploy/terraform/khalifa-data/variables.tf index 4d5439ac..4ae2baef 100644 --- a/deploy/terraform/khalifa-data/variables.tf +++ b/deploy/terraform/khalifa-data/variables.tf @@ -188,6 +188,35 @@ variable "cloudtrail_logs_kms_key_arn" { default = "" } +variable "threat_intel_list_bucket" { + description = "S3 bucket holding the customer's threat-intel lists. Empty leaves the importer without a bucket grant; the URIs still configure the CronJob." + type = string + default = "" +} + +variable "threat_intel_ip_list_s3_uri" { + description = "s3://bucket/key of the newline-separated IP indicator list. Empty disables that list." + type = string + default = "" +} + +variable "threat_intel_account_list_s3_uri" { + description = "s3://bucket/key of the newline-separated account-id indicator list. Empty disables that list." + type = string + default = "" +} + +variable "threat_intel_ttl_days" { + description = "Days an imported indicator stays valid before DynamoDB TTL drops it." + type = number + default = 90 + + validation { + condition = var.threat_intel_ttl_days >= 1 && var.threat_intel_ttl_days <= 365 + error_message = "threat_intel_ttl_days must be between 1 and 365." + } +} + variable "flow_logs_bucket" { description = "S3 bucket that receives VPC flow logs. Empty leaves the flow-log analyzer unconfigured. Requires cloudtrail_logs_bucket too: the analyzer reuses the CloudTrail Athena database and workgroup." type = string diff --git a/deploy/testdata/eks-deployment-values.yaml b/deploy/testdata/eks-deployment-values.yaml index f92806bd..f588d57c 100644 --- a/deploy/testdata/eks-deployment-values.yaml +++ b/deploy/testdata/eks-deployment-values.yaml @@ -128,6 +128,11 @@ flowLogAnalyzer: schedule: '0 4 * * *' timeZone: America/Los_Angeles +threatIntelImport: + image: 111122223333.dkr.ecr.us-west-2.amazonaws.com/khalifa-runtime@sha256:4444444444444444444444444444444444444444444444444444444444444444 + schedule: '0 5 * * *' + timeZone: America/Los_Angeles + dspmScanner: image: 111122223333.dkr.ecr.us-west-2.amazonaws.com/khalifa-runtime@sha256:4444444444444444444444444444444444444444444444444444444444444444 schedule: '0 4 * * *' diff --git a/jest.config.js b/jest.config.js index 9d7a44b6..4ba1fc17 100644 --- a/jest.config.js +++ b/jest.config.js @@ -63,6 +63,14 @@ module.exports = { testMatch: ['/src/**/*.test.ts'], modulePathIgnorePatterns: ['/dist/', '/node_modules/'], }, + { + displayName: 'threat-intel-import', + rootDir: '/lambdas/threat-intel-import', + preset: 'ts-jest', + testEnvironment: 'node', + testMatch: ['/**/*.test.ts'], + modulePathIgnorePatterns: ['/dist/', '/node_modules/'], + }, { displayName: 'flow-log-analyzer', rootDir: '/lambdas/flow-log-analyzer', diff --git a/lambdas/runtime/index.test.ts b/lambdas/runtime/index.test.ts index 9675928b..19e38daa 100644 --- a/lambdas/runtime/index.test.ts +++ b/lambdas/runtime/index.test.ts @@ -28,6 +28,12 @@ const analyzerEnvironment = { ACCESS_ANALYZER_TABLE: 'AccessAnalyzerCache', }; +const threatIntelEnvironment = { + AWS_REGION: 'us-west-2', + THREAT_INDICATORS_TABLE: 'khalifa-threat-indicators', + THREAT_INTEL_IP_LIST_S3_URI: ['s3', 'lists.example.com/threat/ips.txt'].join('://'), +}; + const flowLogEnvironment = { AWS_REGION: 'us-west-2', ATHENA_DATABASE: 'cloudtrail_database', @@ -79,6 +85,25 @@ describe('Kubernetes runtime', () => { }) ).toThrow('GRAPH_WRITER_TOKEN is required'); expect(() => validateRuntime('flow-log-analyzer', flowLogEnvironment)).not.toThrow(); + expect(() => validateRuntime('threat-intel-import', threatIntelEnvironment)).not.toThrow(); + expect(() => + validateRuntime('threat-intel-import', { + AWS_REGION: 'us-west-2', + THREAT_INDICATORS_TABLE: 't', + }) + ).toThrow('THREAT_INTEL_IP_LIST_S3_URI or THREAT_INTEL_ACCOUNT_LIST_S3_URI is required'); + expect(() => + validateRuntime('threat-intel-import', { + ...threatIntelEnvironment, + THREAT_INTEL_IP_LIST_S3_URI: 'https://example.com/ips.txt', + }) + ).toThrow('must be s3://bucket/key'); + expect(() => + validateRuntime('threat-intel-import', { + ...threatIntelEnvironment, + THREAT_INTEL_TTL_DAYS: '400', + }) + ).toThrow('THREAT_INTEL_TTL_DAYS must be an integer between 1 and 365'); expect(() => validateRuntime('flow-log-analyzer', { AWS_REGION: 'us-west-2', @@ -165,6 +190,23 @@ describe('Kubernetes runtime', () => { ).rejects.toThrow('Athena query FAILED'); }); + it('runs the threat-intel importer once and exits nonzero on failure', async () => { + const importFn = jest.fn().mockResolvedValue({ written: 2, lists: [] }); + await expect( + runCommand('threat-intel-import', threatIntelEnvironment, new AbortController().signal, { + importThreatIntel: importFn, + }) + ).resolves.toBeUndefined(); + expect(importFn).toHaveBeenCalledWith({}, {}, threatIntelEnvironment); + + const failing = jest.fn().mockRejectedValue(new Error('NoSuchKey')); + await expect( + runCommand('threat-intel-import', threatIntelEnvironment, new AbortController().signal, { + importThreatIntel: failing, + }) + ).rejects.toThrow('NoSuchKey'); + }); + it('runs the DSPM scanner once with the HTTP graph writer', async () => { const runDspm = jest.fn().mockResolvedValue({ bucketsScanned: 2, diff --git a/lambdas/runtime/index.ts b/lambdas/runtime/index.ts index aebd4a12..a60f8bc8 100644 --- a/lambdas/runtime/index.ts +++ b/lambdas/runtime/index.ts @@ -28,6 +28,7 @@ import { normalizeAthenaResultsLocation as normalizeFlowLogResultsLocation, } from '../flow-log-analyzer'; import { validateObservationDays } from '../flow-log-analyzer/src/queries'; +import { importThreatIntel, parseS3Uri, validateTtlDays } from '../threat-intel-import'; export { createHttpGraphWriter } from '../shared/graph-writer-client'; @@ -39,6 +40,7 @@ export type RuntimeCommand = | 'graph-writer' | 'cloudtrail-analyzer' | 'flow-log-analyzer' + | 'threat-intel-import' | 'dspm-scanner' | 'policy-evaluator' | 'least-privilege-sweep'; @@ -52,6 +54,7 @@ interface RuntimeDependencies { runRisk?: typeof runOnce; analyzeCloudTrail?: typeof analyzeCloudTrail; analyzeFlowLogs?: typeof analyzeFlowLogs; + importThreatIntel?: typeof importThreatIntel; runDspm?: typeof runDspmScan; runPolicyEvaluation?: typeof runPolicyEvaluation; runLeastPrivilegeSweep?: typeof runLeastPrivilegeSweep; @@ -69,6 +72,7 @@ const commands = new Set([ 'graph-writer', 'cloudtrail-analyzer', 'flow-log-analyzer', + 'threat-intel-import', 'dspm-scanner', 'policy-evaluator', 'least-privilege-sweep', @@ -84,6 +88,7 @@ Commands: risk Run all enabled risk rules once cloudtrail-analyzer Query org CloudTrail usage into AccessAnalyzerCache flow-log-analyzer Aggregate VPC flow logs into TrafficObservations + threat-intel-import Import customer threat-intel lists into ThreatIndicators dspm-scanner Scan S3 objects and Lambda env vars for secrets policy-evaluator Resolve effective permissions and escalation paths once least-privilege-sweep Rightsize over-permissioned principals once`; @@ -209,6 +214,21 @@ export function validateRuntime(command: RuntimeCommand, environment: NodeJS.Pro validateObservationDays(environment.FLOW_LOG_DAYS ?? '7', 'FLOW_LOG_DAYS'); return; } + if (command === 'threat-intel-import') { + required(environment, 'AWS_REGION'); + required(environment, 'THREAT_INDICATORS_TABLE'); + const ipList = environment.THREAT_INTEL_IP_LIST_S3_URI?.trim(); + const accountList = environment.THREAT_INTEL_ACCOUNT_LIST_S3_URI?.trim(); + if (!ipList && !accountList) { + throw new Error( + 'THREAT_INTEL_IP_LIST_S3_URI or THREAT_INTEL_ACCOUNT_LIST_S3_URI is required' + ); + } + if (ipList) parseS3Uri(ipList); + if (accountList) parseS3Uri(accountList); + validateTtlDays(environment.THREAT_INTEL_TTL_DAYS ?? '90'); + return; + } if (command === 'cloudtrail-analyzer') { required(environment, 'AWS_REGION'); required(environment, 'ATHENA_DATABASE'); @@ -653,6 +673,13 @@ export async function runCommand( environment ); logger.info('Flow-log analysis complete', summary); + } else if (command === 'threat-intel-import') { + const summary = await (dependencies.importThreatIntel || importThreatIntel)( + {}, + {}, + environment + ); + logger.info('Threat-intel import complete', summary); } else if (command === 'dspm-scanner') { const summary = await (dependencies.runDspm || runDspmScan)({}, {}, environment); logger.info('DSPM scan complete', summary); diff --git a/lambdas/threat-intel-import/index.test.ts b/lambdas/threat-intel-import/index.test.ts new file mode 100644 index 00000000..968727fc --- /dev/null +++ b/lambdas/threat-intel-import/index.test.ts @@ -0,0 +1,88 @@ +import { importThreatIntel, parseS3Uri, validateTtlDays } from './index'; + +const environment = { + THREAT_INDICATORS_TABLE: 'ThreatIndicators', + THREAT_INTEL_IP_LIST_S3_URI: 's3://lists.example.com/threat/ips.txt', + THREAT_INTEL_ACCOUNT_LIST_S3_URI: 's3://lists.example.com/threat/accounts.txt', +}; + +function s3Returning(texts: Record): any { + return { + send: jest.fn(async (command: { input: { Bucket: string; Key: string } }) => { + const key = `${command.input.Bucket}/${command.input.Key}`; + const text = texts[key]; + if (text === undefined) throw new Error(`NoSuchKey: ${key}`); + return { Body: { transformToString: async () => text } }; + }), + }; +} + +describe('validateTtlDays / parseS3Uri', () => { + it('validates the TTL bounds', () => { + expect(validateTtlDays('90')).toBe(90); + expect(() => validateTtlDays('0')).toThrow( + 'THREAT_INTEL_TTL_DAYS must be an integer between 1 and 365' + ); + expect(() => validateTtlDays('400')).toThrow('between 1 and 365'); + }); + + it('parses s3 URIs and rejects anything else', () => { + expect(parseS3Uri('s3://bucket/path/ips.txt')).toEqual({ + bucket: 'bucket', + key: 'path/ips.txt', + }); + expect(() => parseS3Uri('https://example.com/ips.txt')).toThrow('must be s3://bucket/key'); + expect(() => parseS3Uri('s3://bucket')).toThrow('must be s3://bucket/key'); + }); +}); + +describe('importThreatIntel', () => { + it('imports each configured list with its own type and records the summary', async () => { + const s3Client = s3Returning({ + 'lists.example.com/threat/ips.txt': ['# watchlist', '203.0.113.7', 'nope'].join('\n'), + 'lists.example.com/threat/accounts.txt': '222233334444\n', + }); + const put = jest.fn().mockResolvedValue(2); + const warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined); + + const result = await importThreatIntel( + {}, + { + s3Client, + store: { put } as never, + now: () => new Date('2026-10-26T00:00:00Z'), + }, + environment + ); + + warn.mockRestore(); + expect(result.written).toBe(4); + expect(result.lists).toEqual([ + expect.objectContaining({ type: 'ip', indicators: 1, invalid: 1, truncated: false }), + expect.objectContaining({ type: 'account', indicators: 1, invalid: 0 }), + ]); + expect(put).toHaveBeenNthCalledWith( + 1, + [{ type: 'ip', value: '203.0.113.7', source: 's3://lists.example.com/threat/ips.txt' }], + 90, + expect.any(Date) + ); + }); + + it('requires at least one list', async () => { + await expect(importThreatIntel({}, {}, {})).rejects.toThrow( + 'THREAT_INTEL_IP_LIST_S3_URI or THREAT_INTEL_ACCOUNT_LIST_S3_URI is required' + ); + }); + + it('fails when a list object is unreadable so the job is retried', async () => { + const s3Client = s3Returning({}); + await expect( + importThreatIntel( + {}, + { s3Client, store: { put: jest.fn() } as never }, + { ...environment, THREAT_INTEL_ACCOUNT_LIST_S3_URI: '' } + ) + ).rejects.toThrow('NoSuchKey'); + }); +}); diff --git a/lambdas/threat-intel-import/index.ts b/lambdas/threat-intel-import/index.ts new file mode 100644 index 00000000..c11c27c7 --- /dev/null +++ b/lambdas/threat-intel-import/index.ts @@ -0,0 +1,133 @@ +import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { Logger } from '../shared/types'; +import { + parseIndicatorList, + THREAT_INDICATOR_DEFAULT_TTL_DAYS, + ThreatIndicatorStore, + type ThreatIndicatorType, +} from '@khalifa/risk-engine'; + +// Customer-supplied threat intelligence, imported from plain S3 lists (one +// indicator per line, '#' comments). AWS's own managed reputation lists need +// no import: GuardDuty applies them and the findings they drive already feed +// RULE-018/RULE-029. Nothing here calls the internet — the list lives in an +// account-owned bucket, so the deployment keeps its AWS-API-only posture. +const logger = new Logger('threat-intel-import'); +const defaultS3Client = new S3Client({ region: process.env.AWS_REGION || 'us-east-1' }); + +interface ImportEvent { + ttlDays?: number; +} + +interface ImportConfig { + tableName: string; + ttlDays: number; + lists: Array<{ type: ThreatIndicatorType; uri: string }>; +} + +interface ImportDependencies { + s3Client: S3Client; + store: ThreatIndicatorStore; + now: () => Date; +} + +export interface ThreatIntelImportResult { + written: number; + lists: Array<{ + type: ThreatIndicatorType; + uri: string; + indicators: number; + invalid: number; + truncated: boolean; + }>; +} + +export function validateTtlDays(value: unknown, name = 'THREAT_INTEL_TTL_DAYS'): number { + const parsed = + typeof value === 'number' + ? value + : typeof value === 'string' && /^\d+$/.test(value.trim()) + ? Number(value) + : Number.NaN; + if (!Number.isSafeInteger(parsed) || parsed < 1 || parsed > 365) { + throw new Error(`${name} must be an integer between 1 and 365`); + } + return parsed; +} + +// s3://bucket/key; the key may be empty only for listing, which this importer +// never does, so both parts are required. +export function parseS3Uri(uri: string): { bucket: string; key: string } { + const match = /^s3:\/\/([a-z0-9][a-z0-9.-]{1,61}[a-z0-9])\/(.+)$/.exec(uri.trim()); + if (!match) { + throw new Error(`Threat-intel list URI must be s3://bucket/key: ${uri}`); + } + return { bucket: match[1], key: match[2] }; +} + +function loadConfig(environment: NodeJS.ProcessEnv): ImportConfig { + const lists: ImportConfig['lists'] = []; + for (const [type, name] of [ + ['ip', 'THREAT_INTEL_IP_LIST_S3_URI'], + ['account', 'THREAT_INTEL_ACCOUNT_LIST_S3_URI'], + ] as const) { + const uri = environment[name]?.trim(); + if (uri) lists.push({ type, uri }); + } + if (lists.length === 0) { + throw new Error('THREAT_INTEL_IP_LIST_S3_URI or THREAT_INTEL_ACCOUNT_LIST_S3_URI is required'); + } + return { + tableName: environment.THREAT_INDICATORS_TABLE || 'ThreatIndicators', + ttlDays: validateTtlDays( + environment.THREAT_INTEL_TTL_DAYS ?? THREAT_INDICATOR_DEFAULT_TTL_DAYS + ), + lists, + }; +} + +export async function importThreatIntel( + event: ImportEvent = {}, + dependencyOverrides: Partial = {}, + environment: NodeJS.ProcessEnv = process.env +): Promise { + const config = loadConfig(environment); + const dependencies: ImportDependencies = { + s3Client: defaultS3Client, + store: new ThreatIndicatorStore(config.tableName), + now: () => new Date(), + ...dependencyOverrides, + }; + const ttlDays = validateTtlDays(event.ttlDays ?? config.ttlDays); + + const result: ThreatIntelImportResult = { written: 0, lists: [] }; + for (const list of config.lists) { + const { bucket, key } = parseS3Uri(list.uri); + const response = await dependencies.s3Client.send( + new GetObjectCommand({ Bucket: bucket, Key: key }) + ); + const text = (await response.Body?.transformToString()) ?? ''; + const parsed = parseIndicatorList(text, list.type, list.uri); + if (parsed.invalid > 0 || parsed.truncated) { + logger.warn('Threat-intel list had invalid lines or was truncated', { + uri: list.uri, + invalid: parsed.invalid, + truncated: parsed.truncated, + }); + } + const written = await dependencies.store.put(parsed.indicators, ttlDays, dependencies.now()); + result.written += written; + result.lists.push({ + type: list.type, + uri: list.uri, + indicators: parsed.indicators.length, + invalid: parsed.invalid, + truncated: parsed.truncated, + }); + logger.info(`Imported ${written} ${list.type} indicators from ${list.uri}`); + } + return result; +} + +export const handler = async (event: ImportEvent = {}): Promise => + importThreatIntel(event); diff --git a/lambdas/threat-intel-import/jest.config.js b/lambdas/threat-intel-import/jest.config.js new file mode 100644 index 00000000..cc747815 --- /dev/null +++ b/lambdas/threat-intel-import/jest.config.js @@ -0,0 +1,9 @@ +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + roots: [''], + testMatch: ['**/*.test.ts'], + moduleFileExtensions: ['ts', 'js', 'json'], + collectCoverageFrom: ['*.ts', '!*.test.ts', '!index.ts'], + coverageDirectory: 'coverage', +}; diff --git a/lambdas/threat-intel-import/package.json b/lambdas/threat-intel-import/package.json new file mode 100644 index 00000000..b3e7a06c --- /dev/null +++ b/lambdas/threat-intel-import/package.json @@ -0,0 +1,36 @@ +{ + "name": "threat-intel-import", + "version": "1.0.0", + "private": true, + "description": "Imports customer threat-intel lists from S3 into ThreatIndicators", + "main": "dist/index.js", + "scripts": { + "build": "esbuild index.ts --bundle --platform=node --format=cjs --target=node24 --outfile=dist/index.js --sourcemap --sources-content=false", + "typecheck": "tsc --noEmit", + "lint": "eslint . --ext .ts", + "format": "prettier --write \"**/*.ts\"", + "format:check": "prettier --check \"**/*.ts\"", + "test": "jest --passWithNoTests", + "test:watch": "jest --watch" + }, + "dependencies": { + "@aws-sdk/client-dynamodb": "^3.400.0", + "@aws-sdk/client-s3": "^3.400.0", + "@aws-sdk/util-dynamodb": "^3.996.2", + "@khalifa/risk-engine": "*" + }, + "devDependencies": { + "@types/aws-lambda": "^8.10.131", + "@types/jest": "^29.5.0", + "@types/node": "^20.0.0", + "jest": "^29.5.0", + "ts-jest": "^29.1.0", + "typescript": "^5.0.0", + "eslint": "^8.56.0", + "prettier": "^3.2.5", + "@typescript-eslint/eslint-plugin": "^7.0.0", + "@typescript-eslint/parser": "^7.0.0", + "eslint-config-prettier": "^9.1.0", + "eslint-plugin-prettier": "^5.1.3" + } +} diff --git a/lambdas/threat-intel-import/tsconfig.json b/lambdas/threat-intel-import/tsconfig.json new file mode 100644 index 00000000..7cd56647 --- /dev/null +++ b/lambdas/threat-intel-import/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": ["ES2020"], + "outDir": "./dist", + "rootDir": "..", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "moduleResolution": "node" + }, + "include": ["../threat-intel-import/*.ts", "../shared/**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/package-lock.json b/package-lock.json index 171ca92b..8ed03fb4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -455,6 +455,39 @@ "@aws-sdk/client-secrets-manager": "^3.400.0" } }, + "lambdas/threat-intel-import": { + "version": "1.0.0", + "dependencies": { + "@aws-sdk/client-dynamodb": "^3.400.0", + "@aws-sdk/client-s3": "^3.400.0", + "@aws-sdk/util-dynamodb": "^3.996.2", + "@khalifa/risk-engine": "*" + }, + "devDependencies": { + "@types/aws-lambda": "^8.10.131", + "@types/jest": "^29.5.0", + "@types/node": "^20.0.0", + "@typescript-eslint/eslint-plugin": "^7.0.0", + "@typescript-eslint/parser": "^7.0.0", + "eslint": "^8.56.0", + "eslint-config-prettier": "^9.1.0", + "eslint-plugin-prettier": "^5.1.3", + "jest": "^29.5.0", + "prettier": "^3.2.5", + "ts-jest": "^29.1.0", + "typescript": "^5.0.0" + } + }, + "lambdas/threat-intel-import/node_modules/@types/node": { + "version": "20.19.43", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.43.tgz", + "integrity": "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, "node_modules/@adobe/css-tools": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/@adobe/css-tools/-/css-tools-4.5.0.tgz", @@ -13107,6 +13140,10 @@ "node": ">=0.8" } }, + "node_modules/threat-intel-import": { + "resolved": "lambdas/threat-intel-import", + "link": true + }, "node_modules/tinyglobby": { "version": "0.2.17", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", diff --git a/packages/risk-engine/src/index.ts b/packages/risk-engine/src/index.ts index 50b6a1c6..ace9e61c 100644 --- a/packages/risk-engine/src/index.ts +++ b/packages/risk-engine/src/index.ts @@ -52,6 +52,17 @@ export { globMatch } from './glob'; export { extractRulePath, type RulePath } from './rule-path'; export { pathLikelihood, type PathLikelihood, type PathLikelihoodFactors } from './likelihood'; export { forecastSeries, type Forecast, type ForecastProjection } from './forecast'; +export { + indicatorKey, + isValidIndicator, + parseIndicatorList, + pathIndicators, + ThreatIndicatorStore, + THREAT_INDICATOR_DEFAULT_TTL_DAYS, + THREAT_INDICATOR_MAX_LINES, + type ThreatIndicator, + type ThreatIndicatorType, +} from './threat-intel'; export { buildRuleQuality, RULE_QUALITY_MIN_HANDLED, diff --git a/packages/risk-engine/src/runner.test.ts b/packages/risk-engine/src/runner.test.ts index 4d62dcd3..a14c2047 100644 --- a/packages/risk-engine/src/runner.test.ts +++ b/packages/risk-engine/src/runner.test.ts @@ -184,6 +184,94 @@ describe('RiskRuleRunner stale-issue fix', () => { expect(neptune.executeQuery).toHaveBeenCalledTimes(1); }); + test('runRule marks customer threat-intel matches and scores them', async () => { + const { Path } = Gremlin.structure; + const { t } = Gremlin.process; + const element = (id: string, label: string, props: Record = {}) => + new Map([[t.id, id], [t.label, label], ...Object.entries(props)]); + const path = new Path( + ['anomaly', 'account'], + [ + element('anomaly-1', 'CloudTrailAnomaly', { + arn: ['anomaly-1'], + anomaly_type: ['SECRET_READ_NEW_SOURCE_IP'], + source_ip: ['203.0.113.7'], + }), + ] + ); + const threatStore = { + find: jest + .fn() + .mockResolvedValue([{ type: 'ip', value: '203.0.113.7', source: 's3://lists/ips.txt' }]), + }; + const runner = new RiskRuleRunner( + 'localhost:8182', + undefined, + undefined, + undefined, + threatStore as never + ); + const store = (runner as any).issueStore; + const neptune = (runner as any).neptuneClient; + neptune.executeQuery = jest.fn().mockResolvedValue([path]); + store.getIssueByRuleAndResources = jest.fn().mockResolvedValue(null); + store.getActiveIssuesByRule = jest.fn().mockResolvedValue([]); + let created: Issue | undefined; + store.createIssue = jest.fn().mockImplementation(async (issue: Issue) => { + created = issue; + return true; + }); + + await expect(runner.runRule(rule)).resolves.toEqual( + expect.objectContaining({ issuesCreated: 1, errors: undefined }) + ); + + expect(threatStore.find).toHaveBeenCalledWith([{ type: 'ip', value: '203.0.113.7' }]); + expect(created?.metadata.threatIntelMatches).toEqual([ + { type: 'ip', value: '203.0.113.7', source: 's3://lists/ips.txt' }, + ]); + expect(created?.metadata.scoringFactors).toMatchObject({ threatIntelContribution: 10 }); + }); + + test('runRule keeps running when the threat-intel store is unreadable', async () => { + const { Path } = Gremlin.structure; + const { t } = Gremlin.process; + const element = (id: string, label: string) => + new Map([ + [t.id, id], + [t.label, label], + ]); + const path = new Path(['anomaly'], [element('anomaly-1', 'CloudTrailAnomaly')]); + const threatStore = { + find: jest.fn().mockRejectedValue(new Error('ResourceNotFoundException')), + }; + const runner = new RiskRuleRunner( + 'localhost:8182', + undefined, + undefined, + undefined, + threatStore as never + ); + const store = (runner as any).issueStore; + const neptune = (runner as any).neptuneClient; + neptune.executeQuery = jest.fn().mockResolvedValue([path]); + store.getIssueByRuleAndResources = jest.fn().mockResolvedValue(null); + store.getActiveIssuesByRule = jest.fn().mockResolvedValue([]); + let created: Issue | undefined; + store.createIssue = jest.fn().mockImplementation(async (issue: Issue) => { + created = issue; + return true; + }); + const warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined); + + await expect(runner.runRule(rule)).resolves.toEqual( + expect.objectContaining({ issuesCreated: 1, errors: undefined }) + ); + + warn.mockRestore(); + expect(created?.metadata.threatIntelMatches).toBeUndefined(); + expect(created?.metadata.scoringFactors).toMatchObject({ threatIntelContribution: 0 }); + }); test('runRule reads driver Path elements (Maps keyed by T enums, list values, edges between vertices)', async () => { const { Path } = Gremlin.structure; const { t } = Gremlin.process; diff --git a/packages/risk-engine/src/runner.ts b/packages/risk-engine/src/runner.ts index e4f0689e..8b97b9f9 100644 --- a/packages/risk-engine/src/runner.ts +++ b/packages/risk-engine/src/runner.ts @@ -16,6 +16,7 @@ import type { } from './types'; import { extractRulePath, type RulePath } from './rule-path'; import { pathLikelihood } from './likelihood'; +import { pathIndicators, ThreatIndicatorStore, type ThreatIndicator } from './threat-intel'; // Neptune Serverless rejects every path query with MemoryLimitExceededException // for a minute or two after a burst of heavy ones (measured 2026-09-19: the same @@ -42,7 +43,8 @@ export class RiskRuleRunner { neptuneEndpoint ), private readonly sleep: (ms: number) => Promise = (ms) => - new Promise((resolve) => setTimeout(resolve, ms)) + new Promise((resolve) => setTimeout(resolve, ms)), + private readonly threatStore: ThreatIndicatorStore = new ThreatIndicatorStore() ) {} async initialize(): Promise { @@ -74,6 +76,26 @@ export class RiskRuleRunner { throw new Error('Neptune returned a malformed risk-rule path'); } + // Customer threat-intel matches for anything this rule's paths touch + // (anomaly source IPs, trusted external accounts). Failure-tolerant: a + // missing table or grant means no matches, never a failed run. + let threatIntel: ThreatIndicator[] = []; + try { + threatIntel = await this.threatStore.find(paths.flatMap((path) => pathIndicators(path))); + } catch (error) { + console.warn( + `${rule.id}: threat-intel lookup failed; scoring without it (${error instanceof Error ? error.message : String(error)})` + ); + } + const matchesFor = (path: RulePath): ThreatIndicator[] => { + const candidates = pathIndicators(path); + return threatIntel.filter((indicator) => + candidates.some( + (candidate) => candidate.type === indicator.type && candidate.value === indicator.value + ) + ); + }; + const activeIssueIds = new Set(); for (const { identity, resources, path, matchCount } of this.groupPaths(rule, paths)) { @@ -94,14 +116,22 @@ export class RiskRuleRunner { rule, path, resources, - matchCount + matchCount, + matchesFor(path) ); if (refreshed.status === 'suppressed') { await this.endSuppressionIfDue(refreshed, existingIssue.resourcesInvolved); } activeIssueIds.add(existingIssue.id); } else { - const issue = this.createIssueFromMatch(rule, path, resources, identity, matchCount); + const issue = this.createIssueFromMatch( + rule, + path, + resources, + identity, + matchCount, + matchesFor(path) + ); const created = await this.issueStore.createIssue(issue); if (created) { issuesCreated++; @@ -335,10 +365,11 @@ export class RiskRuleRunner { path: RulePath, resources: { resourceId: string; resourceType: string; resourceName?: string }[], identity: string[], - matchCount: number + matchCount: number, + threatIntel: ThreatIndicator[] = [] ): Issue { const now = new Date().toISOString(); - const riskInput = this.buildRiskInput(path.vertices); + const riskInput = this.riskInputWithThreatIntel(path, threatIntel); const scoreResult = computeRiskScore(riskInput); const likelihood = pathLikelihood(riskInput); @@ -362,6 +393,19 @@ export class RiskRuleRunner { ruleDescription: rule.description, environment: riskInput.environment, matchCount, + ...(threatIntel.length > 0 ? { threatIntelMatches: threatIntel } : {}), + }, + }; + } + + // The path's risk input plus customer threat-intel matches, when any. + private riskInputWithThreatIntel(path: RulePath, threatIntel: ThreatIndicator[]): RiskScoreInput { + const riskInput = this.buildRiskInput(path.vertices); + if (threatIntel.length === 0) return riskInput; + return { + ...riskInput, + threatIntel: { + indicators: threatIntel.map((indicator) => `${indicator.type}:${indicator.value}`), }, }; } @@ -375,9 +419,10 @@ export class RiskRuleRunner { rule: RiskRule, path: RulePath, resources: Issue['resourcesInvolved'], - matchCount: number + matchCount: number, + threatIntel: ThreatIndicator[] = [] ): Promise { - const riskInput = this.buildRiskInput(path.vertices); + const riskInput = this.riskInputWithThreatIntel(path, threatIntel); const scoreResult = computeRiskScore(riskInput); const likelihood = pathLikelihood(riskInput); const updatedAt = new Date().toISOString(); @@ -390,17 +435,20 @@ export class RiskRuleRunner { severity: cappedSeverity(rule, scoreResult.severity), remediationHint: getRemediationHint(rule.id, {}), updatedAt, - metadata: { - ...existing.metadata, - scoringFactors: scoreResult.factors, - likelihood: likelihood.value, - likelihoodFactors: likelihood.factors, - ruleName: rule.name, - ruleDescription: rule.description, - environment: riskInput.environment, - matchCount, - lastRefreshedAt: updatedAt, - }, + metadata: metadataWithThreatIntel( + { + ...existing.metadata, + scoringFactors: scoreResult.factors, + likelihood: likelihood.value, + likelihoodFactors: likelihood.factors, + ruleName: rule.name, + ruleDescription: rule.description, + environment: riskInput.environment, + matchCount, + lastRefreshedAt: updatedAt, + }, + threatIntel + ), }; const updated = await this.issueStore.updateIssue(refreshed); @@ -478,3 +526,17 @@ export class RiskRuleRunner { })); } } + +// A refresh clears yesterday's matches when the indicator is gone: metadata +// spread alone would keep a stale match forever. +function metadataWithThreatIntel( + metadata: Record, + threatIntel: ThreatIndicator[] +): Record { + if (threatIntel.length > 0) { + metadata.threatIntelMatches = threatIntel; + } else { + delete metadata.threatIntelMatches; + } + return metadata; +} diff --git a/packages/risk-engine/src/scoring.test.ts b/packages/risk-engine/src/scoring.test.ts index dae6fc61..5b0adbed 100644 --- a/packages/risk-engine/src/scoring.test.ts +++ b/packages/risk-engine/src/scoring.test.ts @@ -289,3 +289,28 @@ describe('riskInputFromPath observed reachability', () => { expect(input.reachabilityObserved).toBeUndefined(); }); }); + +describe('threat-intel factor', () => { + const base: RiskScoreInput = { + exposureLevel: 'internet', + dataClassification: 'public', + environment: 'dev', + isCrownJewel: false, + }; + + test('adds the flat bonus when the path carries a matched indicator', () => { + const clean = computeRiskScore(base); + const matched = computeRiskScore({ + ...base, + threatIntel: { indicators: ['ip:203.0.113.7'] }, + }); + expect(clean.factors.threatIntelContribution).toBe(0); + expect(matched.factors.threatIntelContribution).toBe(10); + expect(matched.score).toBe(clean.score + 10); + }); + + test('adds nothing for an empty indicator list', () => { + const result = computeRiskScore({ ...base, threatIntel: { indicators: [] } }); + expect(result.factors.threatIntelContribution).toBe(0); + }); +}); diff --git a/packages/risk-engine/src/scoring.ts b/packages/risk-engine/src/scoring.ts index 09a6536d..622a1ebf 100644 --- a/packages/risk-engine/src/scoring.ts +++ b/packages/risk-engine/src/scoring.ts @@ -53,6 +53,14 @@ const FACTOR_WEIGHTS = { // enough to order a vulnerable workload above an otherwise identical one. const EXPLOITABILITY_WEIGHT = 0.15; +// A customer threat-intel match (an anomaly's source IP or a trusted external +// account on the customer's list) is additive too, and flat rather than +// proportional: the indicator is a fact about the path, not a strength. Ten +// points ranks a matched path above the same path without the match without +// crossing a severity threshold on its own; auto-ticketing still waits on +// measured precision (rule-quality.ts). +const THREAT_INTEL_BONUS = 10; + const SEVERITY_THRESHOLDS: Record = { critical: 80, high: 60, @@ -86,6 +94,8 @@ export function computeRiskScore(input: RiskScoreInput): RiskScoreOutput { crownJewelContribution += pathPenalty; } + const threatIntelContribution = input.threatIntel?.indicators?.length ? THREAT_INTEL_BONUS : 0; + let exploitabilityContribution = 0; if (input.exploitability) { const cvss = input.exploitability.cvssBaseScore; @@ -102,7 +112,8 @@ export function computeRiskScore(input: RiskScoreInput): RiskScoreOutput { dataClassificationContribution + environmentContribution + crownJewelContribution + - exploitabilityContribution + exploitabilityContribution + + threatIntelContribution ); const severity = mapScoreToSeverity(totalScore); @@ -117,6 +128,7 @@ export function computeRiskScore(input: RiskScoreInput): RiskScoreOutput { environmentContribution: Math.round(environmentContribution * 10) / 10, crownJewelContribution: Math.round(crownJewelContribution * 10) / 10, exploitabilityContribution: Math.round(exploitabilityContribution * 10) / 10, + threatIntelContribution, }, }; } diff --git a/packages/risk-engine/src/threat-intel.store.test.ts b/packages/risk-engine/src/threat-intel.store.test.ts new file mode 100644 index 00000000..8f9c8b68 --- /dev/null +++ b/packages/risk-engine/src/threat-intel.store.test.ts @@ -0,0 +1,69 @@ +import { ThreatIndicatorStore } from './threat-intel'; + +function clientReturning(...responses: unknown[]): any { + const send = jest.fn(); + for (const response of responses) send.mockResolvedValueOnce(response); + return { send }; +} + +describe('ThreatIndicatorStore', () => { + it('writes indicators with a TTL from the import time', async () => { + const client = clientReturning({}); + const store = new ThreatIndicatorStore('ThreatIndicators', client); + const now = new Date('2026-10-26T00:00:00Z'); + + const written = await store.put( + [{ type: 'ip', value: '203.0.113.7', source: 's3://lists/ips.txt' }], + 90, + now + ); + + expect(written).toBe(1); + const item = + client.send.mock.calls[0][0].input.RequestItems.ThreatIndicators[0].PutRequest.Item; + expect(item.indicator).toEqual({ S: 'ip#203.0.113.7' }); + expect(item.expires_at).toEqual({ + N: String(Math.floor(now.getTime() / 1000) + 90 * 86_400), + }); + }); + + it('fails when DynamoDB leaves items unprocessed', async () => { + const client = clientReturning({ UnprocessedItems: { ThreatIndicators: [{}] } }); + const store = new ThreatIndicatorStore('ThreatIndicators', client); + await expect( + store.put([{ type: 'account', value: '222233334444', source: 'x' }]) + ).rejects.toThrow('re-run the import'); + }); + + it('finds stored indicators among the candidates, bounded and deduped', async () => { + const client = clientReturning({ + Responses: { + ThreatIndicators: [ + { + indicator: { S: 'ip#203.0.113.7' }, + type: { S: 'ip' }, + value: { S: '203.0.113.7' }, + source: { S: 'list' }, + }, + ], + }, + }); + const store = new ThreatIndicatorStore('ThreatIndicators', client); + + const found = await store.find([ + { type: 'ip', value: '203.0.113.7' }, + { type: 'ip', value: '203.0.113.7' }, + { type: 'account', value: '222233334444' }, + ]); + + expect(found).toEqual([{ type: 'ip', value: '203.0.113.7', source: 'list' }]); + expect(client.send.mock.calls[0][0].input.RequestItems.ThreatIndicators.Keys).toHaveLength(2); + }); + + it('returns nothing for no candidates without calling DynamoDB', async () => { + const client = clientReturning({}); + const store = new ThreatIndicatorStore('ThreatIndicators', client); + await expect(store.find([])).resolves.toEqual([]); + expect(client.send).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/risk-engine/src/threat-intel.test.ts b/packages/risk-engine/src/threat-intel.test.ts new file mode 100644 index 00000000..1490d02e --- /dev/null +++ b/packages/risk-engine/src/threat-intel.test.ts @@ -0,0 +1,86 @@ +import { + indicatorKey, + isValidIndicator, + parseIndicatorList, + pathIndicators, + THREAT_INDICATOR_MAX_LINES, +} from './threat-intel'; + +describe('isValidIndicator', () => { + it('accepts IPs of either family and 12-digit accounts', () => { + expect(isValidIndicator('ip', '203.0.113.7')).toBe(true); + expect(isValidIndicator('ip', '2001:db8::1')).toBe(true); + expect(isValidIndicator('account', '111122223333')).toBe(true); + expect(isValidIndicator('ip', 'not-an-ip')).toBe(false); + expect(isValidIndicator('ip', '203.0.113.7/24')).toBe(false); + expect(isValidIndicator('account', '123')).toBe(false); + }); +}); + +describe('parseIndicatorList', () => { + it('parses lines, skips comments and blanks, dedupes, and counts invalid', () => { + const parsed = parseIndicatorList( + [ + '# our watchlist', + '203.0.113.7', + '', + '203.0.113.7', + ' 198.51.100.9 ', + '198.51.100.9 # trailing comment', + 'not-an-ip', + ].join('\n'), + 'ip', + 's3://lists/ips.txt' + ); + + expect(parsed.invalid).toBe(1); + expect(parsed.truncated).toBe(false); + expect(parsed.indicators).toEqual([ + { type: 'ip', value: '203.0.113.7', source: 's3://lists/ips.txt' }, + { type: 'ip', value: '198.51.100.9', source: 's3://lists/ips.txt' }, + ]); + }); + + it('caps the list and reports the truncation', () => { + const lines = Array.from({ length: THREAT_INDICATOR_MAX_LINES + 10 }, (_, index) => { + const b = index % 256; + const c = Math.floor(index / 256) % 256; + return `10.${c}.${b}.1`; + }); + const parsed = parseIndicatorList(lines.join('\n'), 'ip', 's3://lists/ips.txt'); + expect(parsed.indicators).toHaveLength(THREAT_INDICATOR_MAX_LINES); + expect(parsed.truncated).toBe(true); + }); +}); + +describe('pathIndicators', () => { + it('reads anomaly source IPs and trusted external accounts', () => { + const candidates = pathIndicators({ + vertices: [ + { label: 'Internet', properties: {} }, + { label: 'CloudTrailAnomaly', properties: { source_ip: '203.0.113.7' } }, + { label: 'ExternalAccount', properties: { account_id: '222233334444' } }, + { label: 'IamRole', properties: { arn: 'arn:aws:iam::111122223333:role/x' } }, + ], + }); + expect(candidates).toEqual([ + { type: 'ip', value: '203.0.113.7' }, + { type: 'account', value: '222233334444' }, + ]); + }); + + it('ignores malformed or unrelated values', () => { + expect( + pathIndicators({ + vertices: [ + { label: 'CloudTrailAnomaly', properties: { source_ip: '-' } }, + { label: 'ExternalAccount', properties: { account_id: 'unknown' } }, + ], + }) + ).toEqual([]); + }); + + it('keys are stable', () => { + expect(indicatorKey('ip', '203.0.113.7')).toBe('ip#203.0.113.7'); + }); +}); diff --git a/packages/risk-engine/src/threat-intel.ts b/packages/risk-engine/src/threat-intel.ts new file mode 100644 index 00000000..1d060b77 --- /dev/null +++ b/packages/risk-engine/src/threat-intel.ts @@ -0,0 +1,183 @@ +import { + BatchGetItemCommand, + BatchWriteItemCommand, + DynamoDBClient, +} from '@aws-sdk/client-dynamodb'; +import { marshall } from '@aws-sdk/util-dynamodb'; +import type { AttributeValue } from '@aws-sdk/client-dynamodb'; +import { isIP } from 'node:net'; + +// Threat-intel indicators a customer supplies as plain lists (one indicator +// per line, '#' comments). AWS-native intelligence needs no import: GuardDuty +// applies AWS's managed reputation lists itself and the findings they drive +// already feed RULE-018/RULE-029 — this store adds the customer's own view +// ("is this IP or account on our list?") so matches can change ranking. +export type ThreatIndicatorType = 'ip' | 'account'; + +export interface ThreatIndicator { + type: ThreatIndicatorType; + value: string; + /** Where the indicator came from, for the record (e.g. the list URI). */ + source: string; +} + +export const THREAT_INDICATOR_MAX_LINES = 50_000; +export const THREAT_INDICATOR_DEFAULT_TTL_DAYS = 90; +const DAY_SECONDS = 86_400; +const MAX_LOOKUPS = 500; +const CHUNK_SIZE = 100; + +export function indicatorKey(type: ThreatIndicatorType, value: string): string { + return `${type}#${value}`; +} + +export function isValidIndicator(type: ThreatIndicatorType, value: string): boolean { + if (type === 'ip') return isIP(value) !== 0; + return /^\d{12}$/.test(value); +} + +export interface ParsedIndicatorList { + indicators: ThreatIndicator[]; + /** Lines that were not valid for the type; skipped, not fatal. */ + invalid: number; + truncated: boolean; +} + +// Exact-value matching only: one indicator per line, whitespace trimmed, +// '#'-comments and blanks skipped, case-sensitive (IPs and account ids have +// no case). Duplicates collapse. The cap bounds the store and the log says +// when it bit. +export function parseIndicatorList( + text: string, + type: ThreatIndicatorType, + source: string +): ParsedIndicatorList { + const seen = new Set(); + const indicators: ThreatIndicator[] = []; + let invalid = 0; + let truncated = false; + for (const rawLine of text.split('\n')) { + const line = rawLine.split('#')[0].trim(); + if (!line) continue; + if (!isValidIndicator(type, line)) { + invalid += 1; + continue; + } + const key = indicatorKey(type, line); + if (seen.has(key)) continue; + seen.add(key); + if (indicators.length >= THREAT_INDICATOR_MAX_LINES) { + truncated = true; + break; + } + indicators.push({ type, value: line, source }); + } + return { indicators, invalid, truncated }; +} + +// Graph facts the matcher joins on: an anomaly's source IP and a trusted +// external account. Kept here (not in the runner) so the join is testable and +// shared with anything else that reads the store. +export function pathIndicators(path: { + vertices: Array<{ label: string; properties: Record }>; +}): Array<{ type: ThreatIndicatorType; value: string }> { + const candidates = new Map(); + for (const vertex of path.vertices) { + if (vertex.label === 'CloudTrailAnomaly') { + const ip = vertex.properties.source_ip; + if (typeof ip === 'string' && isValidIndicator('ip', ip)) { + candidates.set(indicatorKey('ip', ip), { type: 'ip', value: ip }); + } + } + if (vertex.label === 'ExternalAccount') { + const account = vertex.properties.account_id; + if (typeof account === 'string' && isValidIndicator('account', account)) { + candidates.set(indicatorKey('account', account), { type: 'account', value: account }); + } + } + } + return [...candidates.values()]; +} + +export class ThreatIndicatorStore { + private client: DynamoDBClient; + + constructor( + private readonly tableName: string = process.env.THREAT_INDICATORS_TABLE || 'ThreatIndicators', + client?: DynamoDBClient + ) { + this.client = client ?? new DynamoDBClient({ region: process.env.AWS_REGION || 'us-east-1' }); + } + + // One row per indicator, overwritten on every import; the TTL is what makes + // an indicator stale (a list that stops including it drains itself). + async put( + indicators: ThreatIndicator[], + ttlDays: number = THREAT_INDICATOR_DEFAULT_TTL_DAYS, + now: Date = new Date() + ): Promise { + const expiresAt = Math.floor(now.getTime() / 1000) + ttlDays * DAY_SECONDS; + let written = 0; + for (let index = 0; index < indicators.length; index += 25) { + const chunk = indicators.slice(index, index + 25); + const result = await this.client.send( + new BatchWriteItemCommand({ + RequestItems: { + [this.tableName]: chunk.map((indicator) => ({ + PutRequest: { + Item: marshall({ + indicator: indicatorKey(indicator.type, indicator.value), + type: indicator.type, + value: indicator.value, + source: indicator.source, + added_at: now.toISOString(), + expires_at: expiresAt, + }) as Record, + }, + })), + }, + }) + ); + const unprocessed = result.UnprocessedItems?.[this.tableName] ?? []; + written += chunk.length - unprocessed.length; + if (unprocessed.length > 0) { + throw new Error( + `DynamoDB left ${unprocessed.length} threat indicators unprocessed; re-run the import` + ); + } + } + return written; + } + + // The stored indicators among these candidates; bounded like every other + // read in the runner. + async find( + candidates: Array<{ type: ThreatIndicatorType; value: string }> + ): Promise { + const unique = [ + ...new Map(candidates.map((c) => [indicatorKey(c.type, c.value), c])).values(), + ].slice(0, MAX_LOOKUPS); + const found: ThreatIndicator[] = []; + for (let index = 0; index < unique.length; index += CHUNK_SIZE) { + const chunk = unique.slice(index, index + CHUNK_SIZE); + const response = await this.client.send( + new BatchGetItemCommand({ + RequestItems: { + [this.tableName]: { + Keys: chunk.map((candidate) => ({ + indicator: { S: indicatorKey(candidate.type, candidate.value) }, + })), + }, + }, + }) + ); + for (const item of response.Responses?.[this.tableName] ?? []) { + const type = item.type?.S as ThreatIndicatorType | undefined; + const value = item.value?.S; + if (!type || !value) continue; + found.push({ type, value, source: item.source?.S ?? 'unknown' }); + } + } + return found; + } +} diff --git a/packages/risk-engine/src/types.test.ts b/packages/risk-engine/src/types.test.ts index f2ef8208..7114fa6a 100644 --- a/packages/risk-engine/src/types.test.ts +++ b/packages/risk-engine/src/types.test.ts @@ -215,6 +215,7 @@ describe('Types', () => { environmentContribution: 15, crownJewelContribution: 0, exploitabilityContribution: 0, + threatIntelContribution: 0, }, }; expect(output.score).toBe(85); diff --git a/packages/risk-engine/src/types.ts b/packages/risk-engine/src/types.ts index 16a415b4..08e7873f 100644 --- a/packages/risk-engine/src/types.ts +++ b/packages/risk-engine/src/types.ts @@ -146,6 +146,8 @@ export interface RiskScoreInput { reachabilityRestricted?: boolean; /** A path vertex has observed traffic (flow-log analyzer), so reachability is proven. */ reachabilityObserved?: boolean; + /** Customer threat-intel indicators matching this path's facts. */ + threatIntel?: { indicators: string[] }; } export interface RiskScoreOutput { @@ -158,5 +160,6 @@ export interface RiskScoreOutput { environmentContribution: number; crownJewelContribution: number; exploitabilityContribution: number; + threatIntelContribution: number; }; } diff --git a/ui/app/issues/[id]/page.test.tsx b/ui/app/issues/[id]/page.test.tsx index 6e0fd892..f7b2133b 100644 --- a/ui/app/issues/[id]/page.test.tsx +++ b/ui/app/issues/[id]/page.test.tsx @@ -274,4 +274,27 @@ describe('IssuePage fix simulation', () => { await within(card).findByText('No single edge clears a matched path.') ).toBeInTheDocument(); }); + + it('lists the threat-intel matches recorded on the issue', async () => { + mockedIssue.mockResolvedValue( + detail({ + metadata: { + ...detail().metadata, + threatIntelMatches: [{ type: 'ip', value: '203.0.113.7', source: 's3://lists/ips.txt' }], + }, + }) + ); + + render(); + + const card = (await screen.findByText('Threat intelligence')).closest('section')!; + expect(within(card).getByText('203.0.113.7')).toBeInTheDocument(); + expect(within(card).getByText(/ip · s3:\/\/lists\/ips.txt/)).toBeInTheDocument(); + }); + + it('shows no threat-intel card without a match', async () => { + render(); + expect(await screen.findByText('Involved resources')).toBeInTheDocument(); + expect(screen.queryByText('Threat intelligence')).not.toBeInTheDocument(); + }); }); diff --git a/ui/app/issues/[id]/page.tsx b/ui/app/issues/[id]/page.tsx index 37341de8..7782848a 100644 --- a/ui/app/issues/[id]/page.tsx +++ b/ui/app/issues/[id]/page.tsx @@ -58,6 +58,7 @@ const FACTOR_LABELS: Record = { environmentContribution: 'Environment', crownJewelContribution: 'Crown jewel / short path', exploitabilityContribution: 'Exploitability (CVE)', + threatIntelContribution: 'Threat intelligence', }; function metaString( @@ -200,6 +201,40 @@ function FixSimulation({ ruleId }: { ruleId: string }) { ); } +// Customer threat-intel matches recorded on the issue (anomaly source IPs and +// trusted external accounts on the customer's own lists). +function ThreatIntelMatches({ detail }: { detail: IssueDetail }) { + const raw = detail.metadata?.threatIntelMatches; + const matches = Array.isArray(raw) + ? (raw as Array<{ type?: unknown; value?: unknown; source?: unknown }>).filter( + (match): match is { type?: unknown; value: string; source?: unknown } => + !!match && typeof match.value === 'string' + ) + : []; + if (matches.length === 0) return null; + return ( + +
    + {matches.map((match) => ( +
  • + {match.value} + + {typeof match.type === 'string' ? match.type : 'indicator'} + {typeof match.source === 'string' && match.source ? ` · ${match.source}` : ''} + +
    • + ))} +
+ + ); +} + function StatusDetails({ issue }: { issue: IssueDetail }) { const history = [...(issue.statusHistory ?? [])].reverse(); return ( @@ -584,6 +619,8 @@ export default function IssueDetailPage() {
+ + {detail.metadata?.scoringFactors != null && (