From 2656b47b43bfe24f77df5f5ee9391e7d650d1bbf Mon Sep 17 00:00:00 2001 From: Brian McMahon Date: Mon, 6 Apr 2026 14:00:07 -0700 Subject: [PATCH] feat: add VWAP to polygon/daily_closes + SSM secrets + unified push scripts VWAP: polygon_client now extracts vw field from grouped-daily, daily_closes includes VWAP column in parquet. Enables executor VWAP discount entry trigger. SSM: all modules load secrets from AWS SSM Parameter Store (/alpha-engine/*) at startup via ssm_secrets.py, eliminating need to push .env to each target. Scripts: push-secrets.sh (Lambda+EC2), push-configs.sh (config files to EC2), seed-ssm.sh (migrate .env to SSM), add-ssm-policy.sh (IAM permissions). Co-Authored-By: Claude Opus 4.6 (1M context) --- .env.example | 10 +- collectors/daily_closes.py | 4 +- infrastructure/add-ssm-policy.sh | 72 +++++++++++ infrastructure/push-configs.sh | 150 +++++++++++++++++++++++ infrastructure/push-secrets.sh | 204 +++++++++++++++++++++++++++++++ infrastructure/seed-ssm.sh | 126 +++++++++++++++++++ infrastructure/sync-secrets.sh | 5 +- lambda/handler.py | 6 + polygon_client.py | 6 +- ssm_secrets.py | 77 ++++++++++++ weekly_collector.py | 3 + 11 files changed, 654 insertions(+), 9 deletions(-) create mode 100755 infrastructure/add-ssm-policy.sh create mode 100755 infrastructure/push-configs.sh create mode 100755 infrastructure/push-secrets.sh create mode 100755 infrastructure/seed-ssm.sh create mode 100644 ssm_secrets.py diff --git a/.env.example b/.env.example index 6b24fee..4bcb94c 100644 --- a/.env.example +++ b/.env.example @@ -1,9 +1,13 @@ # Alpha Engine — Master Environment Variables # Copy to .env and fill in values. .env is gitignored. # -# This is the SINGLE SOURCE OF TRUTH for all API keys and secrets. -# Run `bash infrastructure/sync-secrets.sh` to push to EC2 instances. -# Lambda deploy scripts read from this file automatically. +# PREFERRED: Store secrets in AWS SSM Parameter Store (under /alpha-engine/). +# All modules load from SSM at startup. Use seed-ssm.sh to migrate: +# bash infrastructure/seed-ssm.sh # create SSM params from .env +# bash infrastructure/add-ssm-policy.sh # add IAM permissions +# +# LEGACY: Push .env directly to Lambda/EC2: +# bash infrastructure/push-secrets.sh # push to all targets # # ─── LLM / AI ──────────────────────────────────────────────────────────────── ANTHROPIC_API_KEY= diff --git a/collectors/daily_closes.py b/collectors/daily_closes.py index 2805c6c..7fcbaa9 100644 --- a/collectors/daily_closes.py +++ b/collectors/daily_closes.py @@ -8,7 +8,7 @@ Data source priority: polygon.io grouped-daily (1 API call for all US stocks), then yfinance batch download for any tickers polygon missed. -Schema: index=ticker (str), columns=[date, Open, High, Low, Close, Adj_Close, Volume] +Schema: index=ticker (str), columns=[date, Open, High, Low, Close, Adj_Close, Volume, VWAP] """ from __future__ import annotations @@ -84,6 +84,7 @@ def collect( "Close": round(g["close"], 4), "Adj_Close": round(g["close"], 4), "Volume": int(g["volume"]), + "VWAP": round(g["vwap"], 4) if g.get("vwap") else None, }) polygon_count = len(records) logger.info("Polygon grouped-daily: %d/%d tickers", polygon_count, len(tickers)) @@ -191,6 +192,7 @@ def _fetch_yfinance_closes( "Close": round(float(last["Close"]), 4), "Adj_Close": round(adj_close, 4), "Volume": int(last["Volume"]) if pd.notna(last.get("Volume")) else 0, + "VWAP": None, }) count += 1 except Exception as e: diff --git a/infrastructure/add-ssm-policy.sh b/infrastructure/add-ssm-policy.sh new file mode 100755 index 0000000..97ebf4d --- /dev/null +++ b/infrastructure/add-ssm-policy.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# add-ssm-policy.sh — Add SSM Parameter Store read access to all Alpha Engine IAM roles. +# +# Adds an inline policy allowing ssm:GetParametersByPath and ssm:GetParameter +# on /alpha-engine/* to each Lambda execution role and the EC2 instance role. +# +# Usage: +# bash infrastructure/add-ssm-policy.sh # apply to all roles +# bash infrastructure/add-ssm-policy.sh --dry-run # show what would be applied + +set -euo pipefail + +REGION="${AWS_REGION:-us-east-1}" +ACCOUNT_ID=$(aws sts get-caller-identity --query "Account" --output text 2>/dev/null) + +if [ -z "$ACCOUNT_ID" ]; then + echo "ERROR: Could not determine AWS account ID. Check AWS credentials." + exit 1 +fi + +POLICY_NAME="alpha-engine-ssm-read" +POLICY_DOC=$(cat </dev/null; then + echo " SKIP $role: role does not exist" + continue + fi + + aws iam put-role-policy --role-name "$role" --policy-name "$POLICY_NAME" --policy-document "$POLICY_DOC" 2>&1 && echo " OK: $role" || echo " FAILED: $role" +done + +echo "" +if [ "$DRY_RUN" = true ]; then + echo "(dry-run — no changes made)" +else + echo "Done. Verify with:" + echo " aws iam get-role-policy --role-name alpha-engine-data-role --policy-name $POLICY_NAME --query 'PolicyDocument' --output json" +fi diff --git a/infrastructure/push-configs.sh b/infrastructure/push-configs.sh new file mode 100755 index 0000000..22b42ab --- /dev/null +++ b/infrastructure/push-configs.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# push-configs.sh — Push gitignored config files from local repos to EC2 instances. +# +# Configs are gitignored and never committed. Edit locally, then push with this script. +# Each config goes to the instance(s) that need it. +# +# Config mapping: +# alpha-engine/config/risk.yaml → trading EC2: ~/alpha-engine/config/risk.yaml +# alpha-engine-data/config.yaml → micro EC2: ~/alpha-engine-data/config.yaml +# alpha-engine-backtester/config.yaml → micro EC2: ~/alpha-engine-backtester/config.yaml +# +# Usage: +# bash infrastructure/push-configs.sh # push all configs +# bash infrastructure/push-configs.sh --dry-run # show what would be pushed +# bash infrastructure/push-configs.sh --trading # trading instance only +# bash infrastructure/push-configs.sh --micro # micro instance only + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +DEV_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +SSH_KEY="$HOME/.ssh/alpha-engine-key.pem" +REGION="${AWS_REGION:-us-east-1}" + +TRADING_INSTANCE="i-018eb3307a21329bf" +MICRO_INSTANCE="i-09b539c844515d549" + +# Config file mapping: local_path → instance_type → remote_path +# Trading instance configs +TRADING_CONFIGS=("$DEV_ROOT/alpha-engine/config/risk.yaml:/home/ec2-user/alpha-engine/config/risk.yaml") + +# Micro instance configs +MICRO_CONFIGS=("$DEV_ROOT/alpha-engine-data/config.yaml:/home/ec2-user/alpha-engine-data/config.yaml" "$DEV_ROOT/alpha-engine-backtester/config.yaml:/home/ec2-user/alpha-engine-backtester/config.yaml") + +# ── Parse args ────────────────────────────────────────────────────────────── + +TARGET="all" +DRY_RUN=false +for arg in "$@"; do + case "$arg" in + --dry-run) DRY_RUN=true ;; + --trading) TARGET="trading" ;; + --micro) TARGET="micro" ;; + *) echo "Unknown arg: $arg"; exit 1 ;; + esac +done + +# ── Validate ──────────────────────────────────────────────────────────────── + +if [ ! -f "$SSH_KEY" ]; then + echo "ERROR: SSH key not found at $SSH_KEY" + exit 1 +fi + +# ── Helpers ───────────────────────────────────────────────────────────────── + +get_instance_ip() { + local instance_id="$1" + aws ec2 describe-instances --instance-ids "$instance_id" --region "$REGION" --query 'Reservations[0].Instances[0].PublicIpAddress' --output text 2>/dev/null +} + +push_config() { + local local_path="$1" + local remote_path="$2" + local ip="$3" + local name="$4" + local basename + basename=$(basename "$local_path") + + if [ ! -f "$local_path" ]; then + echo " SKIP $basename: local file not found at $local_path" + return 1 + fi + + if [ "$DRY_RUN" = true ]; then + echo " $basename → $name:$remote_path" + return 0 + fi + + echo -n " $basename → $name ... " + scp -i "$SSH_KEY" -o StrictHostKeyChecking=no -o ConnectTimeout=10 "$local_path" "ec2-user@${ip}:${remote_path}" 2>/dev/null + if [ $? -eq 0 ]; then + echo "OK" + return 0 + else + echo "FAILED" + return 1 + fi +} + +push_to_instance() { + local name="$1" + local instance_id="$2" + shift 2 + local configs=("$@") + + local ip + ip=$(get_instance_ip "$instance_id") + if [ -z "$ip" ] || [ "$ip" = "None" ]; then + echo " SKIP $name: instance not running or no public IP" + return 1 + fi + + [ "$DRY_RUN" = false ] && echo "=== $name ($ip) ===" + [ "$DRY_RUN" = true ] && echo "=== $name ($instance_id) ===" + + local failed=0 + for entry in "${configs[@]}"; do + local local_path="${entry%%:*}" + local remote_path="${entry#*:}" + push_config "$local_path" "$remote_path" "$ip" "$name" || failed=$((failed + 1)) + done + return $failed +} + +# ── Dry run header ────────────────────────────────────────────────────────── + +if [ "$DRY_RUN" = true ]; then + echo "Configs that would be pushed:" + echo "" +fi + +# ── Push ──────────────────────────────────────────────────────────────────── + +FAILED=0 + +if [ "$TARGET" = "all" ] || [ "$TARGET" = "trading" ]; then + push_to_instance "trading" "$TRADING_INSTANCE" "${TRADING_CONFIGS[@]}" || FAILED=$((FAILED + 1)) + echo "" +fi + +if [ "$TARGET" = "all" ] || [ "$TARGET" = "micro" ]; then + push_to_instance "micro" "$MICRO_INSTANCE" "${MICRO_CONFIGS[@]}" || FAILED=$((FAILED + 1)) + echo "" +fi + +# ── Summary ───────────────────────────────────────────────────────────────── + +if [ "$DRY_RUN" = true ]; then + echo "(dry-run — no changes made)" +else + if [ "$FAILED" -eq 0 ]; then + echo "All configs pushed successfully." + else + echo "Completed with $FAILED failure(s)." + fi + echo "" + echo "Note: Executor reads risk.yaml on each run (morning batch + daemon start)." + echo " Changes take effect on next trading day boot." +fi diff --git a/infrastructure/push-secrets.sh b/infrastructure/push-secrets.sh new file mode 100755 index 0000000..b98cef9 --- /dev/null +++ b/infrastructure/push-secrets.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +# push-secrets.sh — Push master .env to all Lambda functions and EC2 instances. +# +# Single command to sync secrets everywhere after editing .env. +# Lambda gets env vars above the # LAMBDA_SKIP marker. +# EC2 gets the full .env via SCP. +# +# Usage: +# bash infrastructure/push-secrets.sh # push to all targets +# bash infrastructure/push-secrets.sh --dry-run # show what would be pushed +# bash infrastructure/push-secrets.sh --lambda-only # Lambda functions only +# bash infrastructure/push-secrets.sh --ec2-only # EC2 instances only +# +# Prerequisites: +# - .env file in repo root (copy from .env.example) +# - SSH key at ~/.ssh/alpha-engine-key.pem (for EC2) +# - AWS CLI configured (for Lambda) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ENV_FILE="$REPO_ROOT/.env" +SSH_KEY="$HOME/.ssh/alpha-engine-key.pem" +REGION="${AWS_REGION:-us-east-1}" +REMOTE_PATH="/home/ec2-user/.alpha-engine.env" + +# All Lambda functions that receive env vars from master .env +LAMBDA_FUNCTIONS=("alpha-engine-data-collector" "alpha-engine-research-runner" "alpha-engine-research-alerts" "alpha-engine-predictor-inference") + +# EC2 instance IDs +TRADING_INSTANCE="i-018eb3307a21329bf" +MICRO_INSTANCE="i-09b539c844515d549" + +# ── Parse args ────────────────────────────────────────────────────────────── + +TARGET="all" +DRY_RUN=false +for arg in "$@"; do + case "$arg" in + --dry-run) DRY_RUN=true ;; + --lambda-only) TARGET="lambda-only" ;; + --ec2-only) TARGET="ec2-only" ;; + *) echo "Unknown arg: $arg"; exit 1 ;; + esac +done + +# ── Validate ──────────────────────────────────────────────────────────────── + +if [ ! -f "$ENV_FILE" ]; then + echo "ERROR: $ENV_FILE not found." + echo " Copy .env.example to .env and fill in your API keys." + exit 1 +fi + +N_VARS=$(grep -cE "^[A-Z_]+=.+" "$ENV_FILE" 2>/dev/null || echo "0") +N_LAMBDA_VARS=$(awk '/^# LAMBDA_SKIP/{exit} /^[A-Z_]+=.+/{n++} END{print n+0}' "$ENV_FILE") +echo "Master .env: $N_VARS total variables ($N_LAMBDA_VARS for Lambda)" + +if [ "$N_VARS" -lt 5 ]; then + echo "WARNING: Only $N_VARS variables have values — did you fill in .env?" +fi + +# ── Build Lambda env JSON ─────────────────────────────────────────────────── +# Reads .env up to # LAMBDA_SKIP marker and outputs {"Variables": {...}} + +build_lambda_env_json() { + python3 -c " +import json +env = {} +with open('$ENV_FILE') as f: + for line in f: + line = line.strip() + if line == '# LAMBDA_SKIP': + break + if not line or line.startswith('#'): + continue + if '=' not in line: + continue + key, val = line.split('=', 1) + key, val = key.strip(), val.strip() + if len(val) >= 2 and val[0] == val[-1] and val[0] in ('\"', \"'\"): + val = val[1:-1] + if key and val: + env[key] = val +if env: + print(json.dumps({'Variables': env})) +else: + print('') +" +} + +# ── Dry run ───────────────────────────────────────────────────────────────── + +if [ "$DRY_RUN" = true ]; then + echo "" + echo "Lambda variables (above # LAMBDA_SKIP):" + awk '/^# LAMBDA_SKIP/{exit} /^[A-Z_]+=.+/{sub(/=.*/, "=***"); print " " $0}' "$ENV_FILE" | sort + echo "" + if [ "$TARGET" != "ec2-only" ]; then + echo "Lambda targets:" + for fn in "${LAMBDA_FUNCTIONS[@]}"; do + echo " $fn" + done + fi + if [ "$TARGET" != "lambda-only" ]; then + echo "EC2 targets:" + echo " trading ($TRADING_INSTANCE)" + echo " micro ($MICRO_INSTANCE)" + fi + echo "" + echo "(dry-run — no changes made)" + exit 0 +fi + +# ── Push to Lambda ────────────────────────────────────────────────────────── + +FAILED=0 +LAMBDA_OK=0 +EC2_OK=0 + +if [ "$TARGET" != "ec2-only" ]; then + LAMBDA_ENV_JSON=$(build_lambda_env_json) + + if [ -z "$LAMBDA_ENV_JSON" ]; then + echo "WARNING: No Lambda env vars found — skipping Lambda push" + else + echo "" + echo "=== Pushing env vars to Lambda functions ===" + for fn in "${LAMBDA_FUNCTIONS[@]}"; do + echo -n " $fn ... " + result=$(aws lambda update-function-configuration --function-name "$fn" --environment "$LAMBDA_ENV_JSON" --region "$REGION" --query "LastUpdateStatus" --output text 2>&1) || true + if echo "$result" | grep -qE "Successful|InProgress"; then + echo "$result" + LAMBDA_OK=$((LAMBDA_OK + 1)) + else + echo "FAILED: $result" + FAILED=$((FAILED + 1)) + fi + done + fi +fi + +# ── Push to EC2 ───────────────────────────────────────────────────────────── + +sync_to_instance() { + local name="$1" + local instance_id="$2" + + echo -n " $name ($instance_id) ... " + + if [ ! -f "$SSH_KEY" ]; then + echo "SKIP: SSH key not found at $SSH_KEY" + return 1 + fi + + local ip + ip=$(aws ec2 describe-instances --instance-ids "$instance_id" --region "$REGION" --query 'Reservations[0].Instances[0].PublicIpAddress' --output text 2>/dev/null) + + if [ -z "$ip" ] || [ "$ip" = "None" ]; then + echo "SKIP: not running or no public IP" + return 1 + fi + + scp -i "$SSH_KEY" -o StrictHostKeyChecking=no -o ConnectTimeout=10 "$ENV_FILE" "ec2-user@${ip}:${REMOTE_PATH}" 2>/dev/null + + if [ $? -eq 0 ]; then + local remote_vars + remote_vars=$(ssh -i "$SSH_KEY" -o StrictHostKeyChecking=no -o ConnectTimeout=10 "ec2-user@${ip}" "grep -cE '^[A-Z_]+=.+' $REMOTE_PATH 2>/dev/null" 2>/dev/null || echo "0") + echo "OK ($remote_vars vars written)" + else + echo "FAILED: SCP failed" + return 1 + fi +} + +if [ "$TARGET" != "lambda-only" ]; then + echo "" + echo "=== Pushing .env to EC2 instances ===" + if sync_to_instance "trading" "$TRADING_INSTANCE"; then + EC2_OK=$((EC2_OK + 1)) + else + FAILED=$((FAILED + 1)) + fi + if sync_to_instance "micro" "$MICRO_INSTANCE"; then + EC2_OK=$((EC2_OK + 1)) + else + FAILED=$((FAILED + 1)) + fi +fi + +# ── Summary ───────────────────────────────────────────────────────────────── + +echo "" +if [ "$FAILED" -eq 0 ]; then + echo "All targets updated successfully." +else + echo "Completed with $FAILED failure(s)." +fi +[ "$LAMBDA_OK" -gt 0 ] && echo " Lambda: $LAMBDA_OK/${#LAMBDA_FUNCTIONS[@]} functions" +[ "$EC2_OK" -gt 0 ] && echo " EC2: $EC2_OK/2 instances" +echo "" +echo "Note: EC2 systemd services read env on next restart." +echo " Lambda env vars take effect on next cold start." diff --git a/infrastructure/seed-ssm.sh b/infrastructure/seed-ssm.sh new file mode 100755 index 0000000..1ad1b0e --- /dev/null +++ b/infrastructure/seed-ssm.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# seed-ssm.sh — Seed AWS SSM Parameter Store from master .env file. +# +# One-time migration: reads .env and creates SecureString parameters +# under /alpha-engine/ prefix. After this, all modules read from SSM +# at startup — no more pushing .env to Lambda/EC2. +# +# Usage: +# bash infrastructure/seed-ssm.sh # create/update all params +# bash infrastructure/seed-ssm.sh --dry-run # show what would be created +# bash infrastructure/seed-ssm.sh --delete # remove all /alpha-engine/ params +# +# To update a single secret later: +# aws ssm put-parameter --name "/alpha-engine/POLYGON_API_KEY" \ +# --type SecureString --value "new-key" --overwrite \ +# --query "Version" --output text + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +ENV_FILE="$REPO_ROOT/.env" +REGION="${AWS_REGION:-us-east-1}" +PREFIX="/alpha-engine" + +# ── Parse args ────────────────────────────────────────────────────────────── + +DRY_RUN=false +DELETE=false +for arg in "$@"; do + case "$arg" in + --dry-run) DRY_RUN=true ;; + --delete) DELETE=true ;; + *) echo "Unknown arg: $arg"; exit 1 ;; + esac +done + +# ── Delete mode ───────────────────────────────────────────────────────────── + +if [ "$DELETE" = true ]; then + echo "Deleting all parameters under $PREFIX/..." + params=$(aws ssm get-parameters-by-path --path "$PREFIX/" --region "$REGION" --query "Parameters[].Name" --output text 2>/dev/null || echo "") + if [ -z "$params" ]; then + echo " No parameters found." + exit 0 + fi + for name in $params; do + if [ "$DRY_RUN" = true ]; then + echo " Would delete: $name" + else + aws ssm delete-parameter --name "$name" --region "$REGION" --output text 2>/dev/null && echo " Deleted: $name" || echo " Failed: $name" + fi + done + exit 0 +fi + +# ── Validate ──────────────────────────────────────────────────────────────── + +if [ ! -f "$ENV_FILE" ]; then + echo "ERROR: $ENV_FILE not found." + echo " Copy .env.example to .env and fill in your API keys." + exit 1 +fi + +# ── Read .env and seed SSM ────────────────────────────────────────────────── + +echo "Seeding SSM Parameter Store from $ENV_FILE" +echo "Prefix: $PREFIX/" +echo "Region: $REGION" +echo "" + +CREATED=0 +SKIPPED=0 +FAILED=0 + +while IFS= read -r line; do + line=$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + [[ -z "$line" || "$line" == \#* ]] && continue + [[ "$line" != *=* ]] && continue + + key="${line%%=*}" + val="${line#*=}" + key=$(echo "$key" | xargs) + val=$(echo "$val" | xargs) + + # Strip surrounding quotes + if [[ ${#val} -ge 2 && ( ("${val:0:1}" == '"' && "${val: -1}" == '"') || ("${val:0:1}" == "'" && "${val: -1}" == "'") ) ]]; then + val="${val:1:${#val}-2}" + fi + + # Skip empty values + if [ -z "$val" ]; then + continue + fi + + param_name="$PREFIX/$key" + + if [ "$DRY_RUN" = true ]; then + echo " $param_name = ***" + CREATED=$((CREATED + 1)) + continue + fi + + result=$(aws ssm put-parameter --name "$param_name" --type SecureString --value "$val" --overwrite --region "$REGION" --query "Version" --output text 2>&1) || true + + if echo "$result" | grep -qE "^[0-9]+$"; then + echo " $param_name (v$result)" + CREATED=$((CREATED + 1)) + else + echo " FAILED $param_name: $result" + FAILED=$((FAILED + 1)) + fi +done < "$ENV_FILE" + +echo "" +if [ "$DRY_RUN" = true ]; then + echo "$CREATED parameters would be created/updated." + echo "(dry-run — no changes made)" +else + echo "Done: $CREATED created/updated, $FAILED failed." + echo "" + echo "Next steps:" + echo " 1. Add ssm:GetParametersByPath to Lambda/EC2 IAM roles" + echo " 2. Deploy modules with ssm_secrets.py loader" + echo " 3. Verify: aws ssm get-parameters-by-path --path '$PREFIX/' --region $REGION --query 'Parameters[].Name' --output text" +fi diff --git a/infrastructure/sync-secrets.sh b/infrastructure/sync-secrets.sh index 9640557..56cbcc8 100755 --- a/infrastructure/sync-secrets.sh +++ b/infrastructure/sync-secrets.sh @@ -136,7 +136,6 @@ else echo "Sync complete with $FAILED failure(s). Check instance status." fi echo "" -echo "Lambda env vars are synced separately during deploy:" -echo " cd ~/Development/alpha-engine-research && bash infrastructure/deploy.sh" -echo " cd ~/Development/alpha-engine-data && bash infrastructure/deploy.sh" +echo "To push to Lambda + EC2 in one command, use push-secrets.sh instead:" +echo " bash infrastructure/push-secrets.sh" echo "" diff --git a/lambda/handler.py b/lambda/handler.py index c74c17d..91f09ec 100644 --- a/lambda/handler.py +++ b/lambda/handler.py @@ -13,9 +13,15 @@ import logging import os +import sys import time import traceback +# Load secrets from SSM Parameter Store (must run before any os.environ.get) +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) +from ssm_secrets import load_secrets +load_secrets() + logger = logging.getLogger(__name__) diff --git a/polygon_client.py b/polygon_client.py index 316609b..a1b7929 100644 --- a/polygon_client.py +++ b/polygon_client.py @@ -94,10 +94,11 @@ def _get(self, path: str, params: dict | None = None) -> dict: # -- Core endpoints ------------------------------------------------------ def get_grouped_daily(self, date_str: str) -> dict[str, dict]: - """Fetch OHLCV for ALL US stocks on a single date. + """Fetch OHLCV + VWAP for ALL US stocks on a single date. Returns {ticker: {"open": float, "high": float, "low": float, - "close": float, "volume": float}} + "close": float, "volume": float, + "vwap": float | None}} """ data = self._get( f"/v2/aggs/grouped/locale/us/market/stocks/{date_str}", @@ -111,6 +112,7 @@ def get_grouped_daily(self, date_str: str) -> dict[str, dict]: "low": r["l"], "close": r["c"], "volume": r["v"], + "vwap": r.get("vw"), } for r in results if "T" in r diff --git a/ssm_secrets.py b/ssm_secrets.py new file mode 100644 index 0000000..b2802ad --- /dev/null +++ b/ssm_secrets.py @@ -0,0 +1,77 @@ +""" +Load secrets from AWS SSM Parameter Store into os.environ. + +All Alpha Engine secrets are stored under the /alpha-engine/ prefix in SSM +Parameter Store (SecureString type). This module fetches them at startup and +sets os.environ so existing code (which uses os.environ.get()) works unchanged. + +Falls back to .env file if SSM is unavailable (local development). + +Usage (call once at module startup, before any os.environ.get): + from ssm_secrets import load_secrets + load_secrets() +""" + +from __future__ import annotations + +import logging +import os + +logger = logging.getLogger(__name__) + +SSM_PREFIX = "/alpha-engine/" +_loaded = False + + +def load_secrets(prefix: str = SSM_PREFIX, region: str | None = None) -> int: + """ + Fetch all parameters under prefix from SSM and set as env vars. + + Parameter names are converted to env var names by stripping the prefix + and converting to uppercase. E.g., /alpha-engine/POLYGON_API_KEY → POLYGON_API_KEY. + + Returns the number of parameters loaded. Skips any that are already set + in the environment (explicit env vars take precedence over SSM). + + Falls back silently if SSM is unavailable (e.g., local dev without AWS creds). + """ + global _loaded + if _loaded: + return 0 + + region = region or os.environ.get("AWS_REGION", "us-east-1") + count = 0 + + try: + import boto3 + client = boto3.client("ssm", region_name=region) + + paginator = client.get_paginator("get_parameters_by_path") + pages = paginator.paginate( + Path=prefix, + Recursive=False, + WithDecryption=True, + ) + + for page in pages: + for param in page.get("Parameters", []): + name = param["Name"] + value = param["Value"] + env_key = name.replace(prefix, "", 1) + if not env_key: + continue + if env_key not in os.environ: + os.environ[env_key] = value + count += 1 + else: + logger.debug("SSM skip %s: already set in environment", env_key) + + _loaded = True + logger.info("Loaded %d secrets from SSM %s", count, prefix) + + except ImportError: + logger.debug("boto3 not available — skipping SSM secrets load") + except Exception as e: + logger.warning("SSM secrets load failed (falling back to env): %s", e) + + return count diff --git a/weekly_collector.py b/weekly_collector.py index 306e51b..d1afe52 100644 --- a/weekly_collector.py +++ b/weekly_collector.py @@ -31,6 +31,9 @@ import boto3 import yaml +from ssm_secrets import load_secrets +load_secrets() + from collectors import constituents, prices, slim_cache, macro, universe_returns, alternative, daily_closes, fundamentals logger = logging.getLogger(__name__)