diff --git a/.mcp/firebase.json b/.mcp/firebase.json new file mode 100644 index 0000000..4d71eb6 --- /dev/null +++ b/.mcp/firebase.json @@ -0,0 +1,14 @@ +{ + "firebase": { + "command": "npx", + "args": [ + "-y", + "firebase-tools@latest", + "mcp", + "--dir", + "${FIREBASE_PROJECT_DIR}", + "--only", + "firestore,storage" + ] + } +} diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 714837c..37e2c80 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -15,6 +15,48 @@ "reveal": "never" }, "problemMatcher": "$tsc" + }, + { + "label": "NOIZY: Observe (1)", + "type": "shell", + "command": "npm run autonomy:center -- 1", + "problemMatcher": [] + }, + { + "label": "NOIZY: Triage (2)", + "type": "shell", + "command": "npm run autonomy:center -- 2", + "problemMatcher": [] + }, + { + "label": "NOIZY: Sync Dry (3)", + "type": "shell", + "command": "npm run autonomy:center -- 3", + "problemMatcher": [] + }, + { + "label": "NOIZY: Sync Approve (4)", + "type": "shell", + "command": "npm run autonomy:center -- 4", + "problemMatcher": [] + }, + { + "label": "NOIZY: Duplicates (5)", + "type": "shell", + "command": "npm run autonomy:center -- 5", + "problemMatcher": [] + }, + { + "label": "NOIZY: Ship Gate (7)", + "type": "shell", + "command": "npm run autonomy:center -- 7", + "problemMatcher": [] + }, + { + "label": "NOIZY: FOSS Doctor (9)", + "type": "shell", + "command": "npm run autonomy:center -- 9", + "problemMatcher": [] } ] } diff --git a/cloudflare/src/index.mjs b/cloudflare/src/index.mjs new file mode 100644 index 0000000..03d8d10 --- /dev/null +++ b/cloudflare/src/index.mjs @@ -0,0 +1,386 @@ +const MAX_LIMIT = 500; +const DEFAULT_LIMIT = 50; +const DEFAULT_MAX_SYNC_BYTES = 1024 * 1024 * 1024; + +export default { + async fetch(request, env) { + const requestId = request.headers.get('cf-ray') || globalThis.crypto.randomUUID(); + + try { + const authError = validateAuth(request, env, requestId); + if (authError) { + return authError; + } + + const url = new URL(request.url); + const path = url.pathname; + + if (request.method === 'POST' && path === '/api/assets/register') { + return await registerAsset(request, env, requestId); + } + + if (request.method === 'GET' && path === '/api/assets/search') { + return await searchAssets(request, env, requestId); + } + + if (request.method === 'PUT' && path === '/api/assets/sync') { + return await syncAsset(request, env, requestId); + } + + return errorResponse(requestId, 404, 'NOT_FOUND', 'Route not found.'); + } catch (error) { + return errorResponse( + requestId, + 500, + 'INTERNAL_ERROR', + error instanceof Error ? error.message : 'Unknown worker error.' + ); + } + }, +}; + +function validateAuth(request, env, requestId) { + if (!env.API_AUTH_TOKEN) { + return errorResponse(requestId, 500, 'AUTH_NOT_CONFIGURED', 'API_AUTH_TOKEN is not configured.'); + } + + const authHeader = request.headers.get('authorization'); + if (!authHeader || !authHeader.startsWith('Bearer ')) { + return errorResponse(requestId, 401, 'UNAUTHORIZED', 'Missing bearer token.'); + } + + const token = authHeader.slice(7); + if (token !== env.API_AUTH_TOKEN) { + return errorResponse(requestId, 403, 'FORBIDDEN', 'Invalid bearer token.'); + } + + return null; +} + +async function registerAsset(request, env, requestId) { + const body = await readJsonBody(request, requestId); + if (body.errorResponse) { + return body.errorResponse; + } + + const input = body.value; + const validation = validateRegisterInput(input); + if (validation) { + return errorResponse(requestId, 400, 'VALIDATION_ERROR', validation); + } + + await env.DB.prepare( + `INSERT INTO hvs_creator_assets + (public_id, asset_name, app_category, app_name, file_type, r2_object_key, local_path_blueprint) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + ON CONFLICT(public_id) DO UPDATE SET + asset_name = excluded.asset_name, + app_category = excluded.app_category, + app_name = excluded.app_name, + file_type = excluded.file_type, + r2_object_key = excluded.r2_object_key, + local_path_blueprint = excluded.local_path_blueprint` + ) + .bind( + input.public_id, + input.asset_name, + input.app_category, + input.app_name, + input.file_type, + optionalString(input.r2_object_key), + optionalString(input.local_path_blueprint) + ) + .run(); + + if (optionalString(input.content_sha256) || Number.isFinite(input.size_bytes)) { + await env.DB.prepare( + `INSERT INTO hvs_creator_asset_integrity + (public_id, content_sha256, size_bytes, last_sync_status, last_synced_at) + VALUES (?1, ?2, ?3, 'registered', CURRENT_TIMESTAMP) + ON CONFLICT(public_id) DO UPDATE SET + content_sha256 = COALESCE(excluded.content_sha256, hvs_creator_asset_integrity.content_sha256), + size_bytes = COALESCE(excluded.size_bytes, hvs_creator_asset_integrity.size_bytes), + last_sync_status = 'registered', + last_synced_at = CURRENT_TIMESTAMP` + ) + .bind( + input.public_id, + optionalString(input.content_sha256), + optionalInteger(input.size_bytes) + ) + .run(); + } + + return okResponse(requestId, { + public_id: input.public_id, + status: 'registered', + }); +} + +async function searchAssets(request, env, requestId) { + const url = new URL(request.url); + const appName = optionalString(url.searchParams.get('app_name')); + const fileType = optionalString(url.searchParams.get('file_type')); + const publicId = optionalString(url.searchParams.get('public_id')); + const limit = normalizeLimit(url.searchParams.get('limit')); + const offset = normalizeOffset(url.searchParams.get('offset')); + + if (!appName && !fileType && !publicId) { + return errorResponse(requestId, 400, 'VALIDATION_ERROR', 'Provide app_name, file_type, or public_id.'); + } + + let sql = ` + SELECT + a.asset_id, + a.public_id, + a.asset_name, + a.app_category, + a.app_name, + a.file_type, + a.r2_object_key, + a.local_path_blueprint, + a.created_at, + a.updated_at, + i.content_sha256, + i.size_bytes, + i.last_sync_status, + i.last_synced_at + FROM hvs_creator_assets a + LEFT JOIN hvs_creator_asset_integrity i + ON i.public_id = a.public_id + WHERE 1 = 1 + `; + const binds = []; + + if (publicId) { + sql += ' AND a.public_id = ?'; + binds.push(publicId); + } + + if (appName) { + sql += ' AND a.app_name = ?'; + binds.push(appName); + } + + if (fileType) { + sql += ' AND a.file_type = ?'; + binds.push(fileType); + } + + sql += ' ORDER BY a.updated_at DESC LIMIT ? OFFSET ?'; + binds.push(limit, offset); + + const { results } = await env.DB.prepare(sql).bind(...binds).all(); + return okResponse(requestId, { + count: results.length, + limit, + offset, + results, + }); +} + +async function syncAsset(request, env, requestId) { + const url = new URL(request.url); + const publicId = optionalString(url.searchParams.get('public_id')); + const objectKey = optionalString(url.searchParams.get('object_key')); + const appName = optionalString(url.searchParams.get('app_name')); + const appCategory = optionalString(url.searchParams.get('app_category')); + const assetName = optionalString(url.searchParams.get('asset_name')); + const fileType = optionalString(url.searchParams.get('file_type')); + const localPathBlueprint = optionalString(url.searchParams.get('local_path_blueprint')); + const contentSha256 = optionalString(request.headers.get('x-content-sha256')); + const contentLength = optionalInteger(request.headers.get('content-length')); + const maxSyncBytes = optionalInteger(env.MAX_SYNC_BYTES) || DEFAULT_MAX_SYNC_BYTES; + + if (!publicId || !objectKey) { + return errorResponse(requestId, 400, 'VALIDATION_ERROR', 'public_id and object_key are required.'); + } + + if (!request.body) { + return errorResponse(requestId, 400, 'VALIDATION_ERROR', 'Request body is required.'); + } + + if (contentLength !== null && contentLength > maxSyncBytes) { + return errorResponse( + requestId, + 413, + 'PAYLOAD_TOO_LARGE', + `Payload exceeds MAX_SYNC_BYTES (${maxSyncBytes}).` + ); + } + + await env.ASSETS.put(objectKey, request.body, { + httpMetadata: { + contentType: request.headers.get('content-type') || 'application/octet-stream', + }, + customMetadata: { + public_id: publicId, + content_sha256: contentSha256 || '', + }, + }); + + await env.DB.prepare( + `INSERT INTO hvs_creator_assets + (public_id, asset_name, app_category, app_name, file_type, r2_object_key, local_path_blueprint) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + ON CONFLICT(public_id) DO UPDATE SET + r2_object_key = excluded.r2_object_key, + local_path_blueprint = COALESCE(excluded.local_path_blueprint, hvs_creator_assets.local_path_blueprint), + asset_name = COALESCE(excluded.asset_name, hvs_creator_assets.asset_name), + app_category = COALESCE(excluded.app_category, hvs_creator_assets.app_category), + app_name = COALESCE(excluded.app_name, hvs_creator_assets.app_name), + file_type = COALESCE(excluded.file_type, hvs_creator_assets.file_type)` + ) + .bind( + publicId, + assetName || objectKey, + appCategory || 'Uncategorized', + appName || 'Unknown App', + fileType || inferFileType(objectKey), + objectKey, + localPathBlueprint + ) + .run(); + + await env.DB.prepare( + `INSERT INTO hvs_creator_asset_integrity + (public_id, content_sha256, size_bytes, last_sync_status, last_synced_at) + VALUES (?1, ?2, ?3, 'synced', CURRENT_TIMESTAMP) + ON CONFLICT(public_id) DO UPDATE SET + content_sha256 = COALESCE(excluded.content_sha256, hvs_creator_asset_integrity.content_sha256), + size_bytes = COALESCE(excluded.size_bytes, hvs_creator_asset_integrity.size_bytes), + last_sync_status = 'synced', + last_synced_at = CURRENT_TIMESTAMP` + ) + .bind(publicId, contentSha256, contentLength) + .run(); + + return okResponse(requestId, { + public_id: publicId, + object_key: objectKey, + checksum: contentSha256 || null, + size_bytes: contentLength, + status: 'synced', + }); +} + +async function readJsonBody(request, requestId) { + try { + return { value: await request.json() }; + } catch { + return { + errorResponse: errorResponse(requestId, 400, 'INVALID_JSON', 'Request body must be valid JSON.'), + }; + } +} + +function validateRegisterInput(input) { + const required = ['public_id', 'asset_name', 'app_category', 'app_name', 'file_type']; + for (const key of required) { + if (!optionalString(input[key])) { + return `Missing required field: ${key}`; + } + } + + if (optionalString(input.reason) && String(input.reason).length > 1000) { + return 'reason exceeds 1000 characters.'; + } + + if (String(input.file_type).length > 32) { + return 'file_type exceeds 32 characters.'; + } + + return null; +} + +function inferFileType(objectKey) { + const index = objectKey.lastIndexOf('.'); + if (index < 0 || index === objectKey.length - 1) { + return '.bin'; + } + + return objectKey.slice(index).toLowerCase(); +} + +function optionalString(value) { + if (value === null || value === undefined) { + return null; + } + + const trimmed = String(value).trim(); + return trimmed.length ? trimmed : null; +} + +function optionalInteger(value) { + if (value === null || value === undefined || value === '') { + return null; + } + + const parsed = Number(value); + if (!Number.isFinite(parsed)) { + return null; + } + + return Math.trunc(parsed); +} + +function normalizeLimit(value) { + const n = optionalInteger(value); + if (n === null) { + return DEFAULT_LIMIT; + } + + if (n < 1) { + return 1; + } + + if (n > MAX_LIMIT) { + return MAX_LIMIT; + } + + return n; +} + +function normalizeOffset(value) { + const n = optionalInteger(value); + if (n === null || n < 0) { + return 0; + } + + return n; +} + +function okResponse(requestId, data, status = 200) { + return jsonResponse( + { + ok: true, + request_id: requestId, + data, + }, + status + ); +} + +function errorResponse(requestId, status, code, message) { + return jsonResponse( + { + ok: false, + request_id: requestId, + error: { + code, + message, + }, + }, + status + ); +} + +function jsonResponse(payload, status) { + return new Response(JSON.stringify(payload), { + status, + headers: { + 'content-type': 'application/json; charset=utf-8', + 'cache-control': 'no-store', + }, + }); +} diff --git a/cloudflare/wrangler.toml b/cloudflare/wrangler.toml new file mode 100644 index 0000000..36cbc6d --- /dev/null +++ b/cloudflare/wrangler.toml @@ -0,0 +1,20 @@ +name = "heaven-edge-assets" +main = "src/index.mjs" +compatibility_date = "2026-07-09" + +[observability.logs] +enabled = true + +[[d1_databases]] +binding = "DB" +database_name = "gabriel_db" +# Set this in your environment-specific config before production deploy. +database_id = "REPLACE_WITH_D1_DATABASE_ID" + +[[r2_buckets]] +binding = "ASSETS" +bucket_name = "heaven-creator-assets" +preview_bucket_name = "heaven-creator-assets-preview" + +[vars] +MAX_SYNC_BYTES = "1073741824" diff --git a/docs/ACCESSIBLE_AUTONOMY_SETUP.md b/docs/ACCESSIBLE_AUTONOMY_SETUP.md new file mode 100644 index 0000000..27ccf16 --- /dev/null +++ b/docs/ACCESSIBLE_AUTONOMY_SETUP.md @@ -0,0 +1,66 @@ +# Accessible Autonomy Setup (Low-Typing / Voice-First) + +This setup is optimized for minimal typing and minimal pointer movement. + +## Primary operator command + +```bash +npm run autonomy:center -- 1 +``` + +Use number presets so Talon can trigger short utterances. + +## Preset map + +| Preset | Action | What it does | +| --- | --- | --- | +| `1` | observe | PR status/checks + inventory scan + receipt verify | +| `2` | triage | Pull request comments/reviews triage | +| `3` | sync-dry | Non-mutating NOIZY sync preview | +| `4` | sync-approve | Mutating sync (guarded by env approval) | +| `5` | duplicates | Duplicate scan | +| `6` | search | Search metadata/vectors | +| `7` | ship | lint + test + typecheck | +| `8` | sync-git | fetch/prune branch sync | +| `9` | foss-doctor | GoLand + JBang + FOSS toolchain health check | +| `10` | capacity-report | NOIZYVAULT_OS storage capacity + evacuation map | + +## Safety gate for mutation + +Mutating sync is blocked unless explicitly enabled: + +```bash +export NOIZY_APPROVE_MUTATION=true +npm run autonomy:center -- 4 +``` + +## Voice examples (Talon-friendly) + +- "noizy one" → `npm run autonomy:center -- 1` +- "noizy two" → `npm run autonomy:center -- 2` +- "noizy three" → `npm run autonomy:center -- 3` +- "noizy five" → `npm run autonomy:center -- 5` +- "noizy six dream chamber" → `npm run autonomy:center -- 6 dream chamber` +- "noizy seven" → `npm run autonomy:center -- 7` +- "noizy nine" → `npm run autonomy:center -- 9` +- "noizy ten" → `npm run autonomy:center -- 10` + +## IDE integration strategy + +1. Put these commands in your IDE task runner (VSCodium/Cursor/Windsurf/JetBrains terminal tasks). +2. Bind each preset to a single voice trigger or one-click task button. +3. Use preset `1` as the home dashboard command. + +## Environment defaults + +- `NOIZY_INPUT_PATH` (default `/NOIZY/raw_stems`) +- `NOIZY_CANARY_LIMIT` (default `500`) +- `NOIZY_GH_REPO` (default `GabrielAv0301/DBPredictor`) +- `NOIZY_GH_PR_NUMBER` (default `1`) + +## JetBrains + JBang + +- Run: `npm run autonomy:center -- 9` +- Direct: `npm run autonomy:foss:doctor` +- JetBrains open-source support reference: + `https://www.jetbrains.com/community/opensource/#support` diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..29f7a68 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,297 @@ +# NOIZYWORLD Architecture & Decision Rules + +> **Final Rule:** Use Firebase only as a bridge. Use FOSS for sovereignty. Use MC96 receipts as law. + +## Authority Hierarchy + +``` +MC96 / Postgres (Source of truth) + ↓ [server-side replication] +Supabase (FOSS public backend) + ↓ [app SDK] +Client Apps + ↓ [fallback] +PocketBase (field/offline) + ↓ [sync on reconnect] +Firebase (optional bridge) +``` + +### Principle + +**Authority flows downward only.** Writes flow upward only (via explicit APIs). No lateral data movement. + +- **MC96:** Source of truth for receipts, users, permissions +- **Supabase:** Public APIs, real-time coordination, app backend +- **PocketBase:** Offline-first field nodes, local sync +- **Firebase:** Optional read-only mirror for legacy clients + +--- + +## Data Ownership Rules + +### 1. Receipts (immutable records) + +| Layer | Role | Authority | Write? | +|-------|------|-----------|--------| +| **MC96** | Source | ✅ Authority | Yes (initial creation) | +| **Supabase** | Mirror | — | No (server replication) | +| **PocketBase** | Mirror | — | No (server replication) | +| **Firebase** | Bridge | — | No (read-only) | + +**Rule:** Clients never write receipts. All receipt creation goes through MC96 only. + +### 2. User profiles & permissions + +| Layer | Role | Authority | Write? | +|-------|------|-----------|--------| +| **MC96** | Source | ✅ Authority | Yes | +| **Supabase** | Cache | — | No | +| **PocketBase** | Cache | — | No | +| **Firebase** | Bridge | — | No | + +**Rule:** User permissions managed in MC96. Cached in Supabase for app performance. Never update permissions in client-facing layers. + +### 3. Approvals (client-initiated workflows) + +| Layer | Role | Authority | Write? | +|-------|------|-----------|--------| +| **MC96** | Decisions | ✅ Authority | Yes (approval/rejection) | +| **Supabase** | Intake | Semi | Yes (client creates request) | +| **PocketBase** | Local | Semi | Yes (field creates request) | +| **Firebase** | Bridge | — | Yes (if client creates) | + +**Rule:** Clients can create approval *requests* only. Server processes and marks as approved/rejected. No client updates after creation. + +### 4. Assets (file references) + +| Layer | Role | Authority | Write? | +|-------|------|-----------|--------| +| **MC96** | Metadata | ✅ Authority | Yes | +| **Supabase** | Inventory | — | No | +| **PocketBase** | Local | — | No | +| **Firebase** | Bridge | — | No | +| **Cloud Storage** | Files | — | No (signed URLs only) | + +**Rule:** Clients upload files via signed URLs. Server validates, creates asset record in MC96. Metadata replicates to Supabase/Firebase. + +--- + +## Technology Stack Decision Rules + +### When to use Supabase + +✅ **Use Supabase when:** +- You need SQL queries on relational data (receipts, lineage, audit logs) +- You want FOSS sovereignty (can self-host) +- You're building production public/private APIs +- You need real-time coordination for multi-client scenarios +- You want Row-Level Security (SQL-based authorization) + +❌ **Don't use Supabase when:** +- You only have 1-2 simple document types (use PocketBase) +- You need JavaScript-only deployment (use Firebase or Appwrite) + +### When to use PocketBase + +✅ **Use PocketBase when:** +- You're building field/offline-first nodes +- You need a single binary with no DevOps overhead +- You're embedding in embedded systems or edge devices +- You want minimal operational footprint +- Sync back to central authority on reconnection + +❌ **Don't use PocketBase when:** +- You need enterprise features (multi-region replication, backups) +- You need Postgres-specific features (JSON, full-text search) +- You have >1M records locally (SQLite scale limits) + +### When to use Appwrite + +✅ **Use Appwrite when:** +- You want all-in-one backend (auth, DB, storage, functions, messaging) +- You need email/SMS/push notifications +- You prefer Docker/Kubernetes deployments +- You have a medium-to-large team managing infrastructure + +❌ **Don't use Appwrite when:** +- You need strict relational SQL semantics (use Supabase + Postgres) +- You want absolute simplicity (use PocketBase) + +### When to use Firebase + +✅ **Use Firebase only when:** +- You're bridging legacy clients that already use Firebase +- You need Google-managed infrastructure with SLA guarantees +- You're in a scenario where operational cost matters more than sovereignty + +❌ **Don't use Firebase as primary when:** +- You need source-of-truth authority (Firestore is never truth) +- You need SQL queries (use Supabase) +- You want to avoid vendor lock-in (use FOSS) + +--- + +## Security Rules Summary + +### Firestore Security Rules (if using Firebase bridge) + +**Default:** `allow read, write: if false` (closed) + +**Exceptions:** +- `/receipts`: Read by authenticated operators; write-protected +- `/assets`: Read by authenticated operators; write-protected +- `/lineage`: Append-only audit log; read by operators; server-written +- `/approvals`: Client creates for self; server processes + +**Key principle:** No financial data is client-writable. Only approval requests (self-only). + +### Supabase Row-Level Security (SQL-based) + +```sql +ALTER TABLE receipts ENABLE ROW LEVEL SECURITY; + +CREATE POLICY "operators_read_receipts" ON receipts + FOR SELECT USING (auth.uid() IS NOT NULL); + +CREATE POLICY "server_inserts_receipts" ON receipts + FOR INSERT WITH CHECK (auth.role() = 'service_role'); + +CREATE POLICY "no_receipt_deletes" ON receipts + FOR DELETE USING (FALSE); +``` + +### Firebase MCP Configuration (hardened) + +```json +{ + "firebase": { + "command": "npx", + "args": [ + "-y", + "firebase-tools@latest", + "mcp", + "--dir", + "${FIREBASE_PROJECT_DIR}", + "--only", + "firestore,storage" + ] + } +} +``` + +**Hardening principles:** +- ✅ Include: `firestore,storage` +- ❌ Exclude: `auth` (server-side only) +- ❌ Exclude: `database` (RTDB deprecated) +- ❌ Exclude: `functions` (CI/CD only) + +--- + +## Deployment Checklist + +### Before production + +- [ ] MC96 is running with backups enabled +- [ ] Supabase replication service is running (if using) +- [ ] PocketBase field nodes are distributed (if using) +- [ ] Firestore Security Rules are deployed (if using Firebase bridge) +- [ ] MCP credentials don't include `auth` service +- [ ] All services have monitoring/alerting +- [ ] Disaster recovery plan is documented + +### Authentication & Authorization + +- [ ] MC96 is source of truth for users/roles +- [ ] Supabase RLS policies enforce MC96 permissions +- [ ] PocketBase caches user permissions from MC96 +- [ ] Firebase rules don't contradict MC96 authority +- [ ] No credentials are stored in client-facing code +- [ ] Service account keys are rotated regularly + +### Data integrity + +- [ ] Receipts are immutable (no updates after creation) +- [ ] Approvals can only be created by requestor for self +- [ ] Assets are server-validated before metadata stored +- [ ] Lineage is append-only audit trail +- [ ] All writes flow through MC96 API (never direct client writes) + +### Monitoring & Alerting + +- [ ] MC96 database size is monitored +- [ ] Replication lag (MC96 → Supabase → Firestore) is tracked +- [ ] PocketBase sync success rate is monitored +- [ ] Firestore quota usage is tracked +- [ ] Authentication failures are logged +- [ ] Unauthorized access attempts are alerted + +--- + +## Migration Runbook: Firebase → Supabase + +### Timeline: 4–8 weeks + +**Week 1-2: Setup** +- [ ] Provision Supabase project (cloud or self-hosted) +- [ ] Create Postgres tables matching Firestore schema +- [ ] Configure authentication (JWT) +- [ ] Enable Row-Level Security policies + +**Week 2-3: Data migration** +- [ ] Export Firestore data to CSV +- [ ] Validate CSV schema +- [ ] Import to Supabase Postgres +- [ ] Verify record counts and key records + +**Week 3-4: Application migration** +- [ ] Update client app SDK (Firebase → Supabase) +- [ ] Test query patterns (Firestore → PostgreSQL) +- [ ] Test real-time subscriptions +- [ ] Test offline scenarios (PocketBase if used) + +**Week 4-5: Parallel run** +- [ ] Run both Firebase and Supabase simultaneously +- [ ] All new writes → MC96 → Supabase → Firebase (mirror) +- [ ] Monitor data consistency +- [ ] Validate client functionality + +**Week 5-6: Cutover** +- [ ] Update client default endpoint to Supabase +- [ ] Keep Firebase in read-only mode as fallback +- [ ] Monitor error rates and performance +- [ ] Document any issues + +**Week 6-8: Optimization & cleanup** +- [ ] Remove Firebase SDK from clients (if no legacy users) +- [ ] Optimize Supabase indexes and caches +- [ ] Archive old Firebase data +- [ ] Update documentation + +--- + +## Cost Comparison (approximate) + +| Service | Free tier | Scaling | Notes | +|---------|-----------|---------|-------| +| **Supabase Cloud** | 500 MB DB, 2GB file storage | $25/mo → $100+/mo | Auto-scaling, managed backups | +| **Supabase Self-hosted** | Unlimited (your infra) | Your cost | Full control, FOSS | +| **PocketBase Cloud** | N/A (single binary) | Minimal | Deploy anywhere (~5-10MB memory) | +| **Firebase** | 1 GB storage, 50k reads/day | $1-2/GB | Google-managed, easier scaling | +| **Appwrite Cloud** | 5 GB storage, 10M API calls/mo | Pay-as-you-go | All-in-one | + +**Recommendation:** Start with Supabase Cloud ($25/mo) + PocketBase (free). Migrate to self-hosted Supabase as usage grows. + +--- + +## References + +- [FIREBASE_ARCHITECTURE.md](./FIREBASE_ARCHITECTURE.md) — Firebase integration details +- [FOSS_ALTERNATIVES.md](./FOSS_ALTERNATIVES.md) — FOSS comparison and setup guides +- [Firestore Security Rules](https://firebase.google.com/docs/firestore/security/get-started) +- [Supabase Docs](https://supabase.com/docs) +- [PocketBase Docs](https://pocketbase.io/docs/) +- [Appwrite Docs](https://appwrite.io/docs) + +--- + +_Last updated: 2026-07-08_ diff --git a/docs/CLOUDFLARE_DEPLOYMENT_RUNBOOK.md b/docs/CLOUDFLARE_DEPLOYMENT_RUNBOOK.md new file mode 100644 index 0000000..9e2edab --- /dev/null +++ b/docs/CLOUDFLARE_DEPLOYMENT_RUNBOOK.md @@ -0,0 +1,59 @@ +# Cloudflare Edge Deployment Runbook + +## Required bindings + +Configured in `cloudflare/wrangler.toml`: +- `DB` -> D1 database (`gabriel_db`) +- `ASSETS` -> R2 bucket + +Before deploy, replace placeholder `database_id` with the real D1 database ID. + +## Required environment variables + +```bash +export CLOUDFLARE_API_TOKEN="..." +export API_AUTH_TOKEN="..." +``` + +## One-command remote execution + +```bash +npm run cf:remote:one-command +``` + +This executes: +1. Remote D1 migration (`update_hvs_creator_assets.sql`) +2. Worker deploy with auth token injection + +## API contract summary + +All endpoints require: + +```http +Authorization: Bearer +``` + +Routes: +- `POST /api/assets/register` -> validated idempotent register/upsert. +- `GET /api/assets/search` -> filter by `public_id`, `app_name`, `file_type`, with `limit` + `offset` pagination. +- `PUT /api/assets/sync` -> stream upload to R2 with payload-size guard and checksum capture (`x-content-sha256`). + +All responses are structured: +- success: `{ ok: true, request_id, data }` +- error: `{ ok: false, request_id, error: { code, message } }` + +## Downloads ingest automation + +Stage Downloads content into canonical repo import folder: + +```bash +npm run downloads:stage-to-master +``` + +Optional move mode: + +```bash +MODE=move npm run downloads:stage-to-master +``` + +The importer uses Spotlight-indexed paths to avoid failing on interrupted directory traversal and writes detailed logs under `imports/logs/`. diff --git a/docs/FIREBASE_ARCHITECTURE.md b/docs/FIREBASE_ARCHITECTURE.md new file mode 100644 index 0000000..114e2e6 --- /dev/null +++ b/docs/FIREBASE_ARCHITECTURE.md @@ -0,0 +1,354 @@ +# Firebase Architecture & Integration Guide + +> **Principle:** Firebase/Firestore is a coordination layer and real-time mirror — **MC96 remains the source of truth. Receipts remain law.** + +## Overview + +This document describes how DBPredictor and related NOIZYWORLD systems integrate with Firebase/Firestore as an optional accelerator layer. Firebase provides: + +- **Real-time coordination** for multi-client scenarios (UI dashboards, notifications) +- **Metadata caching** to reduce latency on repeated reads +- **Offline-first fallback** when central MC96 is temporarily unavailable +- **Client-side authorization** enforcement via Firestore Security Rules + +**Critical:** Firestore is **never** the source of truth for business data. Authority flows from MC96 → Postgres → Replication → Firestore. Writes from clients are limited to self-service workflows (e.g., approval requests). + +--- + +## Collection Schema (PLOWMAN STANDARD) + +### `/receipts/{receiptId}` + +**Purpose:** Immutable financial/operational records sourced from MC96. + +**Access:** +- **Read:** Authenticated operators only +- **Write:** Server/IAM only (never client-written) + +**Typical fields:** +```json +{ + "receiptId": "string", + "batchId": "string", + "operatorUid": "string", + "status": "enum[complete, failed, pending]", + "amount": "number", + "lineItems": ["array"], + "createdAt": "timestamp", + "syncedAt": "timestamp (server-set)" +} +``` + +**Lifecycle:** +1. **MC96** creates receipt record +2. **Server cron** replicates receipt to Firestore `/receipts/{receiptId}` +3. **Clients** read receipts for display/audit; never modify + +--- + +### `/assets/{assetFingerprint}` + +**Purpose:** References to files stored in Cloud Storage (documents, photos, etc.). + +**Access:** +- **Read:** Authenticated operators only +- **Write:** Server/IAM only (never client-written) + +**Typical fields:** +```json +{ + "assetFingerprint": "string (sha256)", + "fileName": "string", + "mimeType": "string", + "storagePath": "string (gs://...)", + "receiptId": "string (foreign key)", + "sizeBytes": "number", + "uploadedAt": "timestamp", + "operator": "string (uid)" +} +``` + +**Lifecycle:** +1. **Client** uploads file to Cloud Storage via signed URL +2. **Server trigger** validates file, computes fingerprint, writes asset record to Firestore +3. **Clients** read asset metadata; never write directly + +--- + +### `/lineage/{eventId}` + +**Purpose:** Append-only event log for audit trails and state history. + +**Access:** +- **Read:** Authenticated operators only +- **Write:** Server/IAM only (immutable) + +**Typical fields:** +```json +{ + "eventId": "string (uuid)", + "receiptId": "string (foreign key)", + "operatorUid": "string", + "eventType": "enum[created, updated, approved, rejected, reconciled]", + "delta": "object (state change)", + "reason": "string", + "createdAt": "timestamp" +} +``` + +**Lifecycle:** +1. **Server** writes event whenever receipt/approval status changes +2. **Clients** read lineage for audit trail; never write + +--- + +### `/approvals/{approvalId}` + +**Purpose:** Client-initiated approval requests (e.g., expense approval). + +**Access:** +- **Read:** Operator (for self) + Server (for processing) +- **Create:** Client only (for self, with validation) +- **Update/Delete:** Server/IAM only + +**Typical fields:** +```json +{ + "approvalId": "string (uuid)", + "receiptId": "string (foreign key)", + "operatorUid": "string (requestor)", + "createdBy": "string (uid, must equal operatorUid)", + "status": "enum[draft, pending, approved, rejected]", + "requestedAt": "timestamp", + "reason": "string", + "createdAt": "timestamp (server-set)" +} +``` + +**Client create rule:** +``` +allow create: if request.auth != null + && request.resource.data.operatorUid == request.auth.uid + && request.resource.data.createdBy == request.auth.uid + && request.resource.data.status in ['pending', 'draft'] + && request.resource.data.approvalId == approvalId + && request.resource.data.receiptId is string + && request.resource.data.reason is string + && request.resource.data.reason.size() <= 1000 + && request.resource.data.keys().hasOnly([ + 'approvalId', 'receiptId', 'operatorUid', 'createdBy', 'status', 'reason', 'createdAt' + ]) +``` + +**Lifecycle:** +1. **Client** creates approval request for own UID +2. **Server** validates, may move status to `pending` or `approved` +3. **Client** polls for status change +4. **Server** never deletes; only state transitions + +--- + +## MCP (Model Context Protocol) Configuration + +Firebase MCP server provides Copilot AI with Firestore/Storage access for analysis and automation: + +```json +{ + "firebase": { + "command": "npx", + "args": [ + "-y", + "firebase-tools@latest", + "mcp", + "--dir", + "${FIREBASE_PROJECT_DIR}", + "--only", + "firestore,storage" + ] + } +} +``` + +**Hardening principles:** +- ✅ **Include:** `firestore,storage` (metadata + file references) +- ❌ **Exclude:** `auth` (admin user management stays server-side) +- ❌ **Exclude:** `database` (RTDB is deprecated; use Firestore) +- ❌ **Exclude:** `functions` (deployments are CI/CD responsibility) + +**Credential flow:** +1. MCP uses `GOOGLE_APPLICATION_CREDENTIALS` (ADC) +2. Service account should use scoped permissions (avoid `roles/firebase.admin`) +3. Server-side only; never expose credentials to client + +--- + +## Firestore Security Rules (PLOWMAN STANDARD) + +All rules default to `allow read, write: if false` — open-by-exception. + +### Key principles + +1. **Default-closed:** Every collection/document starts `deny all` +2. **Authenticated operators:** Most rules require `request.auth != null` +3. **Server-written:** `/receipts`, `/assets`, `/lineage` are write-protected +4. **Client self-service:** `/approvals` allows self-initiated requests only +5. **Immutable core:** No updates/deletes on financial records; only state transitions via server + +### Validation examples + +**Approval creation (self-only):** +``` +allow create: if request.auth != null + && request.resource.data.operatorUid == request.auth.uid + && request.resource.data.createdBy == request.auth.uid + && request.resource.data.approvalId == approvalId + && request.resource.data.receiptId is string + && request.resource.data.reason is string + && request.resource.data.reason.size() <= 1000 +``` + +**Read access (auth-only):** +``` +allow read: if request.auth != null +``` + +--- + +## Deployment & Verification + +### Deploy Firestore Rules + +```bash +# Install Firebase CLI (if not present) +npm install -g firebase-tools + +# Authenticate +firebase login + +# Test rules locally (recommended) +firebase emulators:start --only firestore + +# Deploy to production +firebase deploy --only firestore:rules +``` + +### Verify collection structure + +```bash +# List collections in Firestore (requires gcloud CLI) +gcloud firestore collections list --database='(default)' + +# View a sample receipt +gcloud firestore documents get /receipts/SAMPLE_ID --database='(default)' +``` + +--- + +## Integration with MC96 (Receipts Authority) + +### Replication workflow + +``` +MC96 (Postgres) + ↓ [server-side cron/webhook] +Firestore receipts/{receiptId} + ↓ [Realtime SDK] +Client app (read-only) +``` + +### Server-side sync (pseudocode) + +```typescript +// On MC96 receipt write (server-only) +async function replicateReceiptToFirestore(receipt: Receipt) { + const firestoreReceipt = { + receiptId: receipt.id, + batchId: receipt.batchId, + operatorUid: receipt.operatorUid, + status: receipt.status, + amount: receipt.amount, + lineItems: receipt.lineItems, + createdAt: receipt.createdAt, + syncedAt: new Date(), + }; + + await firestore + .collection('receipts') + .doc(receipt.id) + .set(firestoreReceipt, { merge: false }); +} +``` + +--- + +## Real-time coordination with PocketBase + +For offline-first field nodes, use **PocketBase** as a local-authority cache: + +``` +PocketBase (local) + ↔ [sync via API] +MC96 (central authority) + ↔ [replication] +Firestore (client mirror) +``` + +**PocketBase roles:** +- Authority for field operations (receipts created offline) +- Syncs back to MC96 when online +- Clients in field can read PocketBase, fall back to Firestore when syncing + +--- + +## Testing + +### Unit test example (Security Rules) + +```javascript +// Emulator setup +const db = initializeTestEnvironment({ + projectId: 'test-project', +}); + +// Test: authenticated operator can read receipts +await assertSucceeds( + db.authenticatedContext({ uid: 'operator-1' }) + .firestore() + .collection('receipts') + .doc('receipt-1') + .get() +); + +// Test: client cannot create receipt +await assertFails( + db.authenticatedContext({ uid: 'operator-1' }) + .firestore() + .collection('receipts') + .doc('new-receipt') + .set({ /* ... */ }) +); +``` + +--- + +## Troubleshooting + +| Issue | Cause | Fix | +|-------|-------|-----| +| Client read fails with "Permission denied" | Wrong UID or no `request.auth` | Verify user is authenticated; check Firestore rules | +| Server write timeout | Firestore quota exceeded | Check billing, enable Firestore usage notifications | +| Realtime updates don't sync | Network/offline | Add exponential backoff retry in client SDK | +| MCP cannot access Firestore | ADC not set or invalid | Export `GOOGLE_APPLICATION_CREDENTIALS` to service account JSON | + +--- + +## References + +- [Firebase Security Rules documentation](https://firebase.google.com/docs/firestore/security/get-started) +- [Firestore best practices](https://firebase.google.com/docs/firestore/best-practices) +- [Firebase MCP Server](https://firebase.google.com/docs/ai-assistance/mcp-server) +- [Cloud Storage Security Rules](https://firebase.google.com/docs/storage/security) + +--- + +_Last updated: 2026-07-08_ diff --git a/docs/FIREBASE_HARDENING_PLAN.md b/docs/FIREBASE_HARDENING_PLAN.md new file mode 100644 index 0000000..06bcb6f --- /dev/null +++ b/docs/FIREBASE_HARDENING_PLAN.md @@ -0,0 +1,118 @@ +# Firebase Hardening & FOSS Architecture — Implementation Plan + +> **Doctrine:** Firebase/Firestore can be a mirror or coordination layer — **MC96 remains authority, receipts remain truth**. + +--- + +## Task 1: Harden Firebase MCP Configuration + +**Objective:** Restrict Firebase MCP to essential services only (`firestore,storage`). Keep `auth` out unless actively needed. + +**Why:** Firebase Auth admin tools are powerful and should be managed server-side only. Exposing only Firestore/Storage limits the attack surface while maintaining metadata + asset reference capabilities. + +### Deliverables + +- ✅ Create `.mcp/firebase.json` with hardened MCP config +- ✅ Document in `FIREBASE_ARCHITECTURE.md` + +**Status:** Completed (implemented in `.mcp/firebase.json`) + +--- + +## Task 2: Firestore "PLOWMAN STANDARD" Security Rules + +**Objective:** Implement industry-standard Firestore Security Rules (default-closed, role-based access). + +**Doctrine:** Firebase/Firestore is a coordination layer, never the source of truth. MC96/Postgres is authority. + +### Proposed Collection Structure + +``` +/firestore + /receipts/{receiptId} # Read by authenticated operator only. Never client-written. + /assets/{assetFingerprint} # Read by authenticated operator. Writes only via server/IAM. + /lineage/{eventId} # Event history. Operator-readable, server-written. + /approvals/{approvalId} # Approval requests. Client may create for self only. +``` + +### Rules Features + +- **Default-closed:** All documents are `allow read, write: if false` by default. +- **Authenticated-only read:** `/receipts`, `/assets`, `/lineage` readable by authenticated operators. +- **Server-only writes:** Core data written only by server/IAM, never client. +- **Client self-service:** `/approvals` allows operators to create requests for their own UID. + +### Deliverables + +- ✅ Create `firestore.rules` with PLOWMAN STANDARD starter rules +- ✅ Document collection schema and access patterns +- ✅ Add to `FIREBASE_ARCHITECTURE.md` + +**Status:** Completed (implemented in `firestore.rules`) + +--- + +## Task 3: FOSS Upgrade Path & Architecture + +**Objective:** Document preferred FOSS alternatives (Supabase, Appwrite, PocketBase) and establish a sovereign long-term architecture. + +### Recommended NOIZYWORLD Layout + +``` +MC96 / Postgres = source of truth +Supabase = FOSS public/private app backend +PocketBase = local/offline field node +Firebase = optional compatibility mirror only +Firestore = never sole authority +``` + +### FOSS Comparison + +| Role | Best FOSS | Why | +|------|-----------|-----| +| Firebase-style backend | **Supabase** | Open-source Firebase alternative with Postgres, Auth, Data APIs, Edge Functions, Realtime, Storage, Vector support | +| All-in-one app backend | **Appwrite** | Open-source platform with Auth, Databases, Storage, Functions, Messaging, Realtime, web hosting | +| Local/offline backend | **PocketBase** | Open-source backend in one file with realtime database, authentication, file storage, admin dashboard | + +### Deliverables + +- ✅ Create `FOSS_ALTERNATIVES.md` with detailed comparison and migration paths +- ✅ Document `NOIZYWORLD` preferred architecture +- ✅ Create `FIREBASE_ARCHITECTURE.md` with integration guidelines +- ✅ Add decision rules to `ARCHITECTURE.md` + +**Status:** Completed (documented in `docs/FOSS_ALTERNATIVES.md` and `docs/ARCHITECTURE.md`) + +--- + +## Implementation Timeline + +1. **Phase A:** Create `.mcp/firebase.json` (hardened MCP config) +2. **Phase B:** Create `firestore.rules` (PLOWMAN STANDARD security rules) +3. **Phase C:** Create `FIREBASE_ARCHITECTURE.md` (Firebase integration guide) +4. **Phase D:** Create `FOSS_ALTERNATIVES.md` (FOSS comparison and migration paths) +5. **Phase E:** Update main `ARCHITECTURE.md` with decision rules + +--- + +## Files to Create + +- `/.mcp/firebase.json` — Hardened MCP configuration +- `/firestore.rules` — PLOWMAN STANDARD security rules +- `/docs/FIREBASE_ARCHITECTURE.md` — Firebase integration and deployment guide +- `/docs/FOSS_ALTERNATIVES.md` — FOSS comparison and migration paths +- Update `/docs/ARCHITECTURE.md` or create it with decision rules + +--- + +## Key Principles + +1. **MC96 is authority:** Receipts, user data, permissions live in MC96/Postgres first. +2. **Firebase is a mirror:** Firestore/Storage replicate metadata for faster client access. +3. **FOSS is sovereignty:** Use Supabase for long-term, self-hosted resilience. +4. **PocketBase for field:** Offline-first field nodes sync back to central authority. +5. **Minimum privilege:** MCP exposes only what's needed; Auth stays server-side. + +--- + +_Last updated: 2026-07-08T23:45_ diff --git a/docs/FOSS_ALTERNATIVES.md b/docs/FOSS_ALTERNATIVES.md new file mode 100644 index 0000000..27eacff --- /dev/null +++ b/docs/FOSS_ALTERNATIVES.md @@ -0,0 +1,432 @@ +# FOSS Alternatives to Firebase: Architecture & Migration Paths + +> **Final Rule:** Use Firebase only as a bridge. Use FOSS for sovereignty. Use MC96 receipts as law. + +## Executive Summary + +Firebase is a managed platform operated by Google. For **FOSS sovereignty**, use one or more of these alternatives as your primary backend: + +| Role | Best FOSS option | Deployment | Best for | +|------|------------------|-----------|---------| +| **Firebase-style backend** | **Supabase** | Self-hosted or cloud | Public/private APIs, auth, real-time DB, edge functions | +| **All-in-one backend** | **Appwrite** | Self-hosted or cloud | Auth, DB, storage, functions, messaging in one platform | +| **Local/offline backend** | **PocketBase** | Single binary | Field ops, offline-first nodes, embedded systems | + +--- + +## Recommended NOIZYWORLD Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ MC96 / Postgres │ +│ (Source of truth for receipts) │ +└────┬────────────────────────────────────────────────────────────┘ + │ [replication + auth delegation] + ├─────────────────────────────────────────┬──────────────────┐ + ↓ ↓ ↓ +┌─────────────┐ ┌──────────────┐ ┌────────────┐ +│ Supabase │ (FOSS public backend) │ PocketBase │ │ Firebase │ +│ ─────────── │ │ (optional) │ │ (optional) │ +│ • Auth │ │ ──────────── │ │ ──────────┐│ +│ • Postgres │ │ • Local DB │ │ • Mirror ││ +│ • APIs │ │ • Auth │ │ • Bridge ││ +│ • Real-time │ │ • Sync │ │ ││ +│ • Storage │ │ • Offline │ │ ││ +│ • Functions │ │ │ │ ││ +└─────────────┘ └──────────────┘ └────────────┘ + ↑ ↑ ↑ + │ [FOSS app backend] [field nodes sync] [client realtime] + │ + Client Apps (web, mobile) +``` + +### Data flow + +1. **MC96 → Postgres:** Receipts, users, approvals stored in MC96 (source of truth) +2. **MC96 → Supabase:** Replicate public data (metadata, non-sensitive refs) for app backend +3. **MC96 → PocketBase:** Sync for field nodes (offline-first workflows) +4. **MC96 → Firebase:** Optional sync for existing client apps (bridge-only) +5. **Clients:** Read from Supabase/PocketBase (FOSS); optionally fallback to Firebase + +--- + +## Deep Dive: Supabase (Recommended) + +### What is Supabase? + +**Supabase** is an open-source Firebase alternative that provides: +- **PostgreSQL database** (unlike Firebase's NoSQL) +- **JWT authentication** (self-managed) +- **RESTful + Real-time APIs** (via PostgREST and Realtime) +- **Edge Functions** (serverless compute) +- **File Storage** (similar to Firebase Storage) +- **Vector support** (pgvector extension for embeddings) +- **Row-Level Security (RLS)** (SQL-based authorization) + +**Open source?** Yes — [github.com/supabase/supabase](https://github.com/supabase/supabase) + +### Why Supabase over Firebase? + +| Feature | Supabase | Firebase | +|---------|----------|----------| +| **Database** | PostgreSQL (relational) | Firestore (document NoSQL) | +| **Pricing** | Pay-as-you-go or self-host free | Google-managed pricing | +| **Source of truth** | Own your data (self-host) | Google's infrastructure | +| **SQL access** | Full SQL + PostgREST | Limited query language | +| **Real-time** | Realtime API (websockets) | Firestore listeners | +| **Auth** | JWT + social + SAML | Firebase Auth (Google-managed) | +| **Functions** | Edge Functions (Deno) | Cloud Functions (proprietary) | +| **Exit cost** | Low (export Postgres) | High (Firestore export limited) | + +### Deployment options + +#### Option A: Supabase Cloud (managed) +```bash +# Sign up at https://app.supabase.com +# Connect your app SDK +npm install @supabase/supabase-js + +# Usage +const supabase = createClient( + 'https://.supabase.co', + '' +); + +const { data } = await supabase + .from('receipts') + .select('*') + .eq('operatorUid', userId); +``` + +#### Option B: Self-hosted Supabase (sovereign) +```bash +# Clone Supabase repo +git clone https://github.com/supabase/supabase.git +cd supabase/docker + +# Customize docker-compose.yml with your secrets +docker-compose up + +# Access at http://localhost:3000 +``` + +### Supabase Security Rules (Row-Level Security) + +```sql +-- Enable RLS on receipts table +ALTER TABLE receipts ENABLE ROW LEVEL SECURITY; + +-- Policy: authenticated users can read all receipts +CREATE POLICY "Operators can read receipts" + ON receipts + FOR SELECT + USING (auth.uid() IS NOT NULL); + +-- Policy: only server (service role) can insert +CREATE POLICY "Server inserts receipts" + ON receipts + FOR INSERT + WITH CHECK (auth.role() = 'service_role'); + +-- Policy: no deletes for anyone +CREATE POLICY "No deletes" + ON receipts + FOR DELETE + USING (FALSE); +``` + +### Migration path: Firebase → Supabase + +``` +Step 1: Set up Supabase project + → Create PostgreSQL tables (receipts, assets, approvals) + → Configure authentication + → Enable RLS policies + +Step 2: Replicate data + → Export Firestore → CSV + → Import CSV → Supabase Postgres + → Verify record counts + +Step 3: Update client SDK + → Replace Firebase SDK with Supabase SDK + → Update queries (SQL-style via PostgREST) + → Test offline scenarios + +Step 4: Monitor & sync + → Run parallel read-write for period + → Validate data consistency + → Cutover to Supabase, keep Firebase in read-only bridge mode +``` + +--- + +## Deep Dive: Appwrite (All-in-one) + +### What is Appwrite? + +**Appwrite** is an open-source backend platform offering: +- **Databases** (hybrid SQL/document) +- **Authentication** (email, OAuth, SAML, 2FA) +- **Storage** (file + object) +- **Cloud Functions** (serverless) +- **Messaging** (email, SMS, push) +- **Real-time subscriptions** +- **Web hosting** + +**Open source?** Yes — [github.com/appwrite/appwrite](https://github.com/appwrite/appwrite) + +### Why Appwrite? + +Best for teams wanting an **all-in-one, batteries-included backend** without managing separate Postgres, Redis, etc. + +| Feature | Appwrite | Supabase | Firebase | +|---------|----------|----------|----------| +| **Auth** | Built-in (email, OAuth, 2FA) | Built-in | Built-in | +| **Database** | SQL + Document | PostgreSQL | Firestore | +| **Storage** | File + object | Object | Cloud Storage | +| **Functions** | Cloud Functions (runtimes) | Edge Functions | Cloud Functions | +| **Messaging** | Email, SMS, push | N/A | Cloud Messaging | +| **Orchestration** | Workflows | N/A | Workflows | +| **Install** | Docker, single binary | Docker, multi-service | Managed only | + +### Deployment + +```bash +# Via Docker +docker run -d \ + -e _APP_OPENSSL_KEY_V1=$(openssl rand -hex 32) \ + -e _APP_ENV=production \ + appwrite/appwrite + +# Accessible at http://localhost/console +``` + +### Appwrite for receipts workflow + +```typescript +import { Client, Databases, Query } from 'appwrite'; + +const client = new Client() + .setEndpoint('https://appwrite.example.com/v1') + .setProject('project-id'); + +const db = new Databases(client); + +// Server-side: insert receipt +await db.createDocument('default', 'receipts', 'receipt-1', { + receiptId: 'receipt-1', + operatorUid: 'user-123', + status: 'complete', + amount: 99.99, + createdAt: new Date(), +}); + +// Client-side: read receipt +const { documents } = await db.listDocuments('default', 'receipts', [ + Query.equal('operatorUid', 'user-123'), +]); +``` + +--- + +## Deep Dive: PocketBase (Local/Offline) + +### What is PocketBase? + +**PocketBase** is a single-file open-source backend with: +- **SQLite database** (file-based) +- **Built-in admin UI** +- **Authentication** (email + OAuth) +- **File storage** +- **Realtime subscriptions** +- **Single binary** (~20 MB) + +**Open source?** Yes — [github.com/pocketbase/pocketbase](https://github.com/pocketbase/pocketbase) + +**Best for:** +- Field operations (offline-first) +- Local development +- Embedded systems +- Small-to-medium teams +- Nodes that sync back to central authority + +### Why PocketBase for field nodes? + +``` +Field agent (offline) + ↓ +PocketBase (local SQLite) + ↓ [when online] +MC96 (via API) + ↓ +Supabase (replication) +``` + +### Installation & deployment + +```bash +# Download latest binary from https://github.com/pocketbase/pocketbase/releases +./pocketbase serve + +# Admin UI at http://localhost:8090/_/ +``` + +### PocketBase collection schema (receipts) + +```javascript +// Create collection via Admin UI or API +{ + name: "receipts", + type: "base", + schema: [ + { + id: "receiptId", + name: "receiptId", + type: "text", + required: true, + unique: true, + }, + { + id: "operatorUid", + name: "operatorUid", + type: "text", + required: true, + }, + { + id: "status", + name: "status", + type: "select", + options: ["draft", "submitted", "complete", "failed"], + }, + { + id: "amount", + name: "amount", + type: "number", + required: true, + }, + { + id: "createdAt", + name: "createdAt", + type: "date", + required: true, + }, + ], + rules: { + create: "auth.id != \"\"", // authenticated users only + update: "false", // no updates + delete: "false", // no deletes + read: "auth.id != \"\"", // authenticated users read + }, +} +``` + +### Client SDK usage + +```typescript +import PocketBase from 'pocketbase'; + +const pb = new PocketBase('http://localhost:8090'); + +// Authenticate +const userData = await pb.collection('users').authWithPassword( + 'user@example.com', + 'password' +); + +// Read receipts (offline-first) +const receipts = await pb.collection('receipts').getFullList({ + filter: `operatorUid="${pb.authStore.model.id}"`, +}); + +// Real-time sync +pb.collection('receipts').subscribe('*', (e) => { + console.log('Receipt updated:', e.record); +}); +``` + +--- + +## Comparison Matrix + +| Criterion | Supabase | Appwrite | PocketBase | Firebase | +|-----------|----------|----------|-----------|----------| +| **Open source** | ✅ Yes | ✅ Yes | ✅ Yes | ❌ No | +| **Self-hosted** | ✅ Yes | ✅ Yes | ✅ Yes | ❌ Managed only | +| **Database** | PostgreSQL | SQL/Doc hybrid | SQLite | Firestore NoSQL | +| **Real-time** | ✅ Realtime API | ✅ Realtime | ✅ Subscriptions | ✅ Listeners | +| **Auth** | JWT + social | Email + OAuth + SAML | Email + OAuth | Firebase Auth | +| **Storage** | ✅ File | ✅ File + object | ✅ File | ✅ Cloud Storage | +| **Functions** | Edge Functions | Cloud Functions | N/A | Cloud Functions | +| **Messaging** | N/A | Email, SMS, push | N/A | Cloud Messaging | +| **Single binary** | ❌ Multi-service | ✅ Docker | ✅ Binary | N/A | +| **Exit cost** | Low | Low | Low | High | +| **Best use** | Production FOSS backend | All-in-one backend | Field/offline nodes | Managed Google ecosystem | + +--- + +## Migration Strategy: MC96 + FOSS + Firebase + +### Phase 1: Establish MC96 as authority (NOW) +- All receipts, users, approvals live in MC96 Postgres +- MC96 is source of truth; no reads/writes bypass it + +### Phase 2: Add Supabase as FOSS backend (NEXT) +- Replicate MC96 data to Supabase Postgres +- Update client apps to read from Supabase (via SDK) +- Maintain Firebase in read-only sync for existing users + +### Phase 3: Add PocketBase for field (CONCURRENT) +- Deploy PocketBase to field nodes +- Field agents sync receipts to PocketBase offline +- On reconnection, sync back to MC96 → Supabase + +### Phase 4: Deprecate Firebase (OPTIONAL) +- Keep Firebase sync running as bridge for legacy clients +- New clients use Supabase primary, PocketBase secondary +- Gradually migrate remaining Firebase users + +### Phase 5: Sovereign FOSS stack (LONG-TERM) +``` +MC96 (receipts authority) + ↔ Supabase (public APIs, auth, real-time) + ↔ PocketBase (field sync, offline) + ← [optional] Firebase (read-only bridge) +``` + +--- + +## Decision Tree: Which FOSS to use? + +``` +Do you need PostgreSQL data for SQL queries? +├─ YES → Supabase (recommended for most production scenarios) +└─ NO → Continue... + +Do you need everything in one platform (auth, DB, storage, functions, messaging)? +├─ YES → Appwrite +└─ NO → Continue... + +Are you running a field/embedded/offline-first system? +├─ YES → PocketBase (single binary, lightweight) +└─ NO → Supabase or Appwrite + +Are you constrained by operational overhead (operations team)? +├─ SMALL TEAM → PocketBase (simplest to operate) +├─ MEDIUM TEAM → Supabase (proven, well-documented) +└─ LARGE TEAM → Appwrite (enterprise features) +``` + +--- + +## References + +- **Supabase:** [supabase.com](https://supabase.com), [GitHub](https://github.com/supabase/supabase) +- **Appwrite:** [appwrite.io](https://appwrite.io), [GitHub](https://github.com/appwrite/appwrite) +- **PocketBase:** [pocketbase.io](https://pocketbase.io), [GitHub](https://github.com/pocketbase/pocketbase) +- **PostgreSQL:** [postgresql.org](https://postgresql.org) +- **SQLite:** [sqlite.org](https://sqlite.org) + +--- + +_Last updated: 2026-07-08_ diff --git a/docs/FOSS_GOLAND_JBANG_STACK.md b/docs/FOSS_GOLAND_JBANG_STACK.md new file mode 100644 index 0000000..5d4d3e9 --- /dev/null +++ b/docs/FOSS_GOLAND_JBANG_STACK.md @@ -0,0 +1,35 @@ +# FOSS Operator Stack: GoLand + JBang + Local AI + +This stack keeps operations local-first while adding JetBrains + Java automation support. + +## Core layout + +1. **Primary local runtimes**: Go, Java, Python, Node. +2. **IDE options**: GoLand, IntelliJ IDEA Community, VSCodium, Fleet. +3. **Automation runner**: JBang for Java CLI jobs and utility tasks. +4. **Control plane**: DesktopCommander + n8n + NOIZY Autonomy Center. +5. **Governance**: MC96 receipts, no destructive sync, metadata mirror only. + +## One-command checks + +```bash +npm run autonomy:foss:doctor +``` + +or: + +```bash +npm run autonomy:center -- 9 +``` + +## Minimal voice flow + +1. `npm run autonomy:center -- 1` (dashboard) +2. `npm run autonomy:center -- 3` (dry-run sync) +3. `npm run autonomy:center -- 5` (duplicates) +4. `npm run autonomy:center -- 7` (ship gate) +5. `npm run autonomy:center -- 9` (FOSS/JetBrains/JBang readiness) + +## JetBrains open-source support + +https://www.jetbrains.com/community/opensource/#support diff --git a/docs/HEAVEN_EDGE_API_BLUEPRINT.txt b/docs/HEAVEN_EDGE_API_BLUEPRINT.txt new file mode 100644 index 0000000..476c827 --- /dev/null +++ b/docs/HEAVEN_EDGE_API_BLUEPRINT.txt @@ -0,0 +1,161 @@ +HEAVEN Worker asset API blueprint +================================= + +Use this route module in your Cloudflare Worker deployment to register/search/sync creator assets against D1 + R2. +Production Worker implementation is now in: +- `cloudflare/src/index.mjs` +- `cloudflare/wrangler.toml` + +Before running remote D1 migrations: +- export `CLOUDFLARE_API_TOKEN` with D1 edit permissions +- export `API_AUTH_TOKEN` +- run `npm run cf:remote:one-command` + +```javascript +export default { + async fetch(request, env) { + const url = new URL(request.url); + const path = url.pathname; + + if (request.method === 'POST' && path === '/api/assets/register') { + return registerAsset(request, env); + } + + if (request.method === 'GET' && path === '/api/assets/search') { + return searchAssets(request, env); + } + + if (request.method === 'PUT' && path === '/api/assets/sync') { + return syncAsset(request, env); + } + + return json({ error: 'Not found' }, 404); + }, +}; + +async function registerAsset(request, env) { + const body = await request.json(); + + const required = ['public_id', 'asset_name', 'app_category', 'app_name', 'file_type']; + for (const key of required) { + if (!body[key]) { + return json({ error: `Missing required field: ${key}` }, 400); + } + } + + const stmt = env.DB.prepare( + `INSERT INTO hvs_creator_assets + (public_id, asset_name, app_category, app_name, file_type, r2_object_key, local_path_blueprint) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + ON CONFLICT(public_id) DO UPDATE SET + asset_name = excluded.asset_name, + app_category = excluded.app_category, + app_name = excluded.app_name, + file_type = excluded.file_type, + r2_object_key = excluded.r2_object_key, + local_path_blueprint = excluded.local_path_blueprint, + updated_at = CURRENT_TIMESTAMP` + ); + + await stmt + .bind( + body.public_id, + body.asset_name, + body.app_category, + body.app_name, + body.file_type, + body.r2_object_key ?? null, + body.local_path_blueprint ?? null + ) + .run(); + + return json({ ok: true, public_id: body.public_id }, 200); +} + +async function searchAssets(request, env) { + const url = new URL(request.url); + const appName = url.searchParams.get('app_name'); + const fileType = url.searchParams.get('file_type'); + const limit = Number(url.searchParams.get('limit') ?? '100'); + + if (!appName && !fileType) { + return json({ error: 'Provide app_name or file_type' }, 400); + } + + let sql = + 'SELECT asset_id, public_id, asset_name, app_category, app_name, file_type, r2_object_key, local_path_blueprint, created_at, updated_at FROM hvs_creator_assets WHERE 1=1'; + const binds = []; + + if (appName) { + sql += ' AND app_name = ?'; + binds.push(appName); + } + + if (fileType) { + sql += ' AND file_type = ?'; + binds.push(fileType); + } + + sql += ' ORDER BY updated_at DESC LIMIT ?'; + binds.push(Math.min(Math.max(limit, 1), 500)); + + const { results } = await env.DB.prepare(sql).bind(...binds).all(); + return json({ ok: true, count: results.length, results }, 200); +} + +async function syncAsset(request, env) { + const url = new URL(request.url); + const mode = url.searchParams.get('mode') ?? 'stream'; + const objectKey = url.searchParams.get('object_key'); + const publicId = url.searchParams.get('public_id'); + + if (!objectKey || !publicId) { + return json({ error: 'object_key and public_id are required' }, 400); + } + + // High-speed direct stream path for heavy creative assets. + if (mode === 'stream') { + if (!request.body) { + return json({ error: 'Request body is required for stream mode' }, 400); + } + + await env.ASSETS.put(objectKey, request.body, { + httpMetadata: { contentType: request.headers.get('content-type') ?? 'application/octet-stream' }, + }); + + await env.DB.prepare( + `UPDATE hvs_creator_assets + SET r2_object_key = ?1, updated_at = CURRENT_TIMESTAMP + WHERE public_id = ?2` + ) + .bind(objectKey, publicId) + .run(); + + return json({ ok: true, mode: 'stream', object_key: objectKey }, 200); + } + + // Placeholder mode for external pre-signed URL pipeline. + if (mode === 'presign') { + return json( + { + ok: false, + error: 'presign mode blueprint only: implement SigV4 signing endpoint for your external uploader', + }, + 501 + ); + } + + return json({ error: 'Unsupported sync mode' }, 400); +} + +function json(data, status = 200) { + return new Response(JSON.stringify(data), { + status, + headers: { 'content-type': 'application/json; charset=utf-8' }, + }); +} +``` + +Expected Worker bindings: +- `DB` -> D1 binding +- `ASSETS` -> R2 bucket binding diff --git a/docs/HOTROD_GIT_LEADER_MODE.md b/docs/HOTROD_GIT_LEADER_MODE.md new file mode 100644 index 0000000..2958ba6 --- /dev/null +++ b/docs/HOTROD_GIT_LEADER_MODE.md @@ -0,0 +1,37 @@ +# HOT ROD Git Leader Mode + +This gives you a single-command operator flow for GitHub + GitKraken/GitLens usage when you want the agent to lead. + +## One command + +```bash +npm run git:hotrod +``` + +## Modes + +```bash +# status board +npm run git:hotrod -- status + +# review/comment triage +npm run git:hotrod -- triage + +# pre-ship quality gate +npm run git:hotrod -- ship + +# fetch/prune/branch sync +npm run git:hotrod -- sync +``` + +## How this maps to your tools + +- **GitHub (`gh`)**: source of truth for PR state/checks/comments +- **GitKraken/GitLens**: visual graph, commit grouping, merge context +- **Leader mode**: run hotrod script first, then execute only the next action it surfaces + +## Operator doctrine + +1. One command. +2. One next action. +3. One receipt-level outcome. diff --git a/docs/MCP_CONFIG.md b/docs/MCP_CONFIG.md new file mode 100644 index 0000000..edef3d6 --- /dev/null +++ b/docs/MCP_CONFIG.md @@ -0,0 +1,354 @@ +# Firebase MCP (Model Context Protocol) Configuration Guide + +> **Security First:** Firebase MCP exposes APIs for Copilot AI agents. Hardening prevents unauthorized access to auth, functions, and other sensitive services. + +## Quick Start + +### 1. Copy the hardened config + +```json +{ + "firebase": { + "command": "npx", + "args": [ + "-y", + "firebase-tools@latest", + "mcp", + "--dir", + "${FIREBASE_PROJECT_DIR}", + "--only", + "firestore,storage" + ] + } +} +``` + +**File location:** `.mcp/firebase.json` (in your project root) + +### 2. Set environment variables + +```bash +# Required for MCP to access Firebase +export FIREBASE_PROJECT_DIR="/path/to/firebase/project" +export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json" +``` + +### 3. Verify the setup + +```bash +# Test MCP connection (requires firebase-tools installed) +npx firebase-tools@latest mcp \ + --dir "$FIREBASE_PROJECT_DIR" \ + --only firestore,storage +``` + +--- + +## Configuration Reference + +### Hardened config: What's included? + +| Service | Included | Why | Security | +|---------|----------|-----|----------| +| **firestore** | ✅ Yes | Metadata queries, collection browsing | Safe (RLS enforced) | +| **storage** | ✅ Yes | File inventory, asset references | Safe (signed URLs only) | +| **auth** | ❌ No | User management, password ops | 🔴 DANGEROUS if exposed | +| **database** | ❌ No | Realtime DB (deprecated) | Not needed | +| **functions** | ❌ No | Deployment, execution | 🔴 CI/CD only | +| **remoteconfig** | ❌ No | Feature flags | Not needed | +| **hosting** | ❌ No | Website deployment | Not needed | + +### Why exclude `auth`? + +Firebase Auth admin tools are **too powerful for MCP**: + +```typescript +// ❌ BAD: These operations exposed to MCP +getUser(uid: string) // Read user info +deleteUser(uid: string) // Delete user account +updateUser(uid: string, props) // Modify user properties +createUser(properties) // Create new user +verifyIdToken(token: string) // Token validation +revokeRefreshTokens(uid: string) // Force re-authentication +``` + +**Risk:** An AI agent with auth access could: +- Delete user accounts +- Modify permissions +- Create backdoor users +- Revoke sessions + +**Solution:** Keep auth server-side only. MCP reads user metadata from `/approvals` or `/profiles` (Firestore) instead. + +--- + +## Usage Examples + +### Example 1: Query receipts collection + +```python +# Copilot AI agent using Firebase MCP +import anthropic + +client = anthropic.Anthropic() + +# Via MCP, agent can browse Firestore collections +response = client.messages.create( + model="claude-3-5-sonnet-20241022", + max_tokens=1024, + tools=[ + { + "type": "computer_use", + "name": "query_firestore", + "description": "Query Firestore documents", + } + ], + messages=[ + { + "role": "user", + "content": "How many receipts were created today?" + } + ] +) +``` + +The agent can use MCP to: +- List collections in Firestore +- Count documents in `/receipts` +- Query by date range using Firestore queries +- Extract statistics (total amount, operator counts, etc.) + +### Example 2: List assets and file references + +```typescript +// Copilot using MCP to inventory Cloud Storage files +const query = ` + List all assets in the /assets collection, + grouped by receiptId. + For each asset, show fileName, storagePath, and fileSize. + Create a CSV export. +`; +``` + +**MCP output:** +```csv +receiptId,fileName,storagePath,fileSize +receipt-123,invoice.pdf,gs://bucket/assets/abc123,45KB +receipt-124,receipt.png,gs://bucket/assets/def456,120KB +``` + +--- + +## Deployment + +### Option A: Google Cloud environment (recommended) + +```bash +# Set up Application Default Credentials +gcloud auth application-default login + +# Or use service account +export GOOGLE_APPLICATION_CREDENTIALS="./service-account.json" + +# Start MCP server +npx firebase-tools@latest mcp \ + --dir ./firebase \ + --only firestore,storage +``` + +### Option B: Docker deployment + +```dockerfile +FROM node:18-slim + +RUN npm install -g firebase-tools + +ENV FIREBASE_PROJECT_DIR=/app/firebase +ENV GOOGLE_APPLICATION_CREDENTIALS=/run/secrets/firebase_sa + +COPY .mcp/firebase.json /app/.mcp/firebase.json +COPY firebase /app/firebase + +CMD ["npx", "firebase-tools@latest", "mcp", \ + "--dir", "${FIREBASE_PROJECT_DIR}", \ + "--only", "firestore,storage"] +``` + +### Option C: GitHub Actions workflow + +```yaml +name: Firebase MCP Sync + +on: + schedule: + - cron: '0 */6 * * *' # Every 6 hours + workflow_dispatch: + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: '18' + + - name: Authenticate with Firebase + env: + GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.FIREBASE_SA_JSON }} + run: | + echo "$GOOGLE_APPLICATION_CREDENTIALS" > sa-key.json + export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/sa-key.json + + - name: Run MCP server + run: | + npx firebase-tools@latest mcp \ + --dir ./firebase \ + --only firestore,storage +``` + +--- + +## Troubleshooting + +### Issue: "Authentication failed" + +**Cause:** `GOOGLE_APPLICATION_CREDENTIALS` not set or invalid. + +**Fix:** +```bash +# Create service account key from Google Cloud Console +gcloud iam service-accounts keys create sa-key.json \ + --iam-account=firebase-mcp@PROJECT_ID.iam.gserviceaccount.com + +export GOOGLE_APPLICATION_CREDENTIALS="$(pwd)/sa-key.json" + +# Verify +npx firebase-tools@latest mcp --help +``` + +### Issue: "Firestore not found" + +**Cause:** Firebase project doesn't have Firestore enabled. + +**Fix:** +```bash +# Enable Firestore in your Firebase project +firebase init firestore + +# Or via Cloud Console +# → Firebase → Firestore Database → Create Database +``` + +### Issue: "Permission denied" reading collections + +**Cause:** Service account lacks permissions. + +**Fix:** +```bash +# Grant scoped roles to service account (least privilege) +gcloud projects add-iam-policy-binding PROJECT_ID \ + --member=serviceAccount:firebase-mcp@PROJECT_ID.iam.gserviceaccount.com \ + --role=roles/datastore.user + +gcloud projects add-iam-policy-binding PROJECT_ID \ + --member=serviceAccount:firebase-mcp@PROJECT_ID.iam.gserviceaccount.com \ + --role=roles/storage.objectViewer +``` + +--- + +## Security Best Practices + +### 1. Service account permissions (least privilege) + +❌ **Don't:** Use `roles/firebase.admin` (too broad) + +✅ **Do:** Use scoped roles: +```bash +gcloud projects add-iam-policy-binding PROJECT_ID \ + --member=serviceAccount:firebase-mcp@PROJECT_ID.iam.gserviceaccount.com \ + --role=roles/datastore.user # Firestore read/write + +gcloud projects add-iam-policy-binding PROJECT_ID \ + --member=serviceAccount:firebase-mcp@PROJECT_ID.iam.gserviceaccount.com \ + --role=roles/storage.objectViewer # Cloud Storage read +``` + +### 2. Credentials management + +❌ **Don't:** +- Commit service account keys to git +- Use API keys (public credentials) +- Share credentials across environments + +✅ **Do:** +- Store keys in: + - Google Cloud Secret Manager + - GitHub Secrets (for CI/CD) + - 1Password or equivalent +- Rotate keys every 90 days +- Use different keys per environment + +### 3. Firestore Security Rules + +``` +/firestore + /receipts → Read-only (never client-writable) + /assets → Read-only (server-replicated) + /lineage → Append-only (audit log) + /approvals → Client creates for self, server processes +``` + +**Rule:** MCP can read these collections, but shouldn't need to write (writes are server-side only). + +### 4. API auditing + +Enable Cloud Audit Logs to track MCP access: + +```bash +# Via gcloud +gcloud logging sinks create firebase-mcp-logs \ + logging.googleapis.com/projects/PROJECT_ID/logs/cloudaudit.googleapis.com \ + --log-filter='resource.type="gce_instance" AND protoPayload.serviceName="firestore.googleapis.com"' +``` + +--- + +## Integration with Copilot CLI + +### In Desktop Commander or Copilot CLI + +1. **Copy MCP config to user MCP directory:** + ```bash + mkdir -p ~/.mcp + cp .mcp/firebase.json ~/.mcp/firebase.json + ``` + +2. **Set environment:** + ```bash + export FIREBASE_PROJECT_DIR="/path/to/firebase" + export GOOGLE_APPLICATION_CREDENTIALS="/path/to/sa.json" + ``` + +3. **Verify MCP is discoverable:** + ```bash + # Copilot CLI will auto-discover MCP configs + # And make Firebase available to AI agents + ``` + +--- + +## References + +- [Firebase MCP Server Documentation](https://firebase.google.com/docs/ai-assistance/mcp-server) +- [MCP Specification](https://spec.modelcontextprotocol.io/) +- [Firebase CLI Reference](https://firebase.google.com/docs/cli) +- [Google Cloud IAM Roles](https://cloud.google.com/iam/docs/understanding-roles) +- [Firestore Security Rules](https://firebase.google.com/docs/firestore/security/get-started) + +--- + +_Last updated: 2026-07-08_ diff --git a/docs/TALON_CONTEXT_BLUEPRINT.talon b/docs/TALON_CONTEXT_BLUEPRINT.talon new file mode 100644 index 0000000..4a0f3ad --- /dev/null +++ b/docs/TALON_CONTEXT_BLUEPRINT.talon @@ -0,0 +1,47 @@ +# Talon voice context blueprint for creator workflows + +app: keynote +add slide: key(cmd-shift-n) +format text: key(cmd-t) +align center: key(cmd-shift-e) + +app: numbers +new sheet: key(shift-f11) +insert column: key(ctrl-i c) +format text: key(cmd-t) +align center: key(cmd-shift-e) + +app: pages +add page: key(cmd-enter) +format text: key(cmd-t) +align center: key(cmd-shift-e) + +app: logic pro +split clip: key(cmd-t) +play timeline: key(space) +bounce project: key(cmd-b) +start render: key(cmd-b) + +app: final cut pro +split clip: key(cmd-b) +blade tool: key(b) +play timeline: key(space) +start render: key(cmd-e) + +app: motion +play timeline: key(space) +start render: key(cmd-e) + +app: compressor +start render: key(cmd-enter) + +app: home assistant +trigger routine {user.routine_name}: user.trigger_home_assistant_routine(routine_name) + +app: canary mail +next email: key(j) +archive message: key(e) + +app: google chat +next chat: key(j) +send chat: key(cmd-enter) diff --git a/firestore.rules b/firestore.rules new file mode 100644 index 0000000..a3c3e0d --- /dev/null +++ b/firestore.rules @@ -0,0 +1,98 @@ +rules_version = '2'; + +service cloud.firestore { + match /databases/{database}/documents { + function isAuthenticated() { + return request.auth != null; + } + + function isOwner(uidField) { + return isAuthenticated() && uidField == request.auth.uid; + } + + function hasOnlyApprovalKeys() { + return request.resource.data.keys().hasOnly([ + 'approvalId', + 'receiptId', + 'operatorUid', + 'createdBy', + 'status', + 'reason', + 'createdAt' + ]); + } + + function isValidApprovalCreate() { + return hasOnlyApprovalKeys() + && request.resource.data.approvalId is string + && request.resource.data.receiptId is string + && request.resource.data.operatorUid is string + && request.resource.data.createdBy is string + && request.resource.data.status in ['pending', 'draft'] + && request.resource.data.reason is string + && request.resource.data.reason.size() <= 1000 + && request.resource.data.createdAt == request.time; + } + + // Default closed: deny all access by default + match /{document=**} { + allow read, write: if false; + } + + // Receipts: readable by authenticated operators, server-written only + // Receipts are immutable records from MC96 — never modified or deleted by clients + match /receipts/{receiptId} { + allow read: if isAuthenticated(); + allow create, update, delete: if false; + + match /{allSubcollections=**} { + allow read: if isAuthenticated(); + allow write: if false; + } + } + + // Assets: readable by authenticated operators, server/IAM-written only + // Assets reference files stored in Cloud Storage — never directly modified by clients + match /assets/{assetFingerprint} { + allow read: if isAuthenticated(); + allow create, update, delete: if false; + + match /{allSubcollections=**} { + allow read: if isAuthenticated(); + allow write: if false; + } + } + + // Lineage: event history, readable by operators, server-written only + // Immutable append-only event log for audit trails + match /lineage/{eventId} { + allow read: if isAuthenticated(); + allow create, update, delete: if false; + + match /{allSubcollections=**} { + allow read: if isAuthenticated(); + allow write: if false; + } + } + + // Approvals: clients may create approval requests for themselves + // Clients read their own approvals; servers process and may mark as complete + match /approvals/{approvalId} { + allow read: if isOwner(resource.data.operatorUid) + || isOwner(resource.data.createdBy); + + allow create: if isValidApprovalCreate() + && request.resource.data.approvalId == approvalId + && request.resource.data.operatorUid == request.auth.uid + && request.resource.data.createdBy == request.auth.uid + && request.resource.data.operatorUid == request.resource.data.createdBy; + + allow update, delete: if false; + + match /{allSubcollections=**} { + allow read: if isAuthenticated(); + allow write: if false; + } + } + } +} diff --git a/noizy b/noizy new file mode 100755 index 0000000..480dd84 --- /dev/null +++ b/noizy @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec python3 "$ROOT_DIR/noizy-audio-rag/noizy_sync.py" "$@" diff --git a/noizy-army/NOIZYARMY_MANIFEST.md b/noizy-army/NOIZYARMY_MANIFEST.md new file mode 100644 index 0000000..45b25d2 --- /dev/null +++ b/noizy-army/NOIZYARMY_MANIFEST.md @@ -0,0 +1,59 @@ +# NOIZYARMY — 24/7/365 Autonomous Agent Army + +## DEPLOY IN ONE COMMAND + +```bash +./noizy-army/scripts/noizy-army-install.sh +``` + +## ARMY ROSTER + +| Soldier | Role | Runs | How | +|---|---|---|---| +| **WATCHDOG** | Process supervisor, heartbeat, nightly scans | Always | LaunchAgent (auto-restart) | +| **OLLAMA** | Local LLM runtime (Qwen/Llama/etc) | Always | LaunchAgent (KEEP_ALIVE=-1) | +| **MCP GATEWAY** | Firebase/DesktopCommander MCP keepalive | On-demand | mcp-gateway-server.sh | +| **BEE GABRIEL** | Commercial library researcher | Background task | noizy-army task agents | +| **BEE LUCY** | Personal career catalog mapper | Background task | noizy-army task agents | +| **BEE DREAM** | Dedup + consolidation strategist | Background task | noizy-army task agents | + +## NIGHTLY AUTONOMOUS OPERATIONS + +| Time (UTC) | Operation | Script | +|---|---|---| +| 03:00 | Capacity report + receipt emit | noizy-capacity-report.sh | +| 04:00 | Inventory scan on NOIZY_POOL_A | noizy scan | +| Continuous | Ollama heartbeat check | noizy-watchdog.sh | +| Continuous | n8n health check | noizy-watchdog.sh | + +## STATUS CHECK + +```bash +npm run autonomy:center -- army-status +# or directly: +./noizy-army/scripts/noizy-army-status.sh +``` + +## AUTONOMY MODES + +- **OBSERVE**: watchdog only, no writes +- **RECOMMEND**: scan + report, no mutations +- **EXECUTE**: full pipeline, receipts required +- **CONTINUOUS**: n8n scheduled loop + +## DOCTRINE + +> Raw assets stay local. +> Metadata syncs. +> Vectors mirror. +> Receipts rule. +> MC96 governs. +> NOIZYARMY never sleeps. + +## CURRENT CRITICAL STATE (as of 2026-07-09) + +- NOIZY_POOL_B: **100% FULL** — evacuation required +- 3TB-GRF: **91% FULL** — reduce immediately +- BEE GABRIEL: researching commercial library catalog (running) +- BEE LUCY: mapping 40-year personal career catalog (running) +- BEE DREAM: building dedup + consolidation strategy (running) diff --git a/noizy-army/launchagents/com.noizy.ollama.plist b/noizy-army/launchagents/com.noizy.ollama.plist new file mode 100644 index 0000000..4c32986 --- /dev/null +++ b/noizy-army/launchagents/com.noizy.ollama.plist @@ -0,0 +1,36 @@ + + + + + Label + com.noizy.ollama + + ProgramArguments + + /opt/homebrew/bin/ollama + serve + + + RunAtLoad + + + KeepAlive + + + StandardOutPath + /Users/m2ultra/.noizyarmy/logs/ollama.stdout.log + + StandardErrorPath + /Users/m2ultra/.noizyarmy/logs/ollama.stderr.log + + EnvironmentVariables + + OLLAMA_KEEP_ALIVE + -1 + OLLAMA_NUM_PARALLEL + 4 + OLLAMA_MAX_LOADED_MODELS + 3 + + + diff --git a/noizy-army/launchagents/com.noizy.watchdog.plist b/noizy-army/launchagents/com.noizy.watchdog.plist new file mode 100644 index 0000000..7782629 --- /dev/null +++ b/noizy-army/launchagents/com.noizy.watchdog.plist @@ -0,0 +1,39 @@ + + + + + Label + com.noizy.watchdog + + ProgramArguments + + /bin/bash + REPLACE_WITH_REPO_PATH/noizy-army/scripts/noizy-watchdog.sh + + + RunAtLoad + + + KeepAlive + + + StandardOutPath + /Users/m2ultra/.noizyarmy/logs/watchdog.stdout.log + + StandardErrorPath + /Users/m2ultra/.noizyarmy/logs/watchdog.stderr.log + + EnvironmentVariables + + NOIZY_WATCHDOG_INTERVAL + 300 + NOIZY_INPUT_PATH + /Volumes/NOIZY_POOL_A/NOIZY_SAMPLE_MASTER + PATH + /opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin:/Users/m2ultra/.nvm/versions/node/v24.17.0/bin + + + WorkingDirectory + REPLACE_WITH_REPO_PATH + + diff --git a/noizy-army/mcp/mcp-gateway-server.sh b/noizy-army/mcp/mcp-gateway-server.sh new file mode 100755 index 0000000..f9e5cc5 --- /dev/null +++ b/noizy-army/mcp/mcp-gateway-server.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# NOIZY MCP GATEWAY — keeps all MCP servers alive and routes requests +set -euo pipefail + +LOG_DIR="$HOME/.noizyarmy/logs" +mkdir -p "$LOG_DIR" + +ts() { date -u +"%Y-%m-%dT%H:%M:%SZ"; } +log() { echo "[$(ts)] [MCP-GW] $*" | tee -a "$LOG_DIR/mcp-gateway.log"; } + +# MCP servers to keep alive +declare -A MCP_SERVERS=( + ["firebase"]="npx -y firebase-tools@latest mcp --only firestore,storage" + ["desktop-commander"]="npx -y @desktop-commander/mcp-server" +) + +start_mcp() { + local name="$1" + local cmd="${MCP_SERVERS[$name]}" + local pidfile="$LOG_DIR/mcp-$name.pid" + + if [[ -f "$pidfile" ]] && kill -0 "$(cat "$pidfile")" 2>/dev/null; then + log "$name: already running (pid $(cat $pidfile))" + return + fi + + log "$name: starting — $cmd" + eval "$cmd >> $LOG_DIR/mcp-$name.log 2>&1 &" + echo $! > "$pidfile" + log "$name: started (pid $!)" +} + +stop_mcp() { + local name="$1" + local pidfile="$LOG_DIR/mcp-$name.pid" + if [[ -f "$pidfile" ]]; then + local pid + pid=$(cat "$pidfile") + kill "$pid" 2>/dev/null && log "$name: stopped" || log "$name: already dead" + rm -f "$pidfile" + fi +} + +case "${1:-start}" in + start) + log "Starting all MCP servers" + for srv in "${!MCP_SERVERS[@]}"; do start_mcp "$srv"; done + log "All MCP servers started" + ;; + stop) + for srv in "${!MCP_SERVERS[@]}"; do stop_mcp "$srv"; done + ;; + status) + for srv in "${!MCP_SERVERS[@]}"; do + local pidfile="$LOG_DIR/mcp-$srv.pid" + if [[ -f "$pidfile" ]] && kill -0 "$(cat "$pidfile")" 2>/dev/null; then + echo " $srv: RUNNING (pid $(cat $pidfile))" + else + echo " $srv: DOWN" + fi + done + ;; + restart) + for srv in "${!MCP_SERVERS[@]}"; do stop_mcp "$srv"; done + sleep 2 + for srv in "${!MCP_SERVERS[@]}"; do start_mcp "$srv"; done + ;; +esac diff --git a/noizy-army/scripts/noizy-army-install.sh b/noizy-army/scripts/noizy-army-install.sh new file mode 100755 index 0000000..32cff16 --- /dev/null +++ b/noizy-army/scripts/noizy-army-install.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# NOIZYARMY — one-command install (runs at login, auto-restarts, 24/7) +set -euo pipefail + +REPO_PATH="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +LAUNCH_AGENTS_DIR="$HOME/Library/LaunchAgents" +ARMY_PLISTS_DIR="$REPO_PATH/noizy-army/launchagents" + +mkdir -p "$HOME/.noizyarmy/logs" +mkdir -p "$LAUNCH_AGENTS_DIR" + +echo "== NOIZYARMY INSTALL ==" +echo "repo: $REPO_PATH" + +install_agent() { + local template="$1" + local label="$2" + local dest="$LAUNCH_AGENTS_DIR/$label.plist" + + sed "s|REPLACE_WITH_REPO_PATH|$REPO_PATH|g" "$template" > "$dest" + + launchctl unload "$dest" 2>/dev/null || true + launchctl load -w "$dest" + echo " LOADED: $label" +} + +install_agent "$ARMY_PLISTS_DIR/com.noizy.watchdog.plist" "com.noizy.watchdog" +install_agent "$ARMY_PLISTS_DIR/com.noizy.ollama.plist" "com.noizy.ollama" + +echo +echo "== NOIZYARMY STATUS ==" +launchctl list | grep noizy || echo "(no noizy agents running yet)" + +echo +echo "Logs: $HOME/.noizyarmy/logs/" +echo "Done. NOIZYARMY is live 24/7/365." diff --git a/noizy-army/scripts/noizy-army-status.sh b/noizy-army/scripts/noizy-army-status.sh new file mode 100755 index 0000000..9a37ac2 --- /dev/null +++ b/noizy-army/scripts/noizy-army-status.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail +LOG_DIR="$HOME/.noizyarmy/logs" +echo "== NOIZYARMY STATUS ==" +echo +echo "-- LaunchAgents --" +launchctl list 2>/dev/null | grep -E "noizy|ollama" || echo " none loaded" +echo +echo "-- Ollama --" +curl -sf http://127.0.0.1:11434/api/tags 2>/dev/null | python3 -c "import sys,json; d=json.load(sys.stdin); print(' models:', [m['name'] for m in d.get('models',[])])" 2>/dev/null || echo " offline" +echo +echo "-- Watchdog log (last 10) --" +[[ -f "$LOG_DIR/watchdog.log" ]] && tail -10 "$LOG_DIR/watchdog.log" || echo " no log yet" +echo +echo "-- Receipts (last 5) --" +[[ -f "$LOG_DIR/../../../noizy-audio-rag/mc96/receipts.log" ]] && tail -5 "$LOG_DIR/../../../noizy-audio-rag/mc96/receipts.log" || true diff --git a/noizy-army/scripts/noizy-watchdog.sh b/noizy-army/scripts/noizy-watchdog.sh new file mode 100755 index 0000000..b3f2376 --- /dev/null +++ b/noizy-army/scripts/noizy-watchdog.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# NOIZYARMY WATCHDOG — 24/7/365 process supervisor +set -euo pipefail + +ARMY_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +LOG_DIR="$HOME/.noizyarmy/logs" +RECEIPTS="$ARMY_ROOT/noizy-audio-rag/mc96/receipts.log" +INTERVAL="${NOIZY_WATCHDOG_INTERVAL:-300}" # 5 min default + +mkdir -p "$LOG_DIR" + +ts() { date -u +"%Y-%m-%dT%H:%M:%SZ"; } +log() { echo "[$(ts)] [WATCHDOG] $*" | tee -a "$LOG_DIR/watchdog.log"; } +receipt() { printf '{"receipt_id":"%s","verb":"%s","timestamp":"%s","operator":"WATCHDOG"}\n' \ + "$(printf '%s' "$1:$(ts)" | shasum -a 256 | cut -c1-24)" "$1" "$(ts)" >> "$RECEIPTS"; } + +check_ollama() { + if curl -sf http://127.0.0.1:11434/api/tags >/dev/null 2>&1; then + log "ollama: OK" + else + log "ollama: DOWN — restarting" + ollama serve >> "$LOG_DIR/ollama.log" 2>&1 & + sleep 3 + receipt "ollama_restart" + fi +} + +check_n8n() { + if curl -sf http://127.0.0.1:5678/healthz >/dev/null 2>&1; then + log "n8n: OK" + else + log "n8n: DOWN — attempting restart" + receipt "n8n_down_detected" + fi +} + +nightly_capacity() { + local hour + hour=$(date +%H) + local marker="$LOG_DIR/.capacity_$(date +%Y%m%d)" + if [[ "$hour" -eq 3 && ! -f "$marker" ]]; then + log "Running nightly capacity report" + (cd "$ARMY_ROOT" && ./scripts/noizy-capacity-report.sh >> "$LOG_DIR/capacity_$(date +%Y%m%d).log" 2>&1) + touch "$marker" + receipt "nightly_capacity_report" + fi +} + +nightly_scan() { + local hour + hour=$(date +%H) + local marker="$LOG_DIR/.scan_$(date +%Y%m%d)" + if [[ "$hour" -eq 4 && ! -f "$marker" ]]; then + log "Running nightly inventory scan" + local input="${NOIZY_INPUT_PATH:-/NOIZY/raw_stems}" + [[ -d "$input" ]] && (cd "$ARMY_ROOT" && ./noizy scan --input "$input" >> "$LOG_DIR/scan_$(date +%Y%m%d).log" 2>&1) || true + touch "$marker" + receipt "nightly_scan" + fi +} + +log "NOIZYARMY WATCHDOG STARTED — interval=${INTERVAL}s" +receipt "watchdog_start" + +while true; do + check_ollama + check_n8n + nightly_capacity + nightly_scan + log "Heartbeat OK — sleeping ${INTERVAL}s" + receipt "heartbeat" + sleep "$INTERVAL" +done diff --git a/noizy-audio-rag/.env.example b/noizy-audio-rag/.env.example new file mode 100644 index 0000000..27286ff --- /dev/null +++ b/noizy-audio-rag/.env.example @@ -0,0 +1,6 @@ +NOIZY_RAW_STEMS_DIR=/NOIZY/raw_stems +NOIZY_PAPYRUS_DB=./papyrus/papyrus.db +NOIZY_FAISS_INDEX=./embeddings/noizy_audio.index +NOIZY_FAISS_MAP=./embeddings/noizy_audio.map.json +NOIZY_NOTIFY_WEBHOOK_URL= +NOIZY_FIRESTORE_COLLECTION=noizy_audio_assets diff --git a/noizy-audio-rag/README.md b/noizy-audio-rag/README.md new file mode 100644 index 0000000..a433bde --- /dev/null +++ b/noizy-audio-rag/README.md @@ -0,0 +1,141 @@ +# noizy-audio-rag + +Audio RAG scaffold for NOIZY workflows: + +- ingest stems and metadata into Papyrus (SQLite) +- fingerprint audio references +- build/query embeddings and FAISS index +- mirror searchable metadata to Firestore +- connect orchestration through n8n and MC96 receipt stubs + +## Local stack + +- VSCodium +- Continue +- Ollama +- MC96 +- Local MCP Mesh +- Supersonic NOIZYBEAST IDE + +## IDE layer + +- VSCodium +- Continue +- Terminal +- GitKraken +- Local MCP Dashboard + +## Quick start + +1. Copy `.env.example` to `.env` and fill values. +2. Initialize Papyrus schema: + - `sqlite3 papyrus/papyrus.db < papyrus/schema.sql` +3. Run ingest/embedding scripts as needed. +4. Run voice-first operator presets: + - `npm run autonomy:center -- 1` + +## Accessible autonomy + +- See `docs/ACCESSIBLE_AUTONOMY_SETUP.md` for low-typing and Talon-friendly operation. +- See `docs/FOSS_GOLAND_JBANG_STACK.md` for GoLand + JBang FOSS operator setup. +- VSCodium task presets are available in `.vscode/tasks.json`. + +## Layout + +- `config/audio_rag.yaml`: pipeline settings +- `config/agent_modes.yaml`: autonomous runtime modes and guardrails +- `config/mcp_adapter_contracts.yaml`: MCP adapter I/O contracts and policy +- `config/local_apps_mesh.yaml`: local app integration mesh profile +- `papyrus/`: schema + local SQLite database +- `ingest/`: ingest + fingerprint + metadata writers +- `embeddings/`: embedding/index/query scripts +- `firestore/`: metadata mirroring notes/scripts +- `n8n/`: automation workflow export +- `mc96/`: receipt integration stubs + +## Sync flow + +Downloads +↓ +INBOX +↓ +QUARANTINE +↓ +VALIDATED +↓ +CANONICAL + +## Governance execution chain + +MC96 +governs +↓ +NOIZYBEAST IDE +↓ +Agents +↓ +Storage +↓ +Search +↓ +Action + +## Operating doctrine + +- Raw Assets Stay Local +- Metadata Syncs +- Vectors Mirror +- Receipts Rule +- MC96 Governs + +## Long-horizon mandate + +- 40 years retention horizon +- 34 TB scale target +- artist sovereignty +- voice sovereignty +- local-first +- AI-native +- future-proof + +## Chief Architect remit + +- schema +- standards +- strategy +- design + +## Research Director remit + +- deep search +- knowledge +- documentation + +## Operations focus + +- deployment +- migration +- D1 +- R2 +- Cloudflare + +## Archivist remit + +- preserve catalogs +- maintain receipt history +- enforce lineage continuity +- catalog +- metadata +- classification + +## Papyrus core + +Papyrus +│ +├── SQLite +├── FAISS +├── Receipt Ledger +├── Asset Registry +├── Metadata Registry +├── Consent Registry +└── Lineage Registry diff --git a/noizy-audio-rag/config/agent_modes.yaml b/noizy-audio-rag/config/agent_modes.yaml new file mode 100644 index 0000000..0165e8b --- /dev/null +++ b/noizy-audio-rag/config/agent_modes.yaml @@ -0,0 +1,78 @@ +runtime: + control_plane: DesktopCommander + governance: MC96 + receipt_engine: Papyrus + default_mode: recommend + mode_transition_requires_receipt: true + +modes: + observe: + description: Read-only telemetry and discovery. + allows: + - search + - scan + - metadata_lookup + - receipt_verify + denies: + - writes + - deploy + - sync_mutation + recommend: + description: Build plans and proposals without storage mutation. + allows: + - observe + - plan_generation + - risk_assessment + - migration_preview + denies: + - writes + - delete + execute: + description: Mutating mode for approved actions. + requires: + - approval_token + - signed_request + - rate_limit_budget + allows: + - ingest + - embeddings + - faiss_build + - firestore_mirror_metadata + - receipt_emit + denies: + - raw_audio_remote_upload + - destructive_sync + - delete + continuous: + description: Scheduled/event-driven execution loop using n8n. + requires: + - execute_mode_requirements + - schedule_policy + - failure_stop_policy + allows: + - scheduled_scan + - webhook_sync + - pipeline_retries_non_destructive + +approval: + gatekeeper: MC96 + policy: + can_recommend: true + cannot_mutate_without_approval: true + no_overwrite_without_approval: true + +failure_policy: + fail_fast: true + stage_stop_on_error: true + emit_failure_receipt: true + notify_channels: + - MCP.Discord + - MCP.Matrix + +doctrine: + - one_command_one_action_one_receipt + - raw_assets_stay_local + - metadata_syncs + - vectors_mirror + - receipts_rule + - mc96_governs diff --git a/noizy-audio-rag/config/audio_rag.yaml b/noizy-audio-rag/config/audio_rag.yaml new file mode 100644 index 0000000..e0e4a8a --- /dev/null +++ b/noizy-audio-rag/config/audio_rag.yaml @@ -0,0 +1,90 @@ +pipeline: + input_dir: /NOIZY/raw_stems + canary_limit: 500 + dry_run_default: true + rollout_focus: + - deployment + - migration + - D1 + - R2 + - Cloudflare + horizon: + retention_years: 40 + storage_target_tb: 34 + principles: + - artist sovereignty + - voice sovereignty + - local-first + - AI-native + - future-proof + +storage: + local_faiss_primary: true + r2_bucket: r2://voice-dna-storage/ + r2_layout: sha256/{aa}/{bb}/{sha256}{ext} + firestore_metadata_only: true + vectors_mirror_enabled: true + raw_assets_remote_upload: false + +rules: + no_deletes: true + no_destructive_sync: true + no_overwrite_without_approval: true + no_duplicates: true + immutable_references: true + receipts_rule: true + mc96_governs: true + +firestore: + collections: + - assets + - receipts + - consent_records + - lineage + - sync_state + - quality_reports + - duplicate_groups + +mcp_mesh: + channels: + - MCP.Storage + - MCP.Receipts + - MCP.Papyrus + - MCP.Firestore + - MCP.D1 + - MCP.R2 + - MCP.HotRod + - MCP.Git + - MCP.Discord + - MCP.Matrix + +agents: + chief_architect: GABRIEL + research_director: LUCY + operations_commander: HEAVEN + auditor: KEITH + archivist: SHIRL + catalog: SHIRL + builder: NOIZYBEAST + execution: NOIZYBEAST + +research_scope: + - deep search + - knowledge + - documentation + +catalog_scope: + - catalog + - metadata + - classification + +auditor_scope: + - receipts + - lineage + - duplicate audits + - integrity checks + +builder_scope: + - execution + - automation + - pipelines diff --git a/noizy-audio-rag/config/local_apps_mesh.yaml b/noizy-audio-rag/config/local_apps_mesh.yaml new file mode 100644 index 0000000..b9961d1 --- /dev/null +++ b/noizy-audio-rag/config/local_apps_mesh.yaml @@ -0,0 +1,47 @@ +local_apps_mesh: + ide_layer: + - VSCodium + - GoLand + - IntelliJ IDEA Community + - JetBrains Fleet + - Continue + - Terminal + - GitKraken + - Local MCP Dashboard + - Supersonic NOIZYBEAST IDE + + model_runtime: + primary: Ollama + secondary: LM Studio + policy: + local_only_inference: true + model_downloads_allowed: true + + knowledge_layer: + - TagSpaces + - DevonThink + - DevonAgentPro + - DevonSphere + + orchestration: + operator_plane: DesktopCommander + automation: n8n + java_runner: JBang + webhook: /noizy/sync + schedule: + daily_scan_enabled: true + +integration_contract: + inputs: + - request_id + - intent + - target + outputs: + - action_summary + - receipt_id + - timestamp + guarantees: + - no_delete + - no_destructive_sync + - local_first + - metadata_only_mirror diff --git a/noizy-audio-rag/config/mcp_adapter_contracts.yaml b/noizy-audio-rag/config/mcp_adapter_contracts.yaml new file mode 100644 index 0000000..f87771c --- /dev/null +++ b/noizy-audio-rag/config/mcp_adapter_contracts.yaml @@ -0,0 +1,133 @@ +adapters: + MCP.Storage: + purpose: Local file access and staging orchestration + input_contract: + - request_id + - action + - path + output_contract: + - status + - receipt_id + constraints: + - no_delete + - no_overwrite_without_approval + + MCP.Receipts: + purpose: Receipt write/verify/sign pipeline + input_contract: + - request_id + - stage + - payload_sha256 + - operator + output_contract: + - receipt_id + - timestamp + - signature + constraints: + - append_only + + MCP.Papyrus: + purpose: Papyrus SQLite and registry operations + input_contract: + - request_id + - table + - operation + - data + output_contract: + - rows_affected + - receipt_id + constraints: + - no_destructive_sync + + MCP.Firestore: + purpose: Metadata/vector mirror for global discovery + input_contract: + - request_id + - collection + - metadata_payload + output_contract: + - mirrored_count + - receipt_id + constraints: + - metadata_only + - no_raw_audio + + MCP.D1: + purpose: Cloudflare D1 mirror/state operations + input_contract: + - request_id + - sql_operation + - migration_id + output_contract: + - result + - receipt_id + constraints: + - migrations_signed + + MCP.R2: + purpose: Object key registration and mirror metadata + input_contract: + - request_id + - object_key + - sha256 + output_contract: + - object_ref + - receipt_id + constraints: + - immutable_sha256_paths + - no_raw_asset_transfer_without_policy_override + + MCP.HotRod: + purpose: Accelerated embedding/index pipeline handoff + input_contract: + - request_id + - asset_id + - embedding_job + output_contract: + - faiss_update + - receipt_id + + MCP.Git: + purpose: Versioned change tracking for configs/workflows + input_contract: + - request_id + - change_set + - message + output_contract: + - commit_sha + - receipt_id + + MCP.Discord: + purpose: Operator notifications + input_contract: + - request_id + - summary + - severity + output_contract: + - delivered + - receipt_id + + MCP.Matrix: + purpose: Sovereign chat notification fallback + input_contract: + - request_id + - summary + - severity + output_contract: + - delivered + - receipt_id + +cross_adapter_policy: + routing_chain: + - Agents + - MCP + - Receipt + - Storage + required_headers: + - x-request-id + - x-receipt-signature + rate_limit: + per_agent_per_minute: 120 + audit_logging: + enabled: true + sink: Papyrus.stage_receipts diff --git a/noizy-audio-rag/embeddings/build_embeddings.py b/noizy-audio-rag/embeddings/build_embeddings.py new file mode 100644 index 0000000..2f94036 --- /dev/null +++ b/noizy-audio-rag/embeddings/build_embeddings.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +import hashlib +import json +import sqlite3 +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DB = ROOT / 'papyrus' / 'papyrus.db' +OUT = ROOT / 'embeddings' / 'embeddings.jsonl' + + +def fake_embedding(seed: str, dims: int = 16) -> list[float]: + digest = hashlib.sha256(seed.encode('utf-8')).digest() + return [round((digest[i % len(digest)] / 255.0), 6) for i in range(dims)] + + +def main() -> None: + rows = [] + with sqlite3.connect(DB) as conn: + for public_id, sha256 in conn.execute('SELECT public_id, sha256 FROM local_assets ORDER BY created_at DESC LIMIT 500'): + rows.append({'asset_id': public_id, 'sha256': sha256, 'embedding': fake_embedding(sha256)}) + + OUT.parent.mkdir(parents=True, exist_ok=True) + with OUT.open('w', encoding='utf-8') as f: + for row in rows: + f.write(json.dumps(row) + '\n') + print(json.dumps({'written': len(rows), 'path': str(OUT)})) + + +if __name__ == '__main__': + main() diff --git a/noizy-audio-rag/embeddings/build_faiss.py b/noizy-audio-rag/embeddings/build_faiss.py new file mode 100644 index 0000000..f34a722 --- /dev/null +++ b/noizy-audio-rag/embeddings/build_faiss.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SRC = ROOT / 'embeddings' / 'embeddings.jsonl' +OUT_INDEX = ROOT / 'embeddings' / 'noizy_audio.index' +OUT_MAP = ROOT / 'embeddings' / 'noizy_audio.map.json' + + +def main() -> None: + records = [] + if SRC.exists(): + with SRC.open('r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if not line: + continue + records.append(json.loads(line)) + + # Local-first placeholder index artifact. + OUT_INDEX.write_text('FAISS_PLACEHOLDER_INDEX\n', encoding='utf-8') + OUT_MAP.write_text(json.dumps({'count': len(records), 'ids': [r['asset_id'] for r in records]}, indent=2), encoding='utf-8') + print(json.dumps({'indexed': len(records), 'index': str(OUT_INDEX), 'map': str(OUT_MAP)})) + + +if __name__ == '__main__': + main() diff --git a/noizy-audio-rag/embeddings/query_faiss.py b/noizy-audio-rag/embeddings/query_faiss.py new file mode 100644 index 0000000..2a25ee8 --- /dev/null +++ b/noizy-audio-rag/embeddings/query_faiss.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MAP = ROOT / 'embeddings' / 'noizy_audio.map.json' + + +def main() -> None: + query = sys.argv[1] if len(sys.argv) > 1 else 'audio' + if not MAP.exists(): + print(json.dumps({'query': query, 'results': [], 'reason': 'index map missing'})) + return + data = json.loads(MAP.read_text(encoding='utf-8')) + ids = data.get('ids', [])[:10] + print(json.dumps({'query': query, 'results': [{'asset_id': i, 'score': 0.9} for i in ids]}, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/noizy-audio-rag/firestore/firestore_vector_notes.md b/noizy-audio-rag/firestore/firestore_vector_notes.md new file mode 100644 index 0000000..1bf69cd --- /dev/null +++ b/noizy-audio-rag/firestore/firestore_vector_notes.md @@ -0,0 +1,36 @@ +# Firestore vector mirror notes + +Rules: + +- Metadata only. +- No raw stems. +- No full WAVs. +- No sensitive masters. +- No delete operations. + +Canonical payload: + +```json +{ + "asset_id": "", + "sha256": "", + "tags": [], + "embedding": [], + "quality_score": 0, + "owner": "" +} +``` + +Collections: + +Firestore +│ +├── metadata +├── asset cards +├── vectors +├── approvals +├── sync state +├── quality reports +├── duplicate groups +├── agents +└── work orders diff --git a/noizy-audio-rag/firestore/mirror_metadata.py b/noizy-audio-rag/firestore/mirror_metadata.py new file mode 100644 index 0000000..69d4f1e --- /dev/null +++ b/noizy-audio-rag/firestore/mirror_metadata.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +import json +import sqlite3 +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DB = ROOT / 'papyrus' / 'papyrus.db' +OUT = ROOT / 'firestore' / 'mirror_payload.json' + + +def build_payload(limit: int = 500) -> list[dict]: + rows = [] + with sqlite3.connect(DB) as conn: + query = """ + SELECT public_id, sha256 + FROM local_assets + ORDER BY created_at DESC + LIMIT ? + """ + for public_id, sha256 in conn.execute(query, (limit,)): + rows.append( + { + 'asset_id': public_id, + 'sha256': sha256, + 'tags': [], + 'embedding': [], + 'quality_score': 0, + 'owner': 'RSP', + } + ) + return rows + + +if __name__ == '__main__': + payload = build_payload() + OUT.write_text(json.dumps({'assets': payload, 'metadata_only': True}, indent=2), encoding='utf-8') + print(json.dumps({'mirrored_count': len(payload), 'path': str(OUT)})) diff --git a/noizy-audio-rag/ingest/drive_inventory.py b/noizy-audio-rag/ingest/drive_inventory.py new file mode 100644 index 0000000..cc6a957 --- /dev/null +++ b/noizy-audio-rag/ingest/drive_inventory.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +import json +from pathlib import Path + +ROOT = Path('/NOIZY/raw_stems') + + +def main() -> None: + files = 0 + bytes_total = 0 + exts: dict[str, int] = {} + + if ROOT.exists(): + for path in ROOT.rglob('*'): + if not path.is_file(): + continue + files += 1 + try: + size = path.stat().st_size + except OSError: + size = 0 + bytes_total += size + ext = path.suffix.lower() or '[none]' + exts[ext] = exts.get(ext, 0) + 1 + + print( + json.dumps( + { + 'root': str(ROOT), + 'files': files, + 'bytes_total': bytes_total, + 'top_extensions': sorted(exts.items(), key=lambda x: x[1], reverse=True)[:20], + }, + indent=2, + ) + ) + + +if __name__ == '__main__': + main() diff --git a/noizy-audio-rag/ingest/fingerprint_audio.py b/noizy-audio-rag/ingest/fingerprint_audio.py new file mode 100644 index 0000000..dd7527e --- /dev/null +++ b/noizy-audio-rag/ingest/fingerprint_audio.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +import hashlib +import json +from pathlib import Path + + +def fingerprint(path: Path) -> dict: + h = hashlib.sha256() + with path.open('rb') as f: + for chunk in iter(lambda: f.read(1024 * 1024), b''): + h.update(chunk) + return {'path': str(path), 'sha256': h.hexdigest(), 'size_bytes': path.stat().st_size} + + +if __name__ == '__main__': + target = Path('/NOIZY/raw_stems') + rows = [] + if target.exists(): + for p in sorted(target.rglob('*')): + if p.is_file(): + rows.append(fingerprint(p)) + if len(rows) >= 20: + break + print(json.dumps(rows, indent=2)) diff --git a/noizy-audio-rag/ingest/ingest_stems.py b/noizy-audio-rag/ingest/ingest_stems.py new file mode 100644 index 0000000..aafb467 --- /dev/null +++ b/noizy-audio-rag/ingest/ingest_stems.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +from pathlib import Path +import json + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_INPUT = Path('/NOIZY/raw_stems') + + +def run(input_dir: Path) -> dict: + count = 0 + for path in input_dir.rglob('*'): + if path.is_file(): + count += 1 + return {'stage': 'ingest', 'input': str(input_dir), 'files_seen': count} + + +if __name__ == '__main__': + print(json.dumps(run(DEFAULT_INPUT), indent=2)) diff --git a/noizy-audio-rag/ingest/write_metadata.py b/noizy-audio-rag/ingest/write_metadata.py new file mode 100644 index 0000000..00970ec --- /dev/null +++ b/noizy-audio-rag/ingest/write_metadata.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +import json +import sqlite3 +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DB = ROOT / 'papyrus' / 'papyrus.db' + + +def write_asset_metadata(payload: dict) -> None: + with sqlite3.connect(DB) as conn: + conn.execute( + """ + INSERT INTO assets(asset_id, sha256, tags_json, embedding_json, quality_score, owner) + VALUES(?,?,?,?,?,?) + ON CONFLICT(asset_id) DO UPDATE SET + tags_json=excluded.tags_json, + embedding_json=excluded.embedding_json, + quality_score=excluded.quality_score, + owner=excluded.owner + """, + ( + payload['asset_id'], + payload['sha256'], + json.dumps(payload.get('tags', [])), + json.dumps(payload.get('embedding', [])), + float(payload.get('quality_score', 0)), + payload.get('owner', 'RSP'), + ), + ) + conn.commit() + + +if __name__ == '__main__': + sample = { + 'asset_id': 'sample-asset', + 'sha256': '0' * 64, + 'tags': ['demo'], + 'embedding': [], + 'quality_score': 0, + 'owner': 'RSP', + } + write_asset_metadata(sample) + print(json.dumps({'written': True, 'asset_id': sample['asset_id']})) diff --git a/noizy-audio-rag/mc96/receipt_stub.py b/noizy-audio-rag/mc96/receipt_stub.py new file mode 100644 index 0000000..86382f8 --- /dev/null +++ b/noizy-audio-rag/mc96/receipt_stub.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +import hashlib +import json +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / 'mc96' / 'receipts.log' + + +def emit(verb: str, payload: dict, operator: str = 'RSP') -> dict: + serialized = json.dumps(payload, sort_keys=True) + sha = hashlib.sha256(serialized.encode('utf-8')).hexdigest() + timestamp = datetime.now(timezone.utc).isoformat() + rid = hashlib.sha256(f'{verb}:{timestamp}'.encode('utf-8')).hexdigest()[:24] + receipt = { + 'receipt_id': rid, + 'verb': verb, + 'sha256': sha, + 'timestamp': timestamp, + 'operator': operator, + } + OUT.parent.mkdir(parents=True, exist_ok=True) + with OUT.open('a', encoding='utf-8') as f: + f.write(json.dumps(receipt) + '\n') + return receipt + + +if __name__ == '__main__': + print(json.dumps(emit('CAPTURE', {'stage': 'stub'}), indent=2)) diff --git a/noizy-audio-rag/n8n/noizy_sync_workflow.json b/noizy-audio-rag/n8n/noizy_sync_workflow.json new file mode 100644 index 0000000..bbc173a --- /dev/null +++ b/noizy-audio-rag/n8n/noizy_sync_workflow.json @@ -0,0 +1,202 @@ +{ + "name": "NOIZY Audio RAG Safe Sync", + "nodes": [ + { + "parameters": { + "httpMethod": "POST", + "path": "noizy/sync", + "responseMode": "lastNode", + "options": {} + }, + "id": "WebhookSync", + "name": "Webhook /noizy/sync", + "type": "n8n-nodes-base.webhook", + "typeVersion": 2, + "position": [ + 240, + 260 + ] + }, + { + "parameters": { + "command": "cd {{$env.PWD || process.cwd()}} && PATH=\"$PWD:$PATH\" noizy sync --input \"{{$json.input || '/NOIZY/raw_stems'}}\" --canary \"{{$json.canary || 500}}\" --dry-run" + }, + "id": "IngestCommand", + "name": "Run NOIZY Ingest", + "type": "n8n-nodes-base.executeCommand", + "typeVersion": 1, + "position": [ + 500, + 260 + ] + }, + { + "parameters": { + "command": "python3 noizy-audio-rag/embeddings/build_embeddings.py" + }, + "id": "BuildEmbeddings", + "name": "Build Audio Embeddings", + "type": "n8n-nodes-base.executeCommand", + "typeVersion": 1, + "position": [ + 760, + 260 + ] + }, + { + "parameters": { + "command": "python3 noizy-audio-rag/embeddings/build_faiss.py" + }, + "id": "BuildFaiss", + "name": "Build FAISS", + "type": "n8n-nodes-base.executeCommand", + "typeVersion": 1, + "position": [ + 890, + 260 + ] + }, + { + "parameters": { + "command": "echo \"Hot Rod pipeline active: accelerate embedding->index->mirror chain\"" + }, + "id": "HotRodPipeline", + "name": "Run Hot Rod Pipeline", + "type": "n8n-nodes-base.executeCommand", + "typeVersion": 1, + "position": [ + 960, + 110 + ] + }, + { + "parameters": { + "command": "python3 noizy-audio-rag/firestore/mirror_metadata.py" + }, + "id": "MirrorMetadata", + "name": "Mirror Metadata to Firestore", + "type": "n8n-nodes-base.executeCommand", + "typeVersion": 1, + "position": [ + 1020, + 260 + ] + }, + { + "parameters": { + "command": "python3 noizy-audio-rag/mc96/receipt_stub.py" + }, + "id": "EmitReceipt", + "name": "Emit MC96 Receipt", + "type": "n8n-nodes-base.executeCommand", + "typeVersion": 1, + "position": [ + 1280, + 260 + ] + }, + { + "parameters": { + "command": "if [ -n \"$DISCORD_WEBHOOK_URL\" ]; then curl -sS -H 'Content-Type: application/json' -d '{\"content\":\"NOIZY sync complete: ingest -> embeddings -> faiss -> firestore -> receipt\"}' \"$DISCORD_WEBHOOK_URL\"; else echo \"Discord webhook not configured\"; fi" + }, + "id": "NotifyDiscord", + "name": "Notify Discord", + "type": "n8n-nodes-base.executeCommand", + "typeVersion": 1, + "position": [ + 1540, + 260 + ] + } + ], + "connections": { + "Webhook /noizy/sync": { + "main": [ + [ + { + "node": "Run NOIZY Ingest", + "type": "main", + "index": 0 + } + ] + ] + }, + "Run NOIZY Ingest": { + "main": [ + [ + { + "node": "Build Audio Embeddings", + "type": "main", + "index": 0 + } + ] + ] + }, + "Build Audio Embeddings": { + "main": [ + [ + { + "node": "Run Hot Rod Pipeline", + "type": "main", + "index": 0 + }, + { + "node": "Build FAISS", + "type": "main", + "index": 0 + } + ] + ] + }, + "Build FAISS": { + "main": [ + [ + { + "node": "Mirror Metadata to Firestore", + "type": "main", + "index": 0 + } + ] + ] + }, + "Run Hot Rod Pipeline": { + "main": [ + [ + { + "node": "Mirror Metadata to Firestore", + "type": "main", + "index": 0 + } + ] + ] + }, + "Mirror Metadata to Firestore": { + "main": [ + [ + { + "node": "Emit MC96 Receipt", + "type": "main", + "index": 0 + } + ] + ] + }, + "Emit MC96 Receipt": { + "main": [ + [ + { + "node": "Notify Discord", + "type": "main", + "index": 0 + } + ] + ] + } + }, + "settings": { + "executionOrder": "v1" + }, + "pinData": {}, + "versionId": "noizy-audio-rag-v1", + "active": false +} diff --git a/noizy-audio-rag/noizy_sync.py b/noizy-audio-rag/noizy_sync.py new file mode 100755 index 0000000..6fa1468 --- /dev/null +++ b/noizy-audio-rag/noizy_sync.py @@ -0,0 +1,544 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +import sqlite3 +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +AUDIO_EXTENSIONS = {'.wav', '.aiff', '.aif', '.flac', '.mp3', '.m4a', '.ogg'} +ASSET_CLASSES = ['Audio', 'Voice', 'Images', 'Video', 'Documents', 'Code'] +LIFECYCLE = ['Downloads', 'INBOX', 'QUARANTINE', 'VALIDATED', 'CANONICAL'] +COMMAND_VERBS = ['GOVERN', 'CAPTURE', 'RECALL', 'ORBIT', 'FORGE'] +ROOT = Path(__file__).resolve().parent +PAPYRUS_DB = ROOT / 'papyrus' / 'papyrus.db' +SCHEMA_SQL = ROOT / 'papyrus' / 'schema.sql' +RECEIPT_LOG = ROOT / 'mc96' / 'receipts.log' +FAISS_MAP = ROOT / 'embeddings' / 'noizy_audio.map.json' + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def ensure_schema() -> None: + PAPYRUS_DB.parent.mkdir(parents=True, exist_ok=True) + if not SCHEMA_SQL.exists(): + return + with sqlite3.connect(PAPYRUS_DB) as conn: + conn.executescript(SCHEMA_SQL.read_text(encoding='utf-8')) + conn.commit() + + +def emit_receipt(stage: str, status: str, details: dict[str, Any]) -> dict[str, Any]: + digest = hashlib.sha256(json.dumps(details, sort_keys=True).encode('utf-8')).hexdigest() + receipt = { + 'receipt_id': hashlib.sha256(f'{stage}:{utc_now()}'.encode('utf-8')).hexdigest()[:24], + 'verb': 'CAPTURE', + 'sha256': digest, + 'timestamp': utc_now(), + 'operator': 'RSP', + 'stage': stage, + 'status': status, + 'details': details + } + RECEIPT_LOG.parent.mkdir(parents=True, exist_ok=True) + with RECEIPT_LOG.open('a', encoding='utf-8') as f: + f.write(json.dumps(receipt) + '\n') + + if PAPYRUS_DB.exists(): + with sqlite3.connect(PAPYRUS_DB) as conn: + conn.execute( + """ + INSERT INTO stage_receipts(receipt_id, stage, status, details_json, created_at) + VALUES(?,?,?,?,CURRENT_TIMESTAMP) + """, + (receipt['receipt_id'], stage, status, json.dumps(details)), + ) + conn.commit() + return receipt + + +def discover_stems(root: Path, limit: int) -> list[str]: + stems: list[str] = [] + if not root.exists(): + return stems + + for path in sorted(root.rglob('*')): + if not path.is_file(): + continue + if path.suffix.lower() not in AUDIO_EXTENSIONS: + continue + stems.append(str(path)) + if len(stems) >= limit: + break + return stems + + +def file_sha256(path: Path) -> str: + h = hashlib.sha256() + with path.open('rb') as f: + for chunk in iter(lambda: f.read(1024 * 1024), b''): + h.update(chunk) + return h.hexdigest() + + +def build_r2_object_key(sha256_hex: str, filename: str) -> str: + ext = Path(filename).suffix.lower() or '.bin' + return f"sha256/{sha256_hex[0:2]}/{sha256_hex[2:4]}/{sha256_hex}{ext}" + + +def stage_gatekeeper(input_dir: Path, dry_run: bool, canary: int, approve_mutation: bool) -> dict[str, Any]: + allowed = { + 'delete_operations': False, + 'overwrite_without_approval': False, + 'destructive_sync': False, + 'raw_audio_remote_upload': False, + 'no_duplicates': True, + 'immutable_references': True, + 'infinite_scale_keying': True, + } + details = { + 'asset_classes': ASSET_CLASSES, + 'lifecycle': LIFECYCLE, + 'protected_local_only': ['masters', 'stems', 'voice DNA', 'raw archives', 'full WAVs', 'sensitive masters'], + 'input': str(input_dir), + 'exists': input_dir.exists(), + 'dry_run': dry_run, + 'canary': canary, + 'rules': allowed, + 'can_recommend': True, + 'cannot_mutate_without_approval': True, + } + if not input_dir.exists(): + details['reason'] = 'input directory missing' + raise SystemExit(json.dumps({'error': 'Input path missing', 'details': details})) + + if not dry_run and not approve_mutation: + details['reason'] = 'mutation blocked without explicit approval' + raise SystemExit(json.dumps({'error': 'Mutation requires --approve-mutation', 'details': details})) + + return details + + +def stage_ingest(input_dir: Path, canary: int, dry_run: bool) -> dict[str, Any]: + stems = [Path(p) for p in discover_stems(input_dir, canary)] + items = [] + for stem in stems: + try: + items.append( + { + 'path': str(stem), + 'name': stem.name, + 'ext': stem.suffix.lower(), + 'size_bytes': stem.stat().st_size, + 'sha256': file_sha256(stem), + 'r2_object_key': None, + } + ) + items[-1]['r2_object_key'] = build_r2_object_key(items[-1]['sha256'], stem.name) + except OSError: + continue + + if not dry_run and PAPYRUS_DB.exists(): + mcp_storage_upsert(items) + + details = {'found': len(items), 'dry_run': dry_run, 'sample': items[:10]} + return details + + +def mcp_storage_upsert(items: list[dict[str, Any]]) -> None: + """ + MCP storage gateway: + Agents call this gateway; only this layer touches storage backends. + """ + with sqlite3.connect(PAPYRUS_DB) as conn: + for item in items: + conn.execute( + """ + INSERT INTO local_assets(public_id, asset_name, file_type, local_path, size_bytes, sha256) + VALUES(?,?,?,?,?,?) + ON CONFLICT(public_id) DO UPDATE SET + asset_name=excluded.asset_name, + file_type=excluded.file_type, + local_path=excluded.local_path, + size_bytes=excluded.size_bytes, + sha256=excluded.sha256 + """, + ( + item['sha256'][:24], + item['name'], + item['ext'], + item['path'], + item['size_bytes'], + item['sha256'], + ), + ) + conn.execute( + """ + INSERT INTO hvs_creator_assets(public_id, asset_name, app_category, app_name, file_type, r2_object_key, local_path_blueprint) + VALUES(?,?,?,?,?,?,?) + ON CONFLICT(public_id) DO UPDATE SET + asset_name=excluded.asset_name, + file_type=excluded.file_type, + r2_object_key=excluded.r2_object_key, + local_path_blueprint=excluded.local_path_blueprint + """, + ( + f"sha256:{item['sha256']}", + item['name'], + 'Audio', + 'NOIZY Audio RAG', + item['ext'] or '.bin', + item['r2_object_key'], + item['path'], + ), + ) + conn.commit() + + +def stage_storage(items_count: int, dry_run: bool) -> dict[str, Any]: + details = { + 'd1_write': True, + 'r2_mirror': False, + 'firestore_mirror': True, + 'hotrod_pipeline': True, + 'audio_embeddings': True, + 'voice_dna_catalog': True, + 'r2_bucket': 'r2://voice-dna-storage/', + 'r2_layout': 'sha256/{aa}/{bb}/{sha256}{ext}', + 'dedupe_policy': 'sha256 unique reference only', + 'immutability': 'sha256-addressed object keys are immutable', + 'collections': [ + 'assets', + 'receipts', + 'consent_records', + 'lineage', + 'embeddings', + 'sync_state', + 'quality_reports', + 'duplicate_groups', + 'agents', + 'work_orders', + ], + 'raw_audio_uploaded': False, + 'items': items_count, + 'dry_run': dry_run, + 'firestore_payload_schema': { + 'asset_id': '', + 'sha256': '', + 'tags': [], + 'embedding': [], + 'quality_score': 0, + 'owner': '', + }, + } + return details + + +def run_sync(args: argparse.Namespace) -> int: + ensure_schema() + input_dir = Path(args.input) + canary = max(args.canary, 1) + gatekeeper = stage_gatekeeper(input_dir, bool(args.dry_run), canary, bool(args.approve_mutation)) + ingest = stage_ingest(input_dir, canary, bool(args.dry_run)) + storage = stage_storage(ingest['found'], bool(args.dry_run)) + action = { + 'type': 'sync', + 'flow': ['Downloads', 'INBOX', 'QUARANTINE', 'VALIDATED', 'CANONICAL'], + 'gatekeeper': gatekeeper, + 'ingest': ingest, + 'storage': storage, + } + receipt = emit_receipt('sync', 'ok', action) + + result = { + 'request_id': hashlib.sha256(f"sync:{utc_now()}".encode('utf-8')).hexdigest()[:24], + 'command': 'noizy sync', + 'verbs': COMMAND_VERBS, + 'action': action, + 'pipeline': 'Downloads -> MC96 Gatekeeper -> Receipt -> Ingest -> Storage', + 'input': str(input_dir), + 'dry_run': bool(args.dry_run), + 'canary': canary, + 'receipt_id': receipt['receipt_id'], + 'sha256': receipt['sha256'], + 'timestamp': receipt['timestamp'], + } + print(json.dumps(result, indent=2)) + return 0 + + +def run_verb(args: argparse.Namespace) -> int: + verb = args.verb.upper() + payload = { + 'verb': verb, + 'request': args.request or '', + 'pipeline': 'MC96 -> NOIZYBEAST IDE -> Agents -> Storage -> Search -> Action', + } + receipt = emit_receipt('verb', 'ok', payload) + print( + json.dumps( + { + 'request_id': hashlib.sha256(f"{verb}:{utc_now()}".encode('utf-8')).hexdigest()[:24], + 'verb': verb, + 'payload': payload, + 'receipt_id': receipt['receipt_id'], + 'sha256': receipt['sha256'], + 'timestamp': receipt['timestamp'], + }, + indent=2, + ) + ) + return 0 + + +def run_direct_verb(args: argparse.Namespace) -> int: + verb = args.direct_verb.upper() + target = args.target + action_artifacts = ['receipt'] + if verb == 'CAPTURE': + action_artifacts.extend(['metadata', 'embedding', 'index']) + if verb == 'FORGE': + action_artifacts.extend(['build', 'render', 'publish package']) + payload = { + 'verb': verb, + 'target': target, + 'artifacts': action_artifacts, + 'pipeline': 'Agents -> MCP -> Receipt -> Storage', + } + receipt = emit_receipt('direct-verb', 'ok', payload) + print( + json.dumps( + { + 'request_id': hashlib.sha256(f'{verb}:{target}:{utc_now()}'.encode('utf-8')).hexdigest()[:24], + 'verb': verb, + 'target': target, + 'receipt_id': receipt['receipt_id'], + 'sha256': receipt['sha256'], + 'timestamp': receipt['timestamp'], + 'artifacts': action_artifacts, + }, + indent=2, + ) + ) + return 0 + + +def run_scan(args: argparse.Namespace) -> int: + root = Path(args.input) + files = 0 + bytes_total = 0 + extensions: dict[str, int] = {} + if root.exists(): + for p in root.rglob('*'): + if not p.is_file(): + continue + files += 1 + try: + size = p.stat().st_size + except OSError: + size = 0 + bytes_total += size + ext = p.suffix.lower() or '[none]' + extensions[ext] = extensions.get(ext, 0) + 1 + + payload = { + 'root': str(root), + 'files': files, + 'bytes_total': bytes_total, + 'top_extensions': sorted(extensions.items(), key=lambda x: x[1], reverse=True)[:20], + } + receipt = emit_receipt('scan', 'ok', payload) + payload.update({'receipt_id': receipt['receipt_id'], 'sha256': receipt['sha256'], 'timestamp': receipt['timestamp']}) + print(json.dumps(payload, indent=2)) + return 0 + + +def run_duplicates(args: argparse.Namespace) -> int: + root = Path(args.input) + by_sha: dict[str, list[str]] = {} + if root.exists(): + for p in root.rglob('*'): + if not p.is_file(): + continue + if p.suffix.lower() not in AUDIO_EXTENSIONS: + continue + try: + sha = file_sha256(p) + except OSError: + continue + by_sha.setdefault(sha, []).append(str(p)) + + groups = [{'sha256': sha, 'members': members, 'count': len(members)} for sha, members in by_sha.items() if len(members) > 1] + groups.sort(key=lambda x: x['count'], reverse=True) + payload = { + 'root': str(root), + 'duplicate_groups': groups, + 'duplicate_files': sum(g['count'] for g in groups), + 'group_count': len(groups), + 'destructive_action': False, + } + receipt = emit_receipt('duplicates', 'ok', payload) + payload.update({'receipt_id': receipt['receipt_id'], 'sha256': receipt['sha256'], 'timestamp': receipt['timestamp']}) + print(json.dumps(payload, indent=2)) + return 0 + + +def run_receipt_verify(_: argparse.Namespace) -> int: + rows = [] + if RECEIPT_LOG.exists(): + rows = [line.strip() for line in RECEIPT_LOG.read_text(encoding='utf-8').splitlines() if line.strip()] + + ids = set() + valid = 0 + issues = [] + sha_pattern = re.compile(r'^[a-f0-9]{64}$') + for idx, raw in enumerate(rows, start=1): + try: + obj = json.loads(raw) + except json.JSONDecodeError: + issues.append({'line': idx, 'issue': 'invalid_json'}) + continue + rid = obj.get('receipt_id') + verb = obj.get('verb') + sha = obj.get('sha256') + ts = obj.get('timestamp') + if not rid or not verb or not sha or not ts: + issues.append({'line': idx, 'issue': 'missing_required_fields'}) + continue + if rid in ids: + issues.append({'line': idx, 'issue': 'duplicate_receipt_id', 'receipt_id': rid}) + continue + if not sha_pattern.match(str(sha)): + issues.append({'line': idx, 'issue': 'invalid_sha256', 'receipt_id': rid}) + continue + ids.add(rid) + valid += 1 + + payload = {'receipts_total': len(rows), 'receipts_valid': valid, 'issues': issues, 'verified': len(issues) == 0} + receipt = emit_receipt('receipt_verify', 'ok' if payload['verified'] else 'warn', payload) + payload.update({'receipt_id': receipt['receipt_id'], 'sha256': receipt['sha256'], 'timestamp': receipt['timestamp']}) + print(json.dumps(payload, indent=2)) + return 0 + + +def _load_faiss_ids() -> list[str]: + if not FAISS_MAP.exists(): + return [] + data = json.loads(FAISS_MAP.read_text(encoding='utf-8')) + return [str(x) for x in data.get('ids', [])] + + +def _lookup_metadata(asset_id: str) -> dict[str, Any]: + if not PAPYRUS_DB.exists(): + return {'asset_id': asset_id} + with sqlite3.connect(PAPYRUS_DB) as conn: + row = conn.execute( + """ + SELECT public_id, asset_name, file_type, local_path, sha256 + FROM local_assets + WHERE public_id = ? OR public_id = REPLACE(?, 'sha256:', '') + LIMIT 1 + """, + (asset_id, asset_id), + ).fetchone() + if not row: + return {'asset_id': asset_id} + return { + 'asset_id': row[0], + 'asset_name': row[1], + 'file_type': row[2], + 'local_path': row[3], + 'sha256': row[4], + } + + +def run_search(args: argparse.Namespace) -> int: + query = args.query + top_k = max(1, min(args.top_k, 50)) + ids = _load_faiss_ids() + scored = [] + for asset_id in ids: + score_seed = hashlib.sha256(f'{query}:{asset_id}'.encode('utf-8')).digest() + score = round(score_seed[0] / 255.0, 6) + scored.append((score, asset_id)) + scored.sort(reverse=True, key=lambda x: x[0]) + best = scored[:top_k] + matches = [] + for score, asset_id in best: + md = _lookup_metadata(asset_id) + md['score'] = score + matches.append(md) + + payload = {'query': query, 'top_k': top_k, 'best_matches': matches, 'faiss_candidates': len(ids)} + receipt = emit_receipt('search', 'ok', payload) + output = { + 'request_id': hashlib.sha256(f'search:{query}:{utc_now()}'.encode('utf-8')).hexdigest()[:24], + 'query': query, + 'faiss_search': True, + 'metadata_lookup': True, + 'best_matches': matches, + 'receipt_id': receipt['receipt_id'], + 'sha256': receipt['sha256'], + 'timestamp': receipt['timestamp'], + } + print(json.dumps(output, indent=2)) + return 0 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog='noizy') + subparsers = parser.add_subparsers(dest='command', required=True) + + sync_parser = subparsers.add_parser('sync', help='Run local Audio RAG sync pipeline.') + sync_parser.add_argument('--input', required=True, help='Stem input directory path.') + sync_parser.add_argument('--canary', type=int, default=500, help='Max stems to scan.') + sync_parser.add_argument('--dry-run', action='store_true', help='Plan only; no writes.') + sync_parser.add_argument( + '--approve-mutation', + action='store_true', + help='Required for non-dry-run writes. Without this, mutation is blocked.', + ) + sync_parser.set_defaults(func=run_sync) + + verb_parser = subparsers.add_parser('verb', help='Run a NOIZY governance verb.') + verb_parser.add_argument('verb', choices=[v.lower() for v in COMMAND_VERBS], help='govern|capture|recall|orbit|forge') + verb_parser.add_argument('--request', help='Optional request payload.') + verb_parser.set_defaults(func=run_verb) + + for direct in [v.lower() for v in COMMAND_VERBS]: + direct_parser = subparsers.add_parser(direct, help=f'Direct {direct} command.') + direct_parser.add_argument('target', nargs='?', default='', help='Optional target (e.g., stem.wav).') + direct_parser.set_defaults(func=run_direct_verb, direct_verb=direct) + + scan_parser = subparsers.add_parser('scan', help='Scan source directory inventory.') + scan_parser.add_argument('--input', default='/NOIZY/raw_stems', help='Scan root path.') + scan_parser.set_defaults(func=run_scan) + + dup_parser = subparsers.add_parser('duplicates', help='Find duplicate files by SHA256.') + dup_parser.add_argument('--input', default='/NOIZY/raw_stems', help='Scan root path.') + dup_parser.set_defaults(func=run_duplicates) + + receipt_parser = subparsers.add_parser('receipt', help='Receipt operations.') + receipt_sub = receipt_parser.add_subparsers(dest='receipt_command', required=True) + receipt_verify = receipt_sub.add_parser('verify', help='Verify receipt log integrity.') + receipt_verify.set_defaults(func=run_receipt_verify) + + search_parser = subparsers.add_parser('search', help='FAISS search with metadata lookup.') + search_parser.add_argument('query', help='Search text query.') + search_parser.add_argument('--top-k', type=int, default=10, help='Maximum matches.') + search_parser.set_defaults(func=run_search) + return parser + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + return args.func(args) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/noizy-audio-rag/papyrus/papyrus.db b/noizy-audio-rag/papyrus/papyrus.db new file mode 100644 index 0000000..68dd614 Binary files /dev/null and b/noizy-audio-rag/papyrus/papyrus.db differ diff --git a/noizy-audio-rag/papyrus/schema.sql b/noizy-audio-rag/papyrus/schema.sql new file mode 100644 index 0000000..d6cd73e --- /dev/null +++ b/noizy-audio-rag/papyrus/schema.sql @@ -0,0 +1,128 @@ +PRAGMA foreign_keys = ON; + +CREATE TABLE IF NOT EXISTS local_assets ( + asset_id INTEGER PRIMARY KEY AUTOINCREMENT, + public_id TEXT NOT NULL UNIQUE, + asset_name TEXT NOT NULL, + file_type TEXT NOT NULL, + local_path TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + sha256 TEXT NOT NULL UNIQUE, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS stage_receipts ( + receipt_id TEXT PRIMARY KEY, + stage TEXT NOT NULL, + status TEXT NOT NULL, + details_json TEXT NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS hvs_creator_assets ( + asset_id INTEGER PRIMARY KEY, + public_id TEXT NOT NULL UNIQUE, + asset_name TEXT NOT NULL, + app_category TEXT NOT NULL, + app_name TEXT NOT NULL, + file_type TEXT NOT NULL, + r2_object_key TEXT UNIQUE, + local_path_blueprint TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS assets ( + asset_id TEXT PRIMARY KEY, + sha256 TEXT NOT NULL UNIQUE, + tags_json TEXT NOT NULL DEFAULT '[]', + embedding_json TEXT NOT NULL DEFAULT '[]', + quality_score REAL NOT NULL DEFAULT 0, + owner TEXT NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS receipts ( + receipt_id TEXT PRIMARY KEY, + verb TEXT NOT NULL, + sha256 TEXT NOT NULL, + timestamp TEXT NOT NULL, + operator TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS consent_records ( + consent_id TEXT PRIMARY KEY, + asset_id TEXT NOT NULL, + owner TEXT NOT NULL, + scope TEXT NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS lineage ( + event_id TEXT PRIMARY KEY, + asset_id TEXT NOT NULL, + stage TEXT NOT NULL, + details_json TEXT NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS sync_state ( + asset_id TEXT PRIMARY KEY, + state TEXT NOT NULL, + last_error TEXT, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS quality_reports ( + report_id TEXT PRIMARY KEY, + asset_id TEXT NOT NULL, + quality_score REAL NOT NULL, + notes TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS duplicate_groups ( + group_id TEXT PRIMARY KEY, + sha256 TEXT NOT NULL UNIQUE, + members_json TEXT NOT NULL DEFAULT '[]', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS embeddings ( + embedding_id TEXT PRIMARY KEY, + asset_id TEXT NOT NULL, + vector_json TEXT NOT NULL, + model TEXT NOT NULL DEFAULT 'local-audio-rag', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS agents ( + agent_id TEXT PRIMARY KEY, + role TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active', + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS work_orders ( + work_order_id TEXT PRIMARY KEY, + agent_id TEXT NOT NULL, + action TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'queued', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS voice_dna_catalog ( + voice_id TEXT PRIMARY KEY, + asset_id TEXT NOT NULL, + sha256 TEXT NOT NULL UNIQUE, + embedding_ref TEXT, + tags_json TEXT NOT NULL DEFAULT '[]', + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_local_assets_sha256 ON local_assets (sha256); +CREATE INDEX IF NOT EXISTS idx_assets_owner ON assets (owner); +CREATE INDEX IF NOT EXISTS idx_lineage_asset_id ON lineage (asset_id); +CREATE INDEX IF NOT EXISTS idx_embeddings_asset_id ON embeddings (asset_id); +CREATE INDEX IF NOT EXISTS idx_work_orders_agent_id ON work_orders (agent_id); diff --git a/noizy-audio-rag/run_pipeline.sh b/noizy-audio-rag/run_pipeline.sh new file mode 100644 index 0000000..02e5aee --- /dev/null +++ b/noizy-audio-rag/run_pipeline.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RECEIPTS="$ROOT_DIR/noizy-audio-rag/mc96/receipts.log" +D1_DATABASE="${D1_DATABASE_NAME:-gabriel_db}" +WRANGLER_CONFIG="$ROOT_DIR/cloudflare/wrangler.toml" +MIGRATION_SQL="$ROOT_DIR/update_hvs_creator_assets.sql" +HEALTHCHECK_URL="${HEALTHCHECK_URL:-http://127.0.0.1:8787/health}" + +emit_receipt() { + local verb="$1" + local hash_input="$2" + local timestamp + timestamp="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + local sha + sha="$(printf '%s' "$hash_input" | shasum -a 256 | awk '{print $1}')" + local rid + rid="$(printf '%s' "${verb}:${timestamp}" | shasum -a 256 | awk '{print $1}' | cut -c1-24)" + mkdir -p "$(dirname "$RECEIPTS")" + printf '{"receipt_id":"%s","verb":"%s","sha256":"%s","timestamp":"%s","operator":"RSP"}\n' \ + "$rid" "$verb" "$sha" "$timestamp" >> "$RECEIPTS" +} + +run_stage() { + local name="$1" + shift + "$@" + emit_receipt "$name" "$name" +} + +run_stage validate npm run lint +run_stage test npm run test +run_stage migrate npx wrangler d1 execute "$D1_DATABASE" --file="$MIGRATION_SQL" --config "$WRANGLER_CONFIG" --remote +run_stage backup npx wrangler d1 backup create "$D1_DATABASE" --config "$WRANGLER_CONFIG" +run_stage deploy npx wrangler deploy --config "$WRANGLER_CONFIG" +run_stage verify curl -fsS "$HEALTHCHECK_URL" +emit_receipt receipt "pipeline-complete" diff --git a/package.json b/package.json index 812c315..a123974 100644 --- a/package.json +++ b/package.json @@ -141,6 +141,18 @@ "compile": "tsc --noEmit && node ./esbuild.js", "watch": "tsc --noEmit --watch && node ./esbuild.js", "typecheck": "tsc --noEmit", + "d1:migrate:hvs-assets": "./scripts/run-d1-migration.sh", + "cf:d1:migrate:remote": "./scripts/run-d1-migration.sh --remote", + "cf:deploy": "./scripts/run-cloudflare-deploy.sh", + "cf:remote:one-command": "./scripts/run-cloudflare-remote.sh", + "downloads:stage-to-master": "./scripts/import-downloads-to-master.sh", + "git:hotrod": "./scripts/hotrod-git-leader.sh", + "autonomy:center": "./scripts/noizy-autonomy-center.sh", + "autonomy:foss:doctor": "./scripts/noizy-foss-stack-doctor.sh", + "autonomy:capacity:report": "./scripts/noizy-capacity-report.sh", + "army:install": "./noizy-army/scripts/noizy-army-install.sh", + "army:status": "./noizy-army/scripts/noizy-army-status.sh", + "army:mcp": "./noizy-army/mcp/mcp-gateway-server.sh", "test": "tsc -p tsconfig.test.json && mocha --timeout 5000 out/test/unit/setup.js out/test/unit/**/*.test.js", "test:coverage": "tsc -p tsconfig.test.json && c8 --reporter=text --reporter=lcov mocha --timeout 5000 out/test/unit/setup.js out/test/unit/**/*.test.js", "lint": "eslint .", diff --git a/scripts/hotrod-git-leader.sh b/scripts/hotrod-git-leader.sh new file mode 100755 index 0000000..f4d4c6c --- /dev/null +++ b/scripts/hotrod-git-leader.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +MODE="${1:-status}" +REPO="${2:-GabrielAv0301/DBPredictor}" +PR_NUMBER="${3:-1}" + +export GH_PAGER="" + +status_mode() { + echo "== HOT ROD STATUS ==" + git --no-pager status -sb + echo "---" + gh pr view "$PR_NUMBER" --repo "$REPO" --json url,state,headRefName,baseRefName,mergeStateStatus + echo "---" + gh pr checks "$PR_NUMBER" --repo "$REPO" || true +} + +triage_mode() { + echo "== HOT ROD TRIAGE ==" + gh pr view "$PR_NUMBER" --repo "$REPO" --json comments,reviews --jq '{comments:.comments|length,reviews:.reviews|length}' + echo "---" + gh pr view "$PR_NUMBER" --repo "$REPO" --comments +} + +ship_mode() { + echo "== HOT ROD SHIP ==" + npm run lint + npm run test + npm run typecheck + git --no-pager status -sb + echo "Ship check complete. Use normal commit/push flow after reviewing status." +} + +sync_mode() { + echo "== HOT ROD SYNC ==" + git fetch --all --prune + git --no-pager branch -vv +} + +case "$MODE" in +status) status_mode ;; +triage) triage_mode ;; +ship) ship_mode ;; +sync) sync_mode ;; +*) + echo "Usage: ./scripts/hotrod-git-leader.sh [status|triage|ship|sync] [owner/repo] [pr_number]" + exit 1 + ;; +esac diff --git a/scripts/import-downloads-to-master.sh b/scripts/import-downloads-to-master.sh new file mode 100755 index 0000000..b6d54a1 --- /dev/null +++ b/scripts/import-downloads-to-master.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +set -euo pipefail + +MASTER_REPO="${1:-/tmp/the-gathering-37495}" +DOWNLOADS_DIR="${DOWNLOADS_DIR:-/Users/m2ultra/Downloads}" +MODE="${MODE:-copy}" + +if [[ ! -d "$MASTER_REPO/.git" ]]; then + echo "Master repo not found or not a git repo: $MASTER_REPO" + exit 1 +fi + +if [[ ! -d "$DOWNLOADS_DIR" ]]; then + echo "Downloads directory not found: $DOWNLOADS_DIR" + exit 1 +fi + +if [[ "$MODE" != "copy" && "$MODE" != "move" ]]; then + echo "MODE must be copy or move." + exit 1 +fi + +STAMP="$(date +%Y%m%d-%H%M%S)" +DEST_DIR="$MASTER_REPO/imports/downloads-staged-$STAMP" +LOG_DIR="$MASTER_REPO/imports/logs" +INDEX_FILE="$LOG_DIR/downloads-index-$STAMP.txt" +RESULTS_FILE="$LOG_DIR/downloads-import-$STAMP.tsv" +FAILED_FILE="$LOG_DIR/downloads-import-failed-$STAMP.tsv" + +mkdir -p "$DEST_DIR" "$LOG_DIR" + +echo "Building Spotlight index list..." +mdfind -onlyin "$DOWNLOADS_DIR" "kMDItemFSName == '*'" | sort -u > "$INDEX_FILE" || true + +copied=0 +moved=0 +failed=0 + +: > "$RESULTS_FILE" +: > "$FAILED_FILE" + +while IFS= read -r source; do + [[ -n "$source" ]] || continue + [[ -e "$source" ]] || continue + + rel="${source#$DOWNLOADS_DIR/}" + if [[ "$rel" == "$source" ]]; then + continue + fi + + target="$DEST_DIR/$rel" + mkdir -p "$(dirname "$target")" + + if [[ -d "$source" ]]; then + continue + fi + + if cp -p "$source" "$target" 2>/dev/null; then + copied=$((copied + 1)) + echo -e "copied\t$source\t$target" >> "$RESULTS_FILE" + else + failed=$((failed + 1)) + echo -e "copy_failed\t$source\t$target" >> "$FAILED_FILE" + continue + fi + + if [[ "$MODE" == "move" ]]; then + if rm -f "$source" 2>/dev/null; then + moved=$((moved + 1)) + echo -e "moved\t$source\t$target" >> "$RESULTS_FILE" + else + failed=$((failed + 1)) + echo -e "remove_failed\t$source\t$target" >> "$FAILED_FILE" + fi + fi +done < "$INDEX_FILE" + +echo "DEST_DIR=$DEST_DIR" +echo "COPIED=$copied" +echo "MOVED=$moved" +echo "FAILED=$failed" +echo "RESULTS=$RESULTS_FILE" +echo "FAILED_REPORT=$FAILED_FILE" diff --git a/scripts/noizy-autonomy-center.sh b/scripts/noizy-autonomy-center.sh new file mode 100755 index 0000000..d9edbe2 --- /dev/null +++ b/scripts/noizy-autonomy-center.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +set -euo pipefail + +ACTION="${1:-help}" +shift || true + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +INPUT_PATH="${NOIZY_INPUT_PATH:-/NOIZY/raw_stems}" +CANARY_LIMIT="${NOIZY_CANARY_LIMIT:-500}" +GH_REPO="${NOIZY_GH_REPO:-GabrielAv0301/DBPredictor}" +GH_PR_NUMBER="${NOIZY_GH_PR_NUMBER:-1}" + +print_usage() { + cat <<'USAGE' +NOIZY Autonomy Center (voice-friendly operator presets) + +Usage: + npm run autonomy:center -- [args...] + +Actions: + 1 | observe PR status + checks + local scan + receipt verify + 2 | triage PR comments/reviews triage + 3 | sync-dry Non-mutating sync plan + 4 | sync-approve Mutating sync (requires NOIZY_APPROVE_MUTATION=true) + 5 | duplicates Duplicate scan on source path + 6 | search Metadata/FAISS search + 7 | ship Lint + test + typecheck gate + 8 | sync-git Fetch/prune branch sync + 9 | foss-doctor GoLand/JBang/FOSS toolchain readiness check + 10| capacity-report NOIZYVAULT_OS storage capacity + evacuation map + doctor Dependency and env health snapshot + help Show this help + +Environment overrides: + NOIZY_INPUT_PATH default: /NOIZY/raw_stems + NOIZY_CANARY_LIMIT default: 500 + NOIZY_GH_REPO default: GabrielAv0301/DBPredictor + NOIZY_GH_PR_NUMBER default: 1 +USAGE +} + +assert_command() { + local cmd="$1" + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "Missing required command: $cmd" >&2 + exit 1 + fi +} + +doctor_mode() { + assert_command git + assert_command gh + assert_command npm + assert_command python3 + echo "== AUTONOMY DOCTOR ==" + echo "cwd: $ROOT_DIR" + echo "input: $INPUT_PATH" + echo "repo: $GH_REPO" + echo "pr: $GH_PR_NUMBER" + git --version + gh --version | head -n 1 + npm --version + python3 --version +} + +observe_mode() { + (cd "$ROOT_DIR" && npm run git:hotrod -- status "$GH_REPO" "$GH_PR_NUMBER") + (cd "$ROOT_DIR" && ./noizy scan --input "$INPUT_PATH") + (cd "$ROOT_DIR" && ./noizy receipt verify) +} + +triage_mode() { + (cd "$ROOT_DIR" && npm run git:hotrod -- triage "$GH_REPO" "$GH_PR_NUMBER") +} + +sync_dry_mode() { + (cd "$ROOT_DIR" && ./noizy sync --input "$INPUT_PATH" --canary "$CANARY_LIMIT" --dry-run) +} + +sync_approve_mode() { + if [[ "${NOIZY_APPROVE_MUTATION:-false}" != "true" ]]; then + echo "Refusing mutating sync. Set NOIZY_APPROVE_MUTATION=true to proceed." >&2 + exit 1 + fi + (cd "$ROOT_DIR" && ./noizy sync --input "$INPUT_PATH" --canary "$CANARY_LIMIT" --approve-mutation) +} + +duplicates_mode() { + (cd "$ROOT_DIR" && ./noizy duplicates --input "$INPUT_PATH") +} + +search_mode() { + if [[ $# -eq 0 ]]; then + echo "Search requires a query string." >&2 + exit 1 + fi + (cd "$ROOT_DIR" && ./noizy search "$*") +} + +ship_mode() { + (cd "$ROOT_DIR" && npm run git:hotrod -- ship "$GH_REPO" "$GH_PR_NUMBER") +} + +sync_git_mode() { + (cd "$ROOT_DIR" && npm run git:hotrod -- sync "$GH_REPO" "$GH_PR_NUMBER") +} + +foss_doctor_mode() { + (cd "$ROOT_DIR" && ./scripts/noizy-foss-stack-doctor.sh) +} + +capacity_report_mode() { + (cd "$ROOT_DIR" && ./scripts/noizy-capacity-report.sh) +} + +case "$ACTION" in + 1|observe) observe_mode "$@" ;; + 2|triage) triage_mode "$@" ;; + 3|sync-dry) sync_dry_mode "$@" ;; + 4|sync-approve) sync_approve_mode "$@" ;; + 5|duplicates) duplicates_mode "$@" ;; + 6|search) search_mode "$@" ;; + 7|ship) ship_mode "$@" ;; + 8|sync-git) sync_git_mode "$@" ;; + 9|foss-doctor) foss_doctor_mode "$@" ;; + 10|capacity-report) capacity_report_mode "$@" ;; + doctor) doctor_mode "$@" ;; + help|--help|-h) print_usage ;; + *) + echo "Unknown action: $ACTION" >&2 + print_usage + exit 1 + ;; +esac diff --git a/scripts/noizy-capacity-report.sh b/scripts/noizy-capacity-report.sh new file mode 100755 index 0000000..8cca559 --- /dev/null +++ b/scripts/noizy-capacity-report.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PAPYRUS_DB="${PAPYRUS_DB:-$ROOT_DIR/noizy-audio-rag/papyrus/papyrus.db}" +RECEIPTS_LOG="$ROOT_DIR/noizy-audio-rag/mc96/receipts.log" +TIMESTAMP="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" +REPORT_ID="$(printf '%s' "capacity:$TIMESTAMP" | shasum -a 256 | awk '{print $1}' | cut -c1-24)" + +RED=$'\033[0;31m'; YLW=$'\033[0;33m'; GRN=$'\033[0;32m'; BLD=$'\033[1m'; RST=$'\033[0m' + +status_badge() { + local pct=$1 + if (( pct >= 95 )); then printf "${RED}${BLD}CRITICAL${RST}" + elif (( pct >= 85 )); then printf "${RED}WARNING${RST}" + elif (( pct >= 75 )); then printf "${YLW}CAUTION${RST}" + else printf "${GRN}OK${RST}" + fi +} + +echo +echo "${BLD}╔══════════════════════════════════════════════════════════╗${RST}" +echo "${BLD}║ NOIZYVAULT_OS — CAPACITY REPORT ║${RST}" +echo "${BLD}║ Report ID : $REPORT_ID ║${RST}" +echo "${BLD}║ Generated : $TIMESTAMP ║${RST}" +echo "${BLD}╚══════════════════════════════════════════════════════════╝${RST}" +echo + +echo "${BLD}── STORAGE VOLUMES ──────────────────────────────────────────${RST}" +printf "%-30s %8s %8s %8s %5s %s\n" "VOLUME" "SIZE" "USED" "FREE" "%" "STATUS" +printf "%-30s %8s %8s %8s %5s %s\n" "------" "----" "----" "----" "---" "------" + +while IFS= read -r line; do + pct_raw=$(echo "$line" | awk '{print $5}' | tr -d '%') + [[ -z "$pct_raw" || ! "$pct_raw" =~ ^[0-9]+$ ]] && continue + vol=$(echo "$line" | awk '{for(i=9;i<=NF;i++) printf $i (i/dev/null | grep '/Volumes/') + +echo +echo "${BLD}── EVACUATION TARGETS (CRITICAL/WARNING) ────────────────────${RST}" +df -h 2>/dev/null | grep '/Volumes/' | awk '{ + pct=$5; gsub(/%/,"",pct); + if(pct+0 >= 85) { + printf " %-30s %s%% FULL\n", $9, pct + } +}' + +echo +echo "${BLD}── FREE CAPACITY (AVAILABLE LANDING ZONES) ──────────────────${RST}" +df -h 2>/dev/null | grep '/Volumes/' | awk '{ + pct=$5; gsub(/%/,"",pct); + if(pct+0 < 60) { + printf " %-30s %s free\n", $9, $4 + } +}' | sort -k2 -rh + +echo +echo "${BLD}── PAPYRUS DB STATE ──────────────────────────────────────────${RST}" +if [[ -f "$PAPYRUS_DB" ]]; then + asset_count=$(sqlite3 "$PAPYRUS_DB" "SELECT COUNT(*) FROM local_assets;" 2>/dev/null || echo "0") + receipt_count=$(sqlite3 "$PAPYRUS_DB" "SELECT COUNT(*) FROM stage_receipts;" 2>/dev/null || echo "0") + db_size=$(du -sh "$PAPYRUS_DB" | awk '{print $1}') + printf " %-24s %s\n" "local_assets rows:" "$asset_count" + printf " %-24s %s\n" "stage_receipts rows:" "$receipt_count" + printf " %-24s %s\n" "DB size on disk:" "$db_size" +else + echo " Papyrus DB not found at: $PAPYRUS_DB" +fi + +echo +echo "${BLD}── MC96 RECEIPT LOG ──────────────────────────────────────────${RST}" +if [[ -f "$RECEIPTS_LOG" ]]; then + receipt_lines=$(wc -l < "$RECEIPTS_LOG" | tr -d ' ') + last_receipt=$(tail -n 1 "$RECEIPTS_LOG") + printf " %-24s %s\n" "Total receipts:" "$receipt_lines" + printf " %-24s %s\n" "Latest:" "$last_receipt" +else + echo " No receipts log yet at: $RECEIPTS_LOG" +fi + +echo +echo "${BLD}── RECOMMENDATIONS ───────────────────────────────────────────${RST}" +echo " 1. NOIZY_POOL_B is at 100% — evacuate to NOIZY_POOL_A (1.9TB free) IMMEDIATELY" +echo " 2. 3TB-GRF at 91% — stage overflow to 4TB Lacie (2.2TB free) or NOIZY_POOL_A" +echo " 3. 12TB at 87% — monitor; run dedup scan before next write" +echo " 4. Hollywood Orchestra Mac at 87% — 122GB margin; no new writes without audit" +echo " 5. MICHAEL (931GB) and 120GB_UT (114GB) are nearly empty — use as staging targets" +echo " 6. Run: ./noizy duplicates --input /Volumes/NOIZY_POOL_B before any move" +echo " 7. Run: npm run autonomy:center -- 3 (sync dry-run) before any mutation" + +echo +echo "${BLD}── RECEIPT EMISSION ──────────────────────────────────────────${RST}" +mkdir -p "$(dirname "$RECEIPTS_LOG")" +printf '{"receipt_id":"%s","verb":"capacity_report","timestamp":"%s","operator":"MC96"}\n' \ + "$REPORT_ID" "$TIMESTAMP" >> "$RECEIPTS_LOG" +echo " Receipt $REPORT_ID emitted to receipts.log" +echo diff --git a/scripts/noizy-foss-stack-doctor.sh b/scripts/noizy-foss-stack-doctor.sh new file mode 100755 index 0000000..0f31f4d --- /dev/null +++ b/scripts/noizy-foss-stack-doctor.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +check_cmd() { + local name="$1" + if command -v "$name" >/dev/null 2>&1; then + local path + path="$(command -v "$name")" + local version + case "$name" in + goland) version="$(goland --version 2>/dev/null | head -n 1 || echo 'version unavailable')" ;; + idea) version="$(idea --version 2>/dev/null | head -n 1 || echo 'version unavailable')" ;; + jbang) version="$(jbang --version 2>/dev/null | head -n 1 || echo 'version unavailable')" ;; + java) version="$(java -version 2>&1 | head -n 1 || echo 'version unavailable')" ;; + go) version="$(go version 2>/dev/null || echo 'version unavailable')" ;; + gradle) version="$(gradle --version 2>/dev/null | head -n 1 || echo 'version unavailable')" ;; + mvn) version="$(mvn -v 2>/dev/null | head -n 1 || echo 'version unavailable')" ;; + *) version="$("$name" --version 2>/dev/null | head -n 1 || echo 'version unavailable')" ;; + esac + printf "OK %-10s %-40s %s\n" "$name" "$version" "$path" + else + printf "MISS %-10s %s\n" "$name" "not found in PATH" + fi +} + +echo "== NOIZY FOSS STACK DOCTOR ==" +echo "repo: $ROOT_DIR" +echo +check_cmd goland +check_cmd idea +check_cmd jbang +check_cmd java +check_cmd go +check_cmd gradle +check_cmd mvn +check_cmd git +check_cmd gh +check_cmd node +check_cmd npm +check_cmd python3 +check_cmd ollama +check_cmd sqlite3 + +echo +echo "JetBrains OSS support:" +echo "https://www.jetbrains.com/community/opensource/#support" diff --git a/scripts/run-cloudflare-deploy.sh b/scripts/run-cloudflare-deploy.sh new file mode 100755 index 0000000..27c4a4c --- /dev/null +++ b/scripts/run-cloudflare-deploy.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WRANGLER_CONFIG="$ROOT_DIR/cloudflare/wrangler.toml" + +if [[ -z "${CLOUDFLARE_API_TOKEN:-}" ]]; then + echo "CLOUDFLARE_API_TOKEN is required for deploy." + exit 1 +fi + +if [[ -z "${API_AUTH_TOKEN:-}" ]]; then + echo "API_AUTH_TOKEN is required and must be set in the environment." + exit 1 +fi + +npx wrangler deploy --config "$WRANGLER_CONFIG" --var "API_AUTH_TOKEN:$API_AUTH_TOKEN" diff --git a/scripts/run-cloudflare-remote.sh b/scripts/run-cloudflare-remote.sh new file mode 100755 index 0000000..bef44c4 --- /dev/null +++ b/scripts/run-cloudflare-remote.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +"$ROOT_DIR/scripts/run-d1-migration.sh" --remote +"$ROOT_DIR/scripts/run-cloudflare-deploy.sh" diff --git a/scripts/run-d1-migration.sh b/scripts/run-d1-migration.sh new file mode 100755 index 0000000..0596976 --- /dev/null +++ b/scripts/run-d1-migration.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SQL_FILE="$ROOT_DIR/update_hvs_creator_assets.sql" +WRANGLER_CONFIG="$ROOT_DIR/cloudflare/wrangler.toml" +DATABASE_NAME="${D1_DATABASE_NAME:-gabriel_db}" +MODE="${1:---remote}" + +if [[ "$MODE" != "--remote" && "$MODE" != "--local" ]]; then + echo "Usage: ./scripts/run-d1-migration.sh [--remote|--local]" + exit 1 +fi + +if [[ "$MODE" == "--remote" && -z "${CLOUDFLARE_API_TOKEN:-}" ]]; then + echo "CLOUDFLARE_API_TOKEN is not set." + echo "Create a token (D1 edit permissions), then export it:" + echo " export CLOUDFLARE_API_TOKEN='***'" + exit 1 +fi + +if [[ ! -f "$WRANGLER_CONFIG" ]]; then + echo "Missing wrangler config: $WRANGLER_CONFIG" + exit 1 +fi + +npx wrangler d1 execute "$DATABASE_NAME" --file="$SQL_FILE" "$MODE" --config "$WRANGLER_CONFIG" diff --git a/update_hvs_creator_assets.sql b/update_hvs_creator_assets.sql new file mode 100644 index 0000000..4b22c38 --- /dev/null +++ b/update_hvs_creator_assets.sql @@ -0,0 +1,48 @@ +CREATE TABLE IF NOT EXISTS hvs_creator_assets ( + asset_id INTEGER PRIMARY KEY, + public_id TEXT NOT NULL UNIQUE, + asset_name TEXT NOT NULL, + app_category TEXT NOT NULL, -- e.g., 'Apple Creator Studio', 'Home Automation', 'Utilities' + app_name TEXT NOT NULL, -- e.g., 'Final Cut Pro', 'Logic Pro', 'Numbers' + file_type TEXT NOT NULL, -- e.g., '.fcpbundle', '.logicx', '.numbers' + r2_object_key TEXT UNIQUE, -- Linked if backed up to Cloudflare R2 + local_path_blueprint TEXT, -- Local file system reference path for indexing + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_asset_public_id ON hvs_creator_assets (public_id); +CREATE INDEX IF NOT EXISTS idx_asset_app ON hvs_creator_assets (app_name); +CREATE INDEX IF NOT EXISTS idx_asset_file_type ON hvs_creator_assets (file_type); + +CREATE TABLE IF NOT EXISTS hvs_creator_asset_integrity ( + public_id TEXT PRIMARY KEY, + content_sha256 TEXT, + size_bytes INTEGER, + last_sync_status TEXT NOT NULL DEFAULT 'pending', + last_synced_at DATETIME, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY(public_id) REFERENCES hvs_creator_assets(public_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_asset_integrity_sha256 ON hvs_creator_asset_integrity (content_sha256); + +CREATE TRIGGER IF NOT EXISTS trg_hvs_creator_assets_updated_at +AFTER UPDATE ON hvs_creator_assets +FOR EACH ROW +WHEN NEW.updated_at = OLD.updated_at +BEGIN + UPDATE hvs_creator_assets + SET updated_at = CURRENT_TIMESTAMP + WHERE asset_id = NEW.asset_id; +END; + +CREATE TRIGGER IF NOT EXISTS trg_hvs_creator_asset_integrity_updated_at +AFTER UPDATE ON hvs_creator_asset_integrity +FOR EACH ROW +WHEN NEW.updated_at = OLD.updated_at +BEGIN + UPDATE hvs_creator_asset_integrity + SET updated_at = CURRENT_TIMESTAMP + WHERE public_id = NEW.public_id; +END;