diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 078befd..0060af3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -232,23 +232,23 @@ jobs: output: trivy-results.sarif severity: CRITICAL,HIGH - - name: Enforce no CRITICAL vulnerabilities (backend image) + - name: Enforce no CRITICAL/HIGH vulnerabilities (backend image) uses: aquasecurity/trivy-action@v0.36.0 with: scan-type: image image-ref: loopless-backend:security format: table - severity: CRITICAL + severity: CRITICAL,HIGH ignore-unfixed: true exit-code: "1" - - name: Enforce no CRITICAL vulnerabilities (frontend image) + - name: Enforce no CRITICAL/HIGH vulnerabilities (frontend image) uses: aquasecurity/trivy-action@v0.36.0 with: scan-type: image image-ref: loopless-frontend:security format: table - severity: CRITICAL + severity: CRITICAL,HIGH ignore-unfixed: true exit-code: "1" @@ -310,9 +310,8 @@ jobs: context: frontend/client file: devops/docker/frontend/Dockerfile push: true - build-args: | - NEXT_PUBLIC_API_BASE_URL=https://api.project-01.gjirafa.dev - NEXT_PUBLIC_KEYCLOAK_URL=https://api.project-01.gjirafa.dev/auth + # No build-args: public config (API/Keycloak URLs) is injected at RUNTIME via + # window.__ENV (Helm sets APP_* on the frontend pod), so the image is env-agnostic. tags: | ${{ env.REGISTRY }}/${{ steps.owner.outputs.value }}/loopless-frontend:sha-${{ github.sha }} ${{ env.REGISTRY }}/${{ steps.owner.outputs.value }}/loopless-frontend:staging diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml deleted file mode 100644 index d11f5c2..0000000 --- a/.github/workflows/terraform.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: Terraform Validate - -on: - push: - branches: [main] - paths: - - "devops/terraform/**" - - ".github/workflows/terraform.yml" - pull_request: - paths: - - "devops/terraform/**" - - ".github/workflows/terraform.yml" - workflow_dispatch: - -permissions: - contents: read - pull-requests: write - -defaults: - run: - shell: bash - -jobs: - fmt: - name: terraform fmt - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: "1.7.0" - terraform_wrapper: false - - - name: Check formatting - working-directory: devops/terraform - run: terraform fmt -check -recursive -diff - - validate: - name: terraform validate (${{ matrix.env }}) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - env: [staging, production] - - steps: - - uses: actions/checkout@v4 - - - uses: hashicorp/setup-terraform@v3 - with: - terraform_version: "1.7.0" - terraform_wrapper: false - - # init with -backend=false: we don't need AWS credentials or remote - # state for static validation — just provider download + module graph. - - name: Init (backend disabled) - working-directory: devops/terraform/envs/${{ matrix.env }} - run: terraform init -backend=false -input=false - - - name: Validate - working-directory: devops/terraform/envs/${{ matrix.env }} - run: terraform validate -no-color - - tflint: - name: tflint - runs-on: ubuntu-latest - continue-on-error: true - steps: - - uses: actions/checkout@v4 - - - uses: terraform-linters/setup-tflint@v4 - with: - tflint_version: latest - - - name: Run tflint on modules - run: | - for module in devops/terraform/modules/*/; do - echo "→ Linting ${module}" - (cd "${module}" && tflint --init && tflint --recursive --format=compact) - done diff --git a/CLAUDE.md b/CLAUDE.md index 63af7d7..4945497 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,7 +18,7 @@ - **Completed:** M1, M2, M3, M4, M5 · Sprints 1–5 - **Frontend:** Next.js 15 — 11 route groups implemented; most pages production-ready with TanStack Query, RHF+Zod, Framer Motion, Zustand. Authed routes live under `app/(protected)/` route group — wrapped in `AuthGuard` (token-presence redirect to `/login`); admin layout additionally wrapped in `AdminGuard` (non-admin → `/discover`). - **Backend:** .NET 10 Clean Architecture — fully scaffolded and functional. 18 domain entities, 12 EF Core migrations (Apr 22–29), 10 endpoint groups, SignalR MessagingHub + NotificationsHub (CP-105 ✅), 4 Hangfire jobs, RabbitMQ publisher + 2 consumers, OpenAI embeddings + summary, S3 storage, Keycloak JWT + RBAC. Rate limiting added (BE-15). Correlation ID middleware added (BE-16). Integration tests scaffolded with Testcontainers (BE-17). -- **DevOps:** Docker Compose dev stack active (15 services: added Elasticsearch, Kibana, Uptime Kuma, Nginx). K8s manifests + Kustomize overlays + Helm chart done. Terraform 4 modules done. CI/CD (GitHub Actions) done + release-please semantic versioning workflow. Prometheus + Grafana + Loki + ELK active in compose. Nginx configured with reverse proxy rules. +- **DevOps:** Docker Compose dev stack active (15 services: PostgreSQL, Redis, MinIO, Elasticsearch, Kibana, Uptime Kuma, Nginx). K8s manifests + Kustomize overlays + Helm chart done (database, cache, storage all via Helm). CI/CD (GitHub Actions) done + release-please semantic versioning workflow. Prometheus + Grafana + Loki + ELK active in compose. Nginx configured with reverse proxy rules. --- @@ -70,9 +70,6 @@ LIFE-Group1/ │ ├── k8s/overlays/production/ Same │ ├── monitoring/prometheus/ prometheus.yml (5 jobs), alert-rules.yml (4 rules) │ ├── monitoring/grafana/ 3 dashboards + provisioning datasources -│ ├── terraform/ versions.tf, backend.tf (S3+DynamoDB state lock) -│ │ ├── modules/ cache, database (RDS PG16+pgvector), network, storage -│ │ └── envs/staging,production Per-env tfvars │ ├── keycloak/loopless-realm.json Keycloak realm export (auto-imported via docker compose) │ └── loadtest/ k6 load test scripts │ diff --git a/README.md b/README.md index fa76b41..1aa2007 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ flowchart LR | AI | OpenAI embeddings (1536-dim) + chat completions for project summaries / triage | | Observability | Prometheus 2.54 + Alertmanager, Grafana 11, Loki 3.1, Elasticsearch + Logstash + Kibana 8.14, Uptime Kuma | | Automation | aiops-triage (Node.js) — Alertmanager webhook → OpenAI → Slack/Discord; n8n for workflow automation | -| Infrastructure | Docker Compose (dev), Kubernetes + Kustomize + Helm (prod), Terraform 1.7+ (AWS), Nginx + Let's Encrypt | +| Infrastructure | Docker Compose (dev), Kubernetes + Kustomize + Helm (prod — all infra via Helm), Nginx + Let's Encrypt | | CI/CD | GitHub Actions — lint/test/Trivy → push GHCR → kustomize deploy; release-please semantic versioning | --- @@ -195,9 +195,8 @@ sudo -E SSH_PUBLIC_KEY='ssh-ed25519 ...' bash devops/scripts/server/bootstrap-ub | What | Where | How to apply | | -------------- | ------------------------------------------------------------- | ------------------------------------------------------------ | -| AWS resources | [`devops/terraform/`](./devops/terraform/) | See [`RUNBOOK.md`](./devops/terraform/RUNBOOK.md) | +| Helm charts | [`devops/helm/`](./devops/helm/) | All infrastructure deployed via Helm | | Kubernetes | [`devops/k8s/`](./devops/k8s/) | `kubectl apply -k devops/k8s/overlays/staging` | -| Server bootstrap | [`devops/scripts/server/`](./devops/scripts/server/) | See [`devops/scripts/README.md`](./devops/scripts/README.md) | | TLS certs | [`devops/scripts/init-letsencrypt.sh`](./devops/scripts/init-letsencrypt.sh) | `DOMAIN=... EMAIL=... sudo bash ...` | | Secrets | Azure Key Vault (`KEYVAULT_URI` env var) | Backend auto-loads when URI is set | @@ -205,7 +204,6 @@ sudo -E SSH_PUBLIC_KEY='ssh-ed25519 ...' bash devops/scripts/server/bootstrap-ub - `ci.yml` — lint, test, build, Trivy scan, push images to GHCR - `cd.yml` — deploy to staging then production (manual approval gate) -- `terraform.yml` — `fmt + validate + tflint` on every Terraform PR - `release.yml` — release-please semantic versioning --- diff --git a/backend/src/Loopless.Api/Endpoints/ProjectEndpoints.cs b/backend/src/Loopless.Api/Endpoints/ProjectEndpoints.cs index f605532..61ec856 100644 --- a/backend/src/Loopless.Api/Endpoints/ProjectEndpoints.cs +++ b/backend/src/Loopless.Api/Endpoints/ProjectEndpoints.cs @@ -16,6 +16,7 @@ using Loopless.Application.Features.Projects.GetProjectInvitations; using Loopless.Application.Features.Projects.GetProjectLinks; using Loopless.Application.Features.Projects.GetSummaries; +using Loopless.Application.Features.Projects.GetSummaryStatus; using Loopless.Application.Features.Projects.InviteFreelancer; using Loopless.Application.Features.Projects.Leave; using Loopless.Application.Features.Projects.RejectApplicant; @@ -27,10 +28,14 @@ using Loopless.Application.Features.Tasks.DeleteTask; using Loopless.Application.Features.Tasks.GetProjectTasks; using Loopless.Application.Features.Tasks.UpdateTaskStatus; +using Loopless.Application.Interfaces; using Loopless.Domain.Enums; using Loopless.Infrastructure.Identity; using MediatR; using Microsoft.AspNetCore.Mvc; +using Microsoft.EntityFrameworkCore; +using System.Security.Cryptography; +using System.Text; namespace Loopless.Api.Endpoints; @@ -186,6 +191,7 @@ public static IEndpointRouteBuilder MapProjectEndpoints(this IEndpointRouteBuild .WithName("GetProjectCommits") .Produces>(StatusCodes.Status200OK) .Produces(StatusCodes.Status401Unauthorized) + .Produces(StatusCodes.Status403Forbidden) .Produces(StatusCodes.Status404NotFound) .ProducesValidationProblem(); @@ -296,24 +302,79 @@ public static IEndpointRouteBuilder MapProjectEndpoints(this IEndpointRouteBuild .Produces(StatusCodes.Status409Conflict) .ProducesValidationProblem(); + group.MapPost("/{projectId:guid}/github/webhook", async ( + [FromRoute] Guid projectId, + HttpRequest request, + IAppDbContext db, + ICommitSyncEnqueuer enqueuer, + CancellationToken ct) => + { + request.EnableBuffering(); + using var ms = new MemoryStream(); + await request.Body.CopyToAsync(ms, ct); + var rawBody = ms.ToArray(); + + var project = await db.Projects + .FirstOrDefaultAsync(p => p.Id == projectId, ct); + + if (project is null) + return Results.NotFound(); + + if (string.IsNullOrWhiteSpace(project.GitHubWebhookSecret)) + return Results.Problem("Webhook secret not configured for this project.", statusCode: StatusCodes.Status400BadRequest); + + var signatureHeader = request.Headers["X-Hub-Signature-256"].FirstOrDefault(); + if (!IsValidGitHubSignature(rawBody, signatureHeader, project.GitHubWebhookSecret)) + return Results.Unauthorized(); + + var eventType = request.Headers["X-GitHub-Event"].FirstOrDefault(); + if (eventType is not "push") + return Results.Ok(); + + enqueuer.EnqueueProjectSync(projectId); + return Results.Accepted($"/api/v1/projects/{projectId}/commits"); + }) + .AllowAnonymous() + .WithName("GitHubWebhook") + .Produces(StatusCodes.Status202Accepted) + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status400BadRequest) + .Produces(StatusCodes.Status401Unauthorized) + .Produces(StatusCodes.Status404NotFound); + group.MapPost("/{projectId:guid}/summary", async ( [FromRoute] Guid projectId, ISender sender, CancellationToken ct) => { var dto = await sender.Send(new GenerateProjectSummaryCommand(projectId), ct); - return Results.Ok(dto); + return Results.Accepted(value: dto); }) .RequireAuthorization() .RequireRateLimiting("ai") .WithName("GenerateProjectSummary") - .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status202Accepted) .Produces(StatusCodes.Status401Unauthorized) .Produces(StatusCodes.Status403Forbidden) .Produces(StatusCodes.Status404NotFound) .Produces(StatusCodes.Status429TooManyRequests) .ProducesValidationProblem(); + group.MapGet("/{projectId:guid}/summary/status", async ( + [FromRoute] Guid projectId, + ISender sender, + CancellationToken ct) => + { + var result = await sender.Send(new GetProjectSummaryStatusQuery(projectId), ct); + return Results.Ok(result); + }) + .RequireAuthorization() + .WithName("GetProjectSummaryStatus") + .Produces(StatusCodes.Status200OK) + .Produces(StatusCodes.Status401Unauthorized) + .Produces(StatusCodes.Status403Forbidden) + .Produces(StatusCodes.Status404NotFound); + group.MapGet("/{projectId:guid}/summaries", async ( [FromRoute] Guid projectId, [FromQuery] int? page, @@ -569,4 +630,18 @@ private sealed record CreateTaskBody( DateTimeOffset? DueDate); private sealed record UpdateTaskStatusBody(ProjectTaskStatus Status); + + private static bool IsValidGitHubSignature(byte[] body, string? signatureHeader, string secret) + { + if (string.IsNullOrEmpty(signatureHeader) || + !signatureHeader.StartsWith("sha256=", StringComparison.Ordinal)) + return false; + + byte[] expectedBytes; + try { expectedBytes = Convert.FromHexString(signatureHeader["sha256=".Length..]); } + catch { return false; } + + var actualHash = HMACSHA256.HashData(Encoding.UTF8.GetBytes(secret), body); + return CryptographicOperations.FixedTimeEquals(actualHash, expectedBytes); + } } diff --git a/backend/src/Loopless.Api/Endpoints/StandupEndpoints.cs b/backend/src/Loopless.Api/Endpoints/StandupEndpoints.cs index cefe22e..ec85147 100644 --- a/backend/src/Loopless.Api/Endpoints/StandupEndpoints.cs +++ b/backend/src/Loopless.Api/Endpoints/StandupEndpoints.cs @@ -52,6 +52,8 @@ public static IEndpointRouteBuilder MapStandupEndpoints(this IEndpointRouteBuild .WithName("GetStandupFeed") .Produces>(StatusCodes.Status200OK) .Produces(StatusCodes.Status401Unauthorized) + .Produces(StatusCodes.Status403Forbidden) + .Produces(StatusCodes.Status404NotFound) .ProducesValidationProblem(); group.MapGet("/analytics", async ( @@ -84,6 +86,7 @@ public static IEndpointRouteBuilder MapStandupEndpoints(this IEndpointRouteBuild .WithName("ExportStandupsCsv") .Produces(StatusCodes.Status200OK, contentType: "text/csv") .Produces(StatusCodes.Status401Unauthorized) + .Produces(StatusCodes.Status403Forbidden) .Produces(StatusCodes.Status404NotFound); return app; diff --git a/backend/src/Loopless.Api/Program.cs b/backend/src/Loopless.Api/Program.cs index 0ef4a11..8344cd0 100644 --- a/backend/src/Loopless.Api/Program.cs +++ b/backend/src/Loopless.Api/Program.cs @@ -109,16 +109,22 @@ QueueLimit = 2, })); - options.AddFixedWindowLimiter("auth", opt => + // auth: Redis-backed, fail-closed — 503 during Redis outages to prevent bypass in k8s multi-pod. + options.AddPolicy("auth", httpContext => { - opt.PermitLimit = 10; - opt.Window = TimeSpan.FromMinutes(1); - opt.QueueProcessingOrder = QueueProcessingOrder.OldestFirst; - opt.QueueLimit = 0; + var ip = httpContext.Connection.RemoteIpAddress?.ToString() ?? "unknown"; + var redis = httpContext.RequestServices.GetService(); + return redis is not null + ? RateLimitPartition.Get(ip, _ => + new RedisFixedWindowRateLimiter(redis, $"ratelimit:auth:{ip}", 10, TimeSpan.FromMinutes(1), + failClosed: true, policyName: "auth")) + : RateLimitPartition.GetFixedWindowLimiter(ip, _ => + new FixedWindowRateLimiterOptions { PermitLimit = 10, Window = TimeSpan.FromMinutes(1) }); }); // Per-user Redis-backed policies for authenticated endpoints. // Falls back to in-memory if Redis is not configured (single-replica dev setup). + // ai: fail-closed — 503 on Redis outage; ai endpoints are high-value abuse targets. options.AddPolicy("ai", httpContext => { var userId = httpContext.User.FindFirst("sub")?.Value @@ -126,11 +132,13 @@ var redis = httpContext.RequestServices.GetService(); return redis is not null ? RateLimitPartition.Get(userId, _ => - new RedisFixedWindowRateLimiter(redis, $"ratelimit:ai:{userId}", 5, TimeSpan.FromMinutes(1))) + new RedisFixedWindowRateLimiter(redis, $"ratelimit:ai:{userId}", 5, TimeSpan.FromMinutes(1), + failClosed: true, policyName: "ai")) : RateLimitPartition.GetFixedWindowLimiter(userId, _ => new FixedWindowRateLimiterOptions { PermitLimit = 5, Window = TimeSpan.FromMinutes(1) }); }); + // matching: fail-open — availability preferred; search abuse is lower severity. options.AddPolicy("matching", httpContext => { var userId = httpContext.User.FindFirst("sub")?.Value @@ -138,45 +146,66 @@ var redis = httpContext.RequestServices.GetService(); return redis is not null ? RateLimitPartition.Get(userId, _ => - new RedisFixedWindowRateLimiter(redis, $"ratelimit:matching:{userId}", 30, TimeSpan.FromMinutes(1))) + new RedisFixedWindowRateLimiter(redis, $"ratelimit:matching:{userId}", 30, TimeSpan.FromMinutes(1), + failClosed: false, policyName: "matching")) : RateLimitPartition.GetFixedWindowLimiter(userId, _ => new FixedWindowRateLimiterOptions { PermitLimit = 30, Window = TimeSpan.FromMinutes(1) }); }); - options.RejectionStatusCode = StatusCodes.Status429TooManyRequests; + options.OnRejected = async (context, cancellationToken) => + { + var isRedisUnavailable = context.Lease.TryGetMetadata("redis_unavailable", out _); + context.HttpContext.Response.StatusCode = isRedisUnavailable + ? StatusCodes.Status503ServiceUnavailable + : StatusCodes.Status429TooManyRequests; + + if (!isRedisUnavailable && context.Lease.TryGetMetadata(MetadataName.RetryAfter, out TimeSpan retryAfter)) + context.HttpContext.Response.Headers.RetryAfter = + ((int)retryAfter.TotalSeconds).ToString(); + + await context.HttpContext.Response.WriteAsync( + isRedisUnavailable ? "Service temporarily unavailable." : "Rate limit exceeded.", + cancellationToken); + }; }); - builder.Services.AddEndpointsApiExplorer(); - builder.Services.AddSwaggerGen(options => + // Swagger/OpenAPI generator + UI are registered ONLY in Development. In Production the + // generator is never added and the /swagger, /swagger/v1/swagger.json and /api-docs routes + // are completely un-routed (see the matching middleware guard below). + if (builder.Environment.IsDevelopment()) { - options.SwaggerDoc("v1", new OpenApiInfo + builder.Services.AddEndpointsApiExplorer(); + builder.Services.AddSwaggerGen(options => { - Title = "Loopless API", - Version = "v1", - Description = "Loopless (FreelanceHub) backend. Use /auth/login to obtain an access token, then click Authorize and paste it as 'Bearer {token}'.", - }); + options.SwaggerDoc("v1", new OpenApiInfo + { + Title = "Loopless API", + Version = "v1", + Description = "Loopless (FreelanceHub) backend. Use /auth/login to obtain an access token, then click Authorize and paste it as 'Bearer {token}'.", + }); - var jwtScheme = new OpenApiSecurityScheme - { - Name = "Authorization", - Description = "Keycloak-issued JWT. Enter 'Bearer {access_token}'.", - In = ParameterLocation.Header, - Type = SecuritySchemeType.Http, - Scheme = "bearer", - BearerFormat = "JWT", - Reference = new OpenApiReference + var jwtScheme = new OpenApiSecurityScheme { - Type = ReferenceType.SecurityScheme, - Id = "Bearer", - }, - }; + Name = "Authorization", + Description = "Keycloak-issued JWT. Enter 'Bearer {access_token}'.", + In = ParameterLocation.Header, + Type = SecuritySchemeType.Http, + Scheme = "bearer", + BearerFormat = "JWT", + Reference = new OpenApiReference + { + Type = ReferenceType.SecurityScheme, + Id = "Bearer", + }, + }; - options.AddSecurityDefinition("Bearer", jwtScheme); - options.AddSecurityRequirement(new OpenApiSecurityRequirement - { - [jwtScheme] = Array.Empty(), + options.AddSecurityDefinition("Bearer", jwtScheme); + options.AddSecurityRequirement(new OpenApiSecurityRequirement + { + [jwtScheme] = Array.Empty(), + }); }); - }); + } var app = builder.Build(); @@ -203,6 +232,8 @@ await Loopless.Infrastructure.Persistence.Seeds.AdminSeeder.SeedAsync(app.Services); app.UseMiddleware(); + // Always the structured exception handler — never the developer exception page. No stack + // traces or developer error pages are exposed in any environment, Production included. app.UseExceptionHandler(); app.UseSerilogRequestLogging(opts => @@ -215,19 +246,33 @@ app.UseAuthentication(); app.UseAuthorization(); - app.UseSwagger(); - app.UseSwaggerUI(options => - { - options.SwaggerEndpoint("/swagger/v1/swagger.json", "Loopless API v1"); - options.DocumentTitle = "Loopless API"; - }); - + // Swagger UI + /swagger/v1/swagger.json are mapped ONLY in Development. In Production these + // routes (and /api-docs) return 404 — no API schema disclosure. if (app.Environment.IsDevelopment()) { + app.UseSwagger(); + app.UseSwaggerUI(options => + { + options.SwaggerEndpoint("/swagger/v1/swagger.json", "Loopless API v1"); + options.DocumentTitle = "Loopless API"; + }); + app.MapGet("/", () => Results.Redirect("/swagger")).ExcludeFromDescription(); } - app.MapMetrics("/metrics"); + // Prometheus scrape endpoint. Snippet-based path blocking is disabled on the prod + // ingress, so restrict /metrics by Host instead: the in-cluster scrape reaches the + // pod directly as "loopless-backend:8080" (allowed), while public traffic arrives as + // the ingress host (api.) which is not in the allowlist and gets 404 — keeping + // metrics off the public internet without breaking scraping. Unset (dev/local) = open. + var metricsEndpoint = app.MapMetrics("/metrics"); + var metricsAllowedHosts = app.Configuration + .GetSection("Metrics:AllowedHosts").Get(); + if (metricsAllowedHosts is { Length: > 0 }) + { + metricsEndpoint.RequireHost(metricsAllowedHosts); + } + app.MapHealthChecks("/health"); app.MapHealthChecks("/health/ready", new HealthCheckOptions { @@ -288,12 +333,17 @@ recurringJobs.AddOrUpdate( GitHubSyncJob.RecurringJobId, job => job.SyncAllAsync(CancellationToken.None), - "*/30 * * * *"); + "0 */6 * * *"); recurringJobs.AddOrUpdate( UnreadMessageEmailJob.RecurringJobId, job => job.RunAsync(CancellationToken.None), "*/5 * * * *"); + + recurringJobs.AddOrUpdate( + OrphanCleanupJob.RecurringJobId, + job => job.RunAsync(CancellationToken.None), + Cron.Daily); } app.Run(); @@ -325,11 +375,6 @@ static void ConfigureAzureKeyVault(WebApplicationBuilder builder) public partial class Program; -static class MyConcreteJob -{ - public static Task ExecuteAsync() => Task.CompletedTask; -} - sealed class SensitiveDataDestructuringPolicy : IDestructuringPolicy { private static readonly HashSet SensitiveNames = new(StringComparer.OrdinalIgnoreCase) diff --git a/backend/src/Loopless.Api/RateLimit/RedisFixedWindowRateLimiter.cs b/backend/src/Loopless.Api/RateLimit/RedisFixedWindowRateLimiter.cs index 6ee4876..39e22de 100644 --- a/backend/src/Loopless.Api/RateLimit/RedisFixedWindowRateLimiter.cs +++ b/backend/src/Loopless.Api/RateLimit/RedisFixedWindowRateLimiter.cs @@ -1,12 +1,15 @@ using System.Threading.RateLimiting; +using Prometheus; using StackExchange.Redis; namespace Loopless.Api.RateLimit; /// /// Redis-backed fixed-window rate limiter. Counts requests via Redis INCR+EXPIRE. -/// Falls back to granting the request if Redis is unavailable (fail-open). -/// Supports distributed deployments — each replica shares the same counter per user. +/// Supports two Redis-failure strategies per policy: +/// fail-open (failClosed=false) — grant the request; preserves availability. +/// fail-closed (failClosed=true) — deny with 503; prevents unenforced abuse during outages. +/// Supports distributed deployments — each replica shares the same Redis counter per user. /// internal sealed class RedisFixedWindowRateLimiter : RateLimiter { @@ -14,6 +17,13 @@ internal sealed class RedisFixedWindowRateLimiter : RateLimiter private readonly string _key; private readonly int _permitLimit; private readonly int _windowSeconds; + private readonly bool _failClosed; + private readonly string _policyName; + + private static readonly Counter FallbackTotal = Metrics.CreateCounter( + "ratelimiter_redis_fallback_total", + "Incremented each time Redis is unreachable and the rate limiter falls back.", + new CounterConfiguration { LabelNames = ["policy"] }); private const string IncrScript = @" local current = redis.call('INCR', KEYS[1]) @@ -22,12 +32,20 @@ internal sealed class RedisFixedWindowRateLimiter : RateLimiter end return current"; - internal RedisFixedWindowRateLimiter(IConnectionMultiplexer redis, string key, int permitLimit, TimeSpan window) + internal RedisFixedWindowRateLimiter( + IConnectionMultiplexer redis, + string key, + int permitLimit, + TimeSpan window, + bool failClosed = false, + string policyName = "unknown") { _db = redis.GetDatabase(); _key = key; _permitLimit = permitLimit; _windowSeconds = (int)window.TotalSeconds; + _failClosed = failClosed; + _policyName = policyName; } public override TimeSpan? IdleDuration => null; @@ -50,8 +68,8 @@ protected override async ValueTask AcquireAsyncCore(int permitCo } catch { - // Redis unavailable: fail open so a Redis outage does not block requests. - return new GrantedLease(); + FallbackTotal.WithLabels(_policyName).Inc(); + return _failClosed ? new ServiceUnavailableLease() : new GrantedLease(); } } @@ -86,4 +104,26 @@ public override bool TryGetMetadata(string metadataName, out object? metadata) return false; } } + + // Returned when Redis is unreachable and failClosed=true. Sets a custom metadata flag + // so OnRejected can distinguish a 503 (infra failure) from a 429 (limit exceeded). + private sealed class ServiceUnavailableLease : RateLimitLease + { + internal const string MetadataKey = "redis_unavailable"; + + public override bool IsAcquired => false; + public override IEnumerable MetadataNames => [MetadataKey]; + + public override bool TryGetMetadata(string metadataName, out object? metadata) + { + if (metadataName == MetadataKey) + { + metadata = true; + return true; + } + + metadata = null; + return false; + } + } } diff --git a/backend/src/Loopless.Application/DTOs/SummaryJobStatusDto.cs b/backend/src/Loopless.Application/DTOs/SummaryJobStatusDto.cs new file mode 100644 index 0000000..afb053c --- /dev/null +++ b/backend/src/Loopless.Application/DTOs/SummaryJobStatusDto.cs @@ -0,0 +1,7 @@ +namespace Loopless.Application.DTOs; + +public sealed record SummaryJobStatusDto( + string JobId, + string Status, + ProjectSummaryDto? Summary = null, + string? FailureReason = null); diff --git a/backend/src/Loopless.Application/DTOs/UserSearchResultDto.cs b/backend/src/Loopless.Application/DTOs/UserSearchResultDto.cs index 0485458..26b72e2 100644 --- a/backend/src/Loopless.Application/DTOs/UserSearchResultDto.cs +++ b/backend/src/Loopless.Application/DTOs/UserSearchResultDto.cs @@ -5,6 +5,5 @@ namespace Loopless.Application.DTOs; public sealed record UserSearchResultDto( Guid Id, string Name, - string Email, UserRole Role, string? AvatarUrl); diff --git a/backend/src/Loopless.Application/Features/Admin/DeleteUser/DeleteUserCommandHandler.cs b/backend/src/Loopless.Application/Features/Admin/DeleteUser/DeleteUserCommandHandler.cs index 873c2fc..3fe347c 100644 --- a/backend/src/Loopless.Application/Features/Admin/DeleteUser/DeleteUserCommandHandler.cs +++ b/backend/src/Loopless.Application/Features/Admin/DeleteUser/DeleteUserCommandHandler.cs @@ -46,6 +46,22 @@ public async Task Handle(DeleteUserCommand request, CancellationToken canc target.AvatarUrl = null; target.UpdatedAt = DateTimeOffset.UtcNow; + // Qualify ExecuteDeleteAsync explicitly: in this project both the core + // EntityFrameworkQueryableExtensions and RelationalQueryableExtensions are in scope + // (Microsoft.EntityFrameworkCore + Pgvector.EntityFrameworkCore -> Relational), which + // makes the unqualified extension call ambiguous. The core overload is provider-agnostic. + await EntityFrameworkQueryableExtensions.ExecuteDeleteAsync( + db.Embeddings + .Where(e => e.EntityId == target.Id && + (e.EntityType == EmbeddingEntityType.FreelancerProfile || + e.EntityType == EmbeddingEntityType.EnterpriseProfile)), + cancellationToken); + + await EntityFrameworkQueryableExtensions.ExecuteDeleteAsync( + db.ContentFlags + .Where(f => f.EntityType == "User" && f.EntityId == target.Id), + cancellationToken); + var newValues = new { target.Email, diff --git a/backend/src/Loopless.Application/Features/Messages/Send/SendMessageCommandHandler.cs b/backend/src/Loopless.Application/Features/Messages/Send/SendMessageCommandHandler.cs index d16316e..7a1bf3a 100644 --- a/backend/src/Loopless.Application/Features/Messages/Send/SendMessageCommandHandler.cs +++ b/backend/src/Loopless.Application/Features/Messages/Send/SendMessageCommandHandler.cs @@ -51,7 +51,11 @@ public async Task Handle(SendMessageCommand request, CancellationTok message.ReadAt, message.CreatedAt); - await notifier.NotifyMessageSentAsync(dto, cancellationToken); + var recipientId = conversation.Participant1Id == me.Id + ? conversation.Participant2Id + : conversation.Participant1Id; + + await notifier.NotifyMessageSentAsync(dto, recipientId, cancellationToken); return dto; } diff --git a/backend/src/Loopless.Application/Features/Moderation/ReviewFlag/ReviewFlagCommandHandler.cs b/backend/src/Loopless.Application/Features/Moderation/ReviewFlag/ReviewFlagCommandHandler.cs index 76a1f6b..cc0e235 100644 --- a/backend/src/Loopless.Application/Features/Moderation/ReviewFlag/ReviewFlagCommandHandler.cs +++ b/backend/src/Loopless.Application/Features/Moderation/ReviewFlag/ReviewFlagCommandHandler.cs @@ -37,10 +37,11 @@ public async Task Handle(ReviewFlagCommand request, Cancellation flag.UpdatedAt = DateTimeOffset.UtcNow; var contentOwnerId = await ResolveContentOwnerAsync(flag, cancellationToken); + var isProjectFlag = string.Equals(flag.EntityType, "Project", StringComparison.OrdinalIgnoreCase); - if (contentOwnerId.HasValue) + if (request.NewStatus == ContentFlagStatus.Warning) { - if (request.NewStatus == ContentFlagStatus.Warning) + if (contentOwnerId.HasValue) { await notifications.SendAsync( contentOwnerId.Value, @@ -48,24 +49,41 @@ await notifications.SendAsync( $"Your {flag.EntityType.ToLowerInvariant()} has been flagged and reviewed. A warning has been issued.", cancellationToken); } - else if (request.NewStatus == ContentFlagStatus.Suspended) - { - var target = await db.Users - .FirstOrDefaultAsync(u => u.Id == contentOwnerId.Value, cancellationToken); + } + else if (request.NewStatus == ContentFlagStatus.Suspended) + { + // Take down the offending content whenever an admin suspends a flag — independent of + // whether a content owner could be resolved, so project takedowns can't be skipped. + await TakedownContentAsync(flag, admin.Id, cancellationToken); - if (target is not null && !target.IsDeleted && target.Role != UserRole.Admin) + if (contentOwnerId.HasValue) + { + if (isProjectFlag) { - target.IsSuspended = true; - target.UpdatedAt = DateTimeOffset.UtcNow; - + // A single flagged project is removed, but the owner's account stays active. await notifications.SendAsync( contentOwnerId.Value, - "Account Suspended", - $"Your account has been suspended following a moderation review of your {flag.EntityType.ToLowerInvariant()}.", + "Project Removed", + "Your project has been removed following a moderation review.", cancellationToken); } - - await TakedownContentAsync(flag, admin.Id, cancellationToken); + else + { + var target = await db.Users + .FirstOrDefaultAsync(u => u.Id == contentOwnerId.Value, cancellationToken); + + if (target is not null && !target.IsDeleted && target.Role != UserRole.Admin) + { + target.IsSuspended = true; + target.UpdatedAt = DateTimeOffset.UtcNow; + + await notifications.SendAsync( + contentOwnerId.Value, + "Account Suspended", + $"Your account has been suspended following a moderation review of your {flag.EntityType.ToLowerInvariant()}.", + cancellationToken); + } + } } } @@ -104,6 +122,15 @@ await notifications.SendAsync( return exists ? flag.EntityId : null; } + if (string.Equals(flag.EntityType, "Project", StringComparison.OrdinalIgnoreCase)) + { + var ownerId = await db.Projects + .Where(p => p.Id == flag.EntityId) + .Select(p => (Guid?)p.OwnerId) + .FirstOrDefaultAsync(cancellationToken); + return ownerId; + } + return null; } diff --git a/backend/src/Loopless.Application/Features/Projects/GenerateSummary/GenerateProjectSummaryCommand.cs b/backend/src/Loopless.Application/Features/Projects/GenerateSummary/GenerateProjectSummaryCommand.cs index 8a0dad3..cb07204 100644 --- a/backend/src/Loopless.Application/Features/Projects/GenerateSummary/GenerateProjectSummaryCommand.cs +++ b/backend/src/Loopless.Application/Features/Projects/GenerateSummary/GenerateProjectSummaryCommand.cs @@ -3,4 +3,4 @@ namespace Loopless.Application.Features.Projects.GenerateSummary; -public sealed record GenerateProjectSummaryCommand(Guid ProjectId) : IRequest; +public sealed record GenerateProjectSummaryCommand(Guid ProjectId) : IRequest; diff --git a/backend/src/Loopless.Application/Features/Projects/GenerateSummary/GenerateProjectSummaryCommandHandler.cs b/backend/src/Loopless.Application/Features/Projects/GenerateSummary/GenerateProjectSummaryCommandHandler.cs index d872050..33c71f0 100644 --- a/backend/src/Loopless.Application/Features/Projects/GenerateSummary/GenerateProjectSummaryCommandHandler.cs +++ b/backend/src/Loopless.Application/Features/Projects/GenerateSummary/GenerateProjectSummaryCommandHandler.cs @@ -1,4 +1,4 @@ -using Loopless.Application.Common.Exceptions; +using Loopless.Application.Common.Exceptions; using Loopless.Application.DTOs; using Loopless.Application.Interfaces; using Loopless.Domain.Entities; @@ -11,9 +11,9 @@ namespace Loopless.Application.Features.Projects.GenerateSummary; public sealed class GenerateProjectSummaryCommandHandler( IAppDbContext db, ICurrentUser currentUser, - IAiSummaryService aiSummary) : IRequestHandler + ISummaryJobEnqueuer jobEnqueuer) : IRequestHandler { - public async Task Handle(GenerateProjectSummaryCommand request, CancellationToken cancellationToken) + public async Task Handle(GenerateProjectSummaryCommand request, CancellationToken cancellationToken) { var keycloakId = currentUser.KeycloakId ?? throw new UnauthorizedException("Missing subject claim."); @@ -37,47 +37,20 @@ public async Task Handle(GenerateProjectSummaryCommand reques if (!isOwner && !isMember) throw new ForbiddenException("Only project members can generate summaries."); - var commitMessages = await db.GitHubCommits - .Where(c => c.ProjectId == project.Id) - .OrderByDescending(c => c.CommittedAt) - .Take(50) - .Select(c => $"{c.Author}: {c.Message}") - .ToListAsync(cancellationToken); - - var standupEntries = await db.Standups - .Where(s => s.ProjectId == project.Id) - .OrderByDescending(s => s.CreatedAt) - .Take(10) - .Join( - db.Users, - s => s.FreelancerId, - u => u.Id, - (s, u) => $"[{u.Name}] Did: {s.WhatDid} | Plan: {s.WhatPlan} | Blockers: {s.Blockers}") - .ToListAsync(cancellationToken); - - var content = await aiSummary.GenerateSummaryAsync( - project.Title, - project.Description, - commitMessages, - standupEntries, - cancellationToken); - var summary = new ProjectSummary { ProjectId = project.Id, - Content = content, - ModelUsed = "gpt-4o-mini", - GeneratedAt = DateTimeOffset.UtcNow, + Status = SummaryStatus.Queued, }; db.ProjectSummaries.Add(summary); await db.SaveChangesAsync(cancellationToken); - return new ProjectSummaryDto( - summary.Id, - summary.ProjectId, - summary.Content, - summary.ModelUsed, - summary.GeneratedAt); + var jobId = jobEnqueuer.Enqueue(project.Id, summary.Id); + + summary.HangfireJobId = jobId; + await db.SaveChangesAsync(cancellationToken); + + return new SummaryJobStatusDto(jobId, "queued"); } } diff --git a/backend/src/Loopless.Application/Features/Projects/GetCommits/GetProjectCommitsQueryHandler.cs b/backend/src/Loopless.Application/Features/Projects/GetCommits/GetProjectCommitsQueryHandler.cs index db811e8..0e0cf12 100644 --- a/backend/src/Loopless.Application/Features/Projects/GetCommits/GetProjectCommitsQueryHandler.cs +++ b/backend/src/Loopless.Application/Features/Projects/GetCommits/GetProjectCommitsQueryHandler.cs @@ -1,21 +1,39 @@ using Loopless.Application.Common.Exceptions; using Loopless.Application.DTOs; using Loopless.Application.Interfaces; +using Loopless.Domain.Enums; using MediatR; using Microsoft.EntityFrameworkCore; namespace Loopless.Application.Features.Projects.GetCommits; public sealed class GetProjectCommitsQueryHandler( - IAppDbContext db) : IRequestHandler> + IAppDbContext db, + ICurrentUser currentUser) : IRequestHandler> { public async Task> Handle(GetProjectCommitsQuery request, CancellationToken cancellationToken) { - var projectExists = await db.Projects - .AnyAsync(p => p.Id == request.ProjectId, cancellationToken); + var keycloakId = currentUser.KeycloakId + ?? throw new UnauthorizedException("Missing subject claim."); - if (!projectExists) - throw new NotFoundException("Project not found."); + var user = await db.Users + .FirstOrDefaultAsync(u => u.KeycloakId == keycloakId, cancellationToken) + ?? throw new NotFoundException("User not found."); + + var project = await db.Projects + .FirstOrDefaultAsync(p => p.Id == request.ProjectId, cancellationToken) + ?? throw new NotFoundException("Project not found."); + + var isOwner = project.OwnerId == user.Id; + var isMember = await db.ProjectInvitations + .AnyAsync( + i => i.ProjectId == project.Id + && (i.EnterpriseId == user.Id || i.FreelancerId == user.Id) + && i.Status == InvitationStatus.Accepted, + cancellationToken); + + if (!isOwner && !isMember) + throw new ForbiddenException("Only project members can view commits."); var baseQuery = db.GitHubCommits.Where(c => c.ProjectId == request.ProjectId); var totalCount = await baseQuery.CountAsync(cancellationToken); diff --git a/backend/src/Loopless.Application/Features/Projects/GetSummaries/GetProjectSummariesQueryHandler.cs b/backend/src/Loopless.Application/Features/Projects/GetSummaries/GetProjectSummariesQueryHandler.cs index 775f3a1..93855a5 100644 --- a/backend/src/Loopless.Application/Features/Projects/GetSummaries/GetProjectSummariesQueryHandler.cs +++ b/backend/src/Loopless.Application/Features/Projects/GetSummaries/GetProjectSummariesQueryHandler.cs @@ -38,14 +38,14 @@ public async Task> Handle(GetProjectSummariesQuer var offset = (request.Page - 1) * request.PageSize; var totalCount = await db.ProjectSummaries - .CountAsync(s => s.ProjectId == project.Id, cancellationToken); + .CountAsync(s => s.ProjectId == project.Id && s.Status == SummaryStatus.Completed, cancellationToken); var items = await db.ProjectSummaries - .Where(s => s.ProjectId == project.Id) + .Where(s => s.ProjectId == project.Id && s.Status == SummaryStatus.Completed) .OrderByDescending(s => s.GeneratedAt) .Skip(offset) .Take(request.PageSize) - .Select(s => new ProjectSummaryDto(s.Id, s.ProjectId, s.Content, s.ModelUsed, s.GeneratedAt)) + .Select(s => new ProjectSummaryDto(s.Id, s.ProjectId, s.Content!, s.ModelUsed!, s.GeneratedAt)) .ToListAsync(cancellationToken); return new PagedResult(items, totalCount, request.Page, request.PageSize); diff --git a/backend/src/Loopless.Application/Features/Projects/GetSummaryStatus/GetProjectSummaryStatusQuery.cs b/backend/src/Loopless.Application/Features/Projects/GetSummaryStatus/GetProjectSummaryStatusQuery.cs new file mode 100644 index 0000000..35aa2ba --- /dev/null +++ b/backend/src/Loopless.Application/Features/Projects/GetSummaryStatus/GetProjectSummaryStatusQuery.cs @@ -0,0 +1,6 @@ +using Loopless.Application.DTOs; +using MediatR; + +namespace Loopless.Application.Features.Projects.GetSummaryStatus; + +public sealed record GetProjectSummaryStatusQuery(Guid ProjectId) : IRequest; diff --git a/backend/src/Loopless.Application/Features/Projects/GetSummaryStatus/GetProjectSummaryStatusQueryHandler.cs b/backend/src/Loopless.Application/Features/Projects/GetSummaryStatus/GetProjectSummaryStatusQueryHandler.cs new file mode 100644 index 0000000..f5291dd --- /dev/null +++ b/backend/src/Loopless.Application/Features/Projects/GetSummaryStatus/GetProjectSummaryStatusQueryHandler.cs @@ -0,0 +1,61 @@ +using Loopless.Application.Common.Exceptions; +using Loopless.Application.DTOs; +using Loopless.Application.Interfaces; +using Loopless.Domain.Enums; +using MediatR; +using Microsoft.EntityFrameworkCore; + +namespace Loopless.Application.Features.Projects.GetSummaryStatus; + +public sealed class GetProjectSummaryStatusQueryHandler( + IAppDbContext db, + ICurrentUser currentUser) : IRequestHandler +{ + public async Task Handle(GetProjectSummaryStatusQuery request, CancellationToken cancellationToken) + { + var keycloakId = currentUser.KeycloakId + ?? throw new UnauthorizedException("Missing subject claim."); + + var user = await db.Users + .FirstOrDefaultAsync(u => u.KeycloakId == keycloakId, cancellationToken) + ?? throw new NotFoundException("User not found."); + + var project = await db.Projects + .FirstOrDefaultAsync(p => p.Id == request.ProjectId, cancellationToken) + ?? throw new NotFoundException("Project not found."); + + var isOwner = project.OwnerId == user.Id; + var isMember = await db.ProjectInvitations + .AnyAsync( + i => i.ProjectId == project.Id + && (i.EnterpriseId == user.Id || i.FreelancerId == user.Id) + && i.Status == InvitationStatus.Accepted, + cancellationToken); + + if (!isOwner && !isMember) + throw new ForbiddenException("Only project members can view summary status."); + + var summary = await db.ProjectSummaries + .Where(s => s.ProjectId == project.Id) + .OrderByDescending(s => s.CreatedAt) + .FirstOrDefaultAsync(cancellationToken) + ?? throw new NotFoundException("No summary job found for this project."); + + ProjectSummaryDto? summaryDto = null; + if (summary.Status == SummaryStatus.Completed) + { + summaryDto = new ProjectSummaryDto( + summary.Id, + summary.ProjectId, + summary.Content!, + summary.ModelUsed!, + summary.GeneratedAt); + } + + return new SummaryJobStatusDto( + summary.HangfireJobId ?? string.Empty, + summary.Status.ToString().ToLowerInvariant(), + summaryDto, + summary.FailureReason); + } +} diff --git a/backend/src/Loopless.Application/Features/Projects/Leave/LeaveProjectCommandHandler.cs b/backend/src/Loopless.Application/Features/Projects/Leave/LeaveProjectCommandHandler.cs index b7c6cfc..56ec9fe 100644 --- a/backend/src/Loopless.Application/Features/Projects/Leave/LeaveProjectCommandHandler.cs +++ b/backend/src/Loopless.Application/Features/Projects/Leave/LeaveProjectCommandHandler.cs @@ -39,13 +39,18 @@ public async Task Handle(LeaveProjectCommand request, CancellationToken cancella project.IsDeleted = true; + // The owner is closing the project — notify the accepted freelancer members, not the + // owner themselves (invitation.EnterpriseId is the owner on owner-created projects). foreach (var invitation in acceptedInvitations) { - await notifications.SendAsync( - invitation.EnterpriseId, - "Freelancer Left Project", - $"Freelancer {user.Name} has left project {project.Title}.", - cancellationToken); + if (invitation.FreelancerId is Guid freelancerId) + { + await notifications.SendAsync( + freelancerId, + "Project Closed", + $"Project {project.Title} has been closed by the owner.", + cancellationToken); + } } } else if (user.Role == UserRole.Freelancer) diff --git a/backend/src/Loopless.Application/Features/Standups/GetAnalytics/ExportStandupsCsvQueryHandler.cs b/backend/src/Loopless.Application/Features/Standups/GetAnalytics/ExportStandupsCsvQueryHandler.cs index 744b69e..05f1ff3 100644 --- a/backend/src/Loopless.Application/Features/Standups/GetAnalytics/ExportStandupsCsvQueryHandler.cs +++ b/backend/src/Loopless.Application/Features/Standups/GetAnalytics/ExportStandupsCsvQueryHandler.cs @@ -1,18 +1,39 @@ using Loopless.Application.Common.Exceptions; using Loopless.Application.DTOs; using Loopless.Application.Interfaces; +using Loopless.Domain.Enums; using MediatR; using Microsoft.EntityFrameworkCore; namespace Loopless.Application.Features.Standups.GetAnalytics; public sealed class ExportStandupsCsvQueryHandler( - IAppDbContext db) : IRequestHandler> + IAppDbContext db, + ICurrentUser currentUser) : IRequestHandler> { public async Task> Handle(ExportStandupsCsvQuery request, CancellationToken cancellationToken) { - var projectExists = await db.Projects.AnyAsync(p => p.Id == request.ProjectId, cancellationToken); - if (!projectExists) throw new NotFoundException("Project not found."); + var keycloakId = currentUser.KeycloakId + ?? throw new UnauthorizedException("Missing subject claim."); + + var user = await db.Users + .FirstOrDefaultAsync(u => u.KeycloakId == keycloakId, cancellationToken) + ?? throw new NotFoundException("User not found."); + + var project = await db.Projects + .FirstOrDefaultAsync(p => p.Id == request.ProjectId, cancellationToken) + ?? throw new NotFoundException("Project not found."); + + var isOwner = project.OwnerId == user.Id; + var isMember = await db.ProjectInvitations + .AnyAsync( + i => i.ProjectId == project.Id + && (i.EnterpriseId == user.Id || i.FreelancerId == user.Id) + && i.Status == InvitationStatus.Accepted, + cancellationToken); + + if (!isOwner && !isMember) + throw new ForbiddenException("Only project members can export standups."); return await db.Standups .Where(s => s.ProjectId == request.ProjectId) diff --git a/backend/src/Loopless.Application/Features/Standups/GetAnalytics/GetStandupAnalyticsQueryHandler.cs b/backend/src/Loopless.Application/Features/Standups/GetAnalytics/GetStandupAnalyticsQueryHandler.cs index 2c80523..7b8fcf8 100644 --- a/backend/src/Loopless.Application/Features/Standups/GetAnalytics/GetStandupAnalyticsQueryHandler.cs +++ b/backend/src/Loopless.Application/Features/Standups/GetAnalytics/GetStandupAnalyticsQueryHandler.cs @@ -2,13 +2,15 @@ using Loopless.Application.Common.Exceptions; using Loopless.Application.DTOs; using Loopless.Application.Interfaces; +using Loopless.Domain.Enums; using MediatR; using Microsoft.EntityFrameworkCore; namespace Loopless.Application.Features.Standups.GetAnalytics; public sealed partial class GetStandupAnalyticsQueryHandler( - IAppDbContext db) : IRequestHandler + IAppDbContext db, + ICurrentUser currentUser) : IRequestHandler { private const int MinDays = 1; private const int MaxDays = 90; @@ -30,10 +32,28 @@ public async Task Handle(GetStandupAnalyticsQuery request, { var days = Math.Clamp(request.Days <= 0 ? 7 : request.Days, MinDays, MaxDays); + var keycloakId = currentUser.KeycloakId + ?? throw new UnauthorizedException("Missing subject claim."); + + var user = await db.Users + .FirstOrDefaultAsync(u => u.KeycloakId == keycloakId, cancellationToken) + ?? throw new NotFoundException("User not found."); + var project = await db.Projects .FirstOrDefaultAsync(p => p.Id == request.ProjectId, cancellationToken) ?? throw new NotFoundException("Project not found."); + var isOwner = project.OwnerId == user.Id; + var isMember = await db.ProjectInvitations + .AnyAsync( + i => i.ProjectId == project.Id + && (i.EnterpriseId == user.Id || i.FreelancerId == user.Id) + && i.Status == InvitationStatus.Accepted, + cancellationToken); + + if (!isOwner && !isMember) + throw new ForbiddenException("Only project members can view standup analytics."); + var today = DateTimeOffset.UtcNow.Date; var windowEnd = new DateTimeOffset(today, TimeSpan.Zero).AddDays(1); var windowStart = new DateTimeOffset(today.AddDays(-days + 1), TimeSpan.Zero); diff --git a/backend/src/Loopless.Application/Features/Standups/GetFeed/GetStandupFeedQueryHandler.cs b/backend/src/Loopless.Application/Features/Standups/GetFeed/GetStandupFeedQueryHandler.cs index ad46e1c..9b84893 100644 --- a/backend/src/Loopless.Application/Features/Standups/GetFeed/GetStandupFeedQueryHandler.cs +++ b/backend/src/Loopless.Application/Features/Standups/GetFeed/GetStandupFeedQueryHandler.cs @@ -1,15 +1,40 @@ -using Loopless.Application.DTOs; +using Loopless.Application.Common.Exceptions; +using Loopless.Application.DTOs; using Loopless.Application.Interfaces; +using Loopless.Domain.Enums; using MediatR; using Microsoft.EntityFrameworkCore; namespace Loopless.Application.Features.Standups.GetFeed; public sealed class GetStandupFeedQueryHandler( - IAppDbContext db) : IRequestHandler> + IAppDbContext db, + ICurrentUser currentUser) : IRequestHandler> { public async Task> Handle(GetStandupFeedQuery request, CancellationToken cancellationToken) { + var keycloakId = currentUser.KeycloakId + ?? throw new UnauthorizedException("Missing subject claim."); + + var user = await db.Users + .FirstOrDefaultAsync(u => u.KeycloakId == keycloakId, cancellationToken) + ?? throw new NotFoundException("User not found."); + + var project = await db.Projects + .FirstOrDefaultAsync(p => p.Id == request.ProjectId, cancellationToken) + ?? throw new NotFoundException("Project not found."); + + var isOwner = project.OwnerId == user.Id; + var isMember = await db.ProjectInvitations + .AnyAsync( + i => i.ProjectId == project.Id + && (i.EnterpriseId == user.Id || i.FreelancerId == user.Id) + && i.Status == InvitationStatus.Accepted, + cancellationToken); + + if (!isOwner && !isMember) + throw new ForbiddenException("Only project members can view the standup feed."); + var baseQuery = db.Standups.Where(s => s.ProjectId == request.ProjectId); var totalCount = await baseQuery.CountAsync(cancellationToken); diff --git a/backend/src/Loopless.Application/Features/Standups/Submit/SubmitStandupCommandHandler.cs b/backend/src/Loopless.Application/Features/Standups/Submit/SubmitStandupCommandHandler.cs index 2a49276..83951e5 100644 --- a/backend/src/Loopless.Application/Features/Standups/Submit/SubmitStandupCommandHandler.cs +++ b/backend/src/Loopless.Application/Features/Standups/Submit/SubmitStandupCommandHandler.cs @@ -11,7 +11,6 @@ namespace Loopless.Application.Features.Standups.Submit; public sealed class SubmitStandupCommandHandler( IAppDbContext db, ICurrentUser currentUser, - INlpService nlp, IEventPublisher publisher) : IRequestHandler { public async Task Handle(SubmitStandupCommand request, CancellationToken cancellationToken) @@ -39,10 +38,6 @@ public async Task Handle(SubmitStandupCommand request, CancellationT if (duplicate) throw new ConflictException("Standup already submitted for this project today."); - var sentiment = string.IsNullOrWhiteSpace(request.Blockers) - ? (Sentiment?)null - : nlp.AnalyseSentiment(request.Blockers); - var project = await db.Projects .Where(p => p.Id == request.ProjectId) .Select(p => new { p.Id, p.Title, p.OwnerId }) @@ -56,13 +51,13 @@ public async Task Handle(SubmitStandupCommand request, CancellationT WhatDid = request.DidYesterday, WhatPlan = request.WillToday, Blockers = request.Blockers, - BlockerSentiment = sentiment, + BlockerSentiment = null, }; db.Standups.Add(standup); await db.SaveChangesAsync(cancellationToken); - await publisher.PublishAsync("standup.submitted", new + await publisher.PublishAsync("standups.submitted", new { StandupId = standup.Id, standup.ProjectId, diff --git a/backend/src/Loopless.Application/Features/Users/SearchUsers/SearchUsersQueryHandler.cs b/backend/src/Loopless.Application/Features/Users/SearchUsers/SearchUsersQueryHandler.cs index 50cab41..1b39b6a 100644 --- a/backend/src/Loopless.Application/Features/Users/SearchUsers/SearchUsersQueryHandler.cs +++ b/backend/src/Loopless.Application/Features/Users/SearchUsers/SearchUsersQueryHandler.cs @@ -37,7 +37,6 @@ public async Task> Handle(SearchUsersQuery re .Select(u => new UserSearchResultDto( u.Id, u.Name, - u.Email, u.Role, u.AvatarUrl)) .ToListAsync(cancellationToken); diff --git a/backend/src/Loopless.Application/Interfaces/IMessagingNotifier.cs b/backend/src/Loopless.Application/Interfaces/IMessagingNotifier.cs index bddd935..5e9e8a9 100644 --- a/backend/src/Loopless.Application/Interfaces/IMessagingNotifier.cs +++ b/backend/src/Loopless.Application/Interfaces/IMessagingNotifier.cs @@ -4,7 +4,7 @@ namespace Loopless.Application.Interfaces; public interface IMessagingNotifier { - Task NotifyMessageSentAsync(MessageDto message, CancellationToken cancellationToken = default); + Task NotifyMessageSentAsync(MessageDto message, Guid recipientId, CancellationToken cancellationToken = default); Task NotifyMessageReadAsync( Guid conversationId, diff --git a/backend/src/Loopless.Application/Interfaces/ISummaryJobEnqueuer.cs b/backend/src/Loopless.Application/Interfaces/ISummaryJobEnqueuer.cs new file mode 100644 index 0000000..952acb4 --- /dev/null +++ b/backend/src/Loopless.Application/Interfaces/ISummaryJobEnqueuer.cs @@ -0,0 +1,6 @@ +namespace Loopless.Application.Interfaces; + +public interface ISummaryJobEnqueuer +{ + string Enqueue(Guid projectId, Guid summaryId); +} diff --git a/backend/src/Loopless.Domain/Entities/Project.cs b/backend/src/Loopless.Domain/Entities/Project.cs index cf04ca0..23c59e6 100644 --- a/backend/src/Loopless.Domain/Entities/Project.cs +++ b/backend/src/Loopless.Domain/Entities/Project.cs @@ -9,6 +9,7 @@ public class Project : BaseEntity public required string Title { get; set; } public string? Description { get; set; } public string? GitHubRepoUrl { get; set; } + public string? GitHubWebhookSecret { get; set; } public required ProjectStatus Status { get; set; } public bool StandupEnabled { get; set; } public bool StandupRemindersEnabled { get; set; } = true; diff --git a/backend/src/Loopless.Domain/Entities/ProjectSummary.cs b/backend/src/Loopless.Domain/Entities/ProjectSummary.cs index a04f106..4e2360a 100644 --- a/backend/src/Loopless.Domain/Entities/ProjectSummary.cs +++ b/backend/src/Loopless.Domain/Entities/ProjectSummary.cs @@ -1,13 +1,17 @@ using Loopless.Domain.Common; +using Loopless.Domain.Enums; namespace Loopless.Domain.Entities; public class ProjectSummary : BaseEntity { public required Guid ProjectId { get; set; } - public required string Content { get; set; } - public required string ModelUsed { get; set; } - public DateTimeOffset GeneratedAt { get; init; } = DateTimeOffset.UtcNow; + public string? Content { get; set; } + public string? ModelUsed { get; set; } + public DateTimeOffset GeneratedAt { get; set; } = DateTimeOffset.UtcNow; + public SummaryStatus Status { get; set; } = SummaryStatus.Queued; + public string? HangfireJobId { get; set; } + public string? FailureReason { get; set; } public Project? Project { get; set; } } diff --git a/backend/src/Loopless.Domain/Enums/SummaryStatus.cs b/backend/src/Loopless.Domain/Enums/SummaryStatus.cs new file mode 100644 index 0000000..f24ea6f --- /dev/null +++ b/backend/src/Loopless.Domain/Enums/SummaryStatus.cs @@ -0,0 +1,9 @@ +namespace Loopless.Domain.Enums; + +public enum SummaryStatus +{ + Queued = 0, + Running = 1, + Completed = 2, + Failed = 3, +} diff --git a/backend/src/Loopless.Infrastructure/BackgroundJobs/GitHubSyncJob.cs b/backend/src/Loopless.Infrastructure/BackgroundJobs/GitHubSyncJob.cs index 7bf4c49..1c6e990 100644 --- a/backend/src/Loopless.Infrastructure/BackgroundJobs/GitHubSyncJob.cs +++ b/backend/src/Loopless.Infrastructure/BackgroundJobs/GitHubSyncJob.cs @@ -1,6 +1,7 @@ using Hangfire; using Loopless.Application.Interfaces; using Loopless.Domain.Entities; +using Loopless.Domain.Enums; using Loopless.Infrastructure.GitHub; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; @@ -123,6 +124,21 @@ public async Task SyncProjectAsync(Guid projectId, CancellationToken cancellatio _logger.LogInformation("GitHubSyncJob: stored {Count} new commits for {ProjectId}", newCommits.Count, projectId); - _jobClient.Enqueue(job => job.GenerateAsync(projectId, CancellationToken.None)); + // Auto-sync also triggers an AI summary. Create the ProjectSummary row first so the job + // can resolve it by id (mirrors GenerateProjectSummaryCommandHandler); Hangfire injects + // the real PerformContext + cancellation token at run time (null!/None are placeholders). + var summary = new ProjectSummary + { + ProjectId = projectId, + Status = SummaryStatus.Queued, + }; + _db.ProjectSummaries.Add(summary); + await _db.SaveChangesAsync(cancellationToken); + + var jobId = _jobClient.Enqueue( + job => job.GenerateAsync(projectId, summary.Id, null!, CancellationToken.None)); + + summary.HangfireJobId = jobId; + await _db.SaveChangesAsync(cancellationToken); } } diff --git a/backend/src/Loopless.Infrastructure/BackgroundJobs/OrphanCleanupJob.cs b/backend/src/Loopless.Infrastructure/BackgroundJobs/OrphanCleanupJob.cs new file mode 100644 index 0000000..f2fa3b1 --- /dev/null +++ b/backend/src/Loopless.Infrastructure/BackgroundJobs/OrphanCleanupJob.cs @@ -0,0 +1,57 @@ +using Loopless.Application.Interfaces; +using Loopless.Domain.Enums; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Loopless.Infrastructure.BackgroundJobs; + +public sealed class OrphanCleanupJob( + IAppDbContext db, + ILogger logger) +{ + public const string RecurringJobId = "orphan-cleanup"; + + public async Task RunAsync(CancellationToken cancellationToken = default) + { + try + { + int deleted = 0; + + // Embeddings whose profile no longer exists (hard-delete bypass) + deleted += await db.Embeddings + .Where(e => e.EntityType == EmbeddingEntityType.FreelancerProfile + && !db.FreelancerProfiles.Any(fp => fp.UserId == e.EntityId)) + .ExecuteDeleteAsync(cancellationToken); + + deleted += await db.Embeddings + .Where(e => e.EntityType == EmbeddingEntityType.EnterpriseProfile + && !db.EnterpriseProfiles.Any(ep => ep.UserId == e.EntityId)) + .ExecuteDeleteAsync(cancellationToken); + + // db.Projects global filter excludes soft-deleted; orphaned project embeddings pruned here + deleted += await db.Embeddings + .Where(e => e.EntityType == EmbeddingEntityType.Project + && !db.Projects.Any(p => p.Id == e.EntityId)) + .ExecuteDeleteAsync(cancellationToken); + + // Content flags against soft-deleted or missing entities + // db.Users global filter excludes IsDeleted rows, so !Any(...) captures both soft-deleted and missing + deleted += await db.ContentFlags + .Where(f => f.EntityType == "User" + && !db.Users.Any(u => u.Id == f.EntityId)) + .ExecuteDeleteAsync(cancellationToken); + + deleted += await db.ContentFlags + .Where(f => f.EntityType == "Project" + && !db.Projects.Any(p => p.Id == f.EntityId)) + .ExecuteDeleteAsync(cancellationToken); + + logger.LogInformation("OrphanCleanupJob: pruned {Count} orphaned rows", deleted); + } + catch (Exception ex) + { + logger.LogError(ex, "OrphanCleanupJob: sweep failed"); + throw; + } + } +} diff --git a/backend/src/Loopless.Infrastructure/BackgroundJobs/SummaryGenerationJob.cs b/backend/src/Loopless.Infrastructure/BackgroundJobs/SummaryGenerationJob.cs index 55f0807..2689105 100644 --- a/backend/src/Loopless.Infrastructure/BackgroundJobs/SummaryGenerationJob.cs +++ b/backend/src/Loopless.Infrastructure/BackgroundJobs/SummaryGenerationJob.cs @@ -1,6 +1,7 @@ using Hangfire; +using Hangfire.Server; using Loopless.Application.Interfaces; -using Loopless.Domain.Entities; +using Loopless.Domain.Enums; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; @@ -11,11 +12,22 @@ public sealed class SummaryGenerationJob( IAiSummaryService aiSummary, ILogger logger) { - public const string JobIdPrefix = "summary-generation"; - [AutomaticRetry(Attempts = 3, DelaysInSeconds = [30, 120, 300])] - public async Task GenerateAsync(Guid projectId, CancellationToken cancellationToken = default) + public async Task GenerateAsync( + Guid projectId, + Guid summaryId, + PerformContext? performContext, + CancellationToken cancellationToken = default) { + var summary = await db.ProjectSummaries + .FirstOrDefaultAsync(s => s.Id == summaryId, cancellationToken); + + if (summary is null) + { + logger.LogWarning("SummaryGenerationJob: summary record {SummaryId} not found – skipping", summaryId); + return; + } + var project = await db.Projects .FirstOrDefaultAsync(p => p.Id == projectId, cancellationToken); @@ -25,26 +37,30 @@ public async Task GenerateAsync(Guid projectId, CancellationToken cancellationTo return; } - var commitMessages = await db.GitHubCommits - .Where(c => c.ProjectId == projectId) - .OrderByDescending(c => c.CommittedAt) - .Take(50) - .Select(c => $"{c.Author}: {c.Message}") - .ToListAsync(cancellationToken); - - var standupEntries = await db.Standups - .Where(s => s.ProjectId == projectId) - .OrderByDescending(s => s.CreatedAt) - .Take(10) - .Join( - db.Users, - s => s.FreelancerId, - u => u.Id, - (s, u) => $"[{u.Name}] Did: {s.WhatDid} | Plan: {s.WhatPlan} | Blockers: {s.Blockers}") - .ToListAsync(cancellationToken); + summary.Status = SummaryStatus.Running; + summary.FailureReason = null; + await db.SaveChangesAsync(cancellationToken); try { + var commitMessages = await db.GitHubCommits + .Where(c => c.ProjectId == projectId) + .OrderByDescending(c => c.CommittedAt) + .Take(50) + .Select(c => $"{c.Author}: {c.Message}") + .ToListAsync(cancellationToken); + + var standupEntries = await db.Standups + .Where(s => s.ProjectId == projectId) + .OrderByDescending(s => s.CreatedAt) + .Take(10) + .Join( + db.Users, + s => s.FreelancerId, + u => u.Id, + (s, u) => $"[{u.Name}] Did: {s.WhatDid} | Plan: {s.WhatPlan} | Blockers: {s.Blockers}") + .ToListAsync(cancellationToken); + var content = await aiSummary.GenerateSummaryAsync( project.Title, project.Description, @@ -52,22 +68,21 @@ public async Task GenerateAsync(Guid projectId, CancellationToken cancellationTo standupEntries, cancellationToken); - var summary = new ProjectSummary - { - ProjectId = projectId, - Content = content, - ModelUsed = "gpt-4o-mini", - GeneratedAt = DateTimeOffset.UtcNow, - }; - - db.ProjectSummaries.Add(summary); + summary.Content = content; + summary.ModelUsed = "gpt-4o-mini"; + summary.GeneratedAt = DateTimeOffset.UtcNow; + summary.Status = SummaryStatus.Completed; await db.SaveChangesAsync(cancellationToken); - logger.LogInformation("SummaryGenerationJob: generated summary for project {ProjectId}", projectId); + logger.LogInformation("SummaryGenerationJob: completed summary for project {ProjectId}", projectId); } - catch (HttpRequestException ex) + catch (Exception ex) { - logger.LogError(ex, "SummaryGenerationJob: OpenAI request failed for project {ProjectId}, will retry", projectId); + summary.Status = SummaryStatus.Failed; + summary.FailureReason = ex.Message; + await db.SaveChangesAsync(CancellationToken.None); + + logger.LogError(ex, "SummaryGenerationJob: failed for project {ProjectId} (summary {SummaryId})", projectId, summaryId); throw; } } diff --git a/backend/src/Loopless.Infrastructure/BackgroundJobs/SummaryJobEnqueuer.cs b/backend/src/Loopless.Infrastructure/BackgroundJobs/SummaryJobEnqueuer.cs new file mode 100644 index 0000000..a5ad875 --- /dev/null +++ b/backend/src/Loopless.Infrastructure/BackgroundJobs/SummaryJobEnqueuer.cs @@ -0,0 +1,10 @@ +using Hangfire; +using Loopless.Application.Interfaces; + +namespace Loopless.Infrastructure.BackgroundJobs; + +internal sealed class SummaryJobEnqueuer(IBackgroundJobClient jobs) : ISummaryJobEnqueuer +{ + public string Enqueue(Guid projectId, Guid summaryId) => + jobs.Enqueue(j => j.GenerateAsync(projectId, summaryId, null!, CancellationToken.None)); +} diff --git a/backend/src/Loopless.Infrastructure/DependencyInjection.cs b/backend/src/Loopless.Infrastructure/DependencyInjection.cs index e6d62c0..ac6d6d8 100644 --- a/backend/src/Loopless.Infrastructure/DependencyInjection.cs +++ b/backend/src/Loopless.Infrastructure/DependencyInjection.cs @@ -81,6 +81,7 @@ public static IServiceCollection AddInfrastructure(this IServiceCollection servi services.AddScoped(); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); @@ -103,6 +104,7 @@ public static IServiceCollection AddInfrastructure(this IServiceCollection servi services.AddSingleton(new ConnectionFactory { Uri = new Uri(rabbitMqConnectionString) }); services.AddSingleton(); services.AddHostedService(); + services.AddHostedService(); services.AddHostedService(); services.AddHostedService(); services.AddHostedService(); diff --git a/backend/src/Loopless.Infrastructure/Hubs/MessagingHub.cs b/backend/src/Loopless.Infrastructure/Hubs/MessagingHub.cs index bdc2b1d..790ec1b 100644 --- a/backend/src/Loopless.Infrastructure/Hubs/MessagingHub.cs +++ b/backend/src/Loopless.Infrastructure/Hubs/MessagingHub.cs @@ -25,6 +25,11 @@ public override async Task OnConnectedAsync() await Groups.AddToGroupAsync(Context.ConnectionId, GroupName(conversationId)); } + // Per-user group lets the server reach this user for conversations created AFTER they + // connected (the conversation-group joins above only cover conversations that already + // existed at connect time). + await Groups.AddToGroupAsync(Context.ConnectionId, UserGroupName(userId)); + await base.OnConnectedAsync(); } @@ -133,4 +138,6 @@ private async Task EnsureParticipantAsync(Guid conversationId, Guid userId) } private static string GroupName(Guid conversationId) => $"conv:{conversationId}"; + + internal static string UserGroupName(Guid userId) => $"user:{userId}"; } diff --git a/backend/src/Loopless.Infrastructure/Hubs/MessagingNotifier.cs b/backend/src/Loopless.Infrastructure/Hubs/MessagingNotifier.cs index 457fe27..fa48cdb 100644 --- a/backend/src/Loopless.Infrastructure/Hubs/MessagingNotifier.cs +++ b/backend/src/Loopless.Infrastructure/Hubs/MessagingNotifier.cs @@ -6,10 +6,16 @@ namespace Loopless.Infrastructure.Hubs; internal sealed class MessagingNotifier(IHubContext hub) : IMessagingNotifier { - public Task NotifyMessageSentAsync(MessageDto message, CancellationToken cancellationToken = default) + public Task NotifyMessageSentAsync(MessageDto message, Guid recipientId, CancellationToken cancellationToken = default) { - return hub.Clients.Group(GroupName(message.ConversationId)) - .SendAsync("ReceiveMessage", message, cancellationToken); + // Deliver to the conversation group (online participants of an existing conversation) AND + // the recipient's per-user group, so messages in conversations created after the recipient + // connected are still delivered live. The client dedupes by message id. + return Task.WhenAll( + hub.Clients.Group(GroupName(message.ConversationId)) + .SendAsync("ReceiveMessage", message, cancellationToken), + hub.Clients.Group(MessagingHub.UserGroupName(recipientId)) + .SendAsync("ReceiveMessage", message, cancellationToken)); } public Task NotifyMessageReadAsync( diff --git a/backend/src/Loopless.Infrastructure/Identity/KeycloakAdminClient.cs b/backend/src/Loopless.Infrastructure/Identity/KeycloakAdminClient.cs index 17a00b8..06dacb8 100644 --- a/backend/src/Loopless.Infrastructure/Identity/KeycloakAdminClient.cs +++ b/backend/src/Loopless.Infrastructure/Identity/KeycloakAdminClient.cs @@ -80,7 +80,9 @@ public async Task PasswordGrantAsync( ["client_secret"] = _options.AdminClientSecret, ["username"] = email, ["password"] = password, - ["scope"] = "openid", + // offline_access yields a long-lived offline refresh token that is persisted in + // Keycloak's DB, so sessions survive both browser close and Keycloak restarts. + ["scope"] = "openid offline_access", }; using var req = new HttpRequestMessage(HttpMethod.Post, BuildTokenUri()) @@ -120,6 +122,9 @@ public async Task AuthorizationCodeGrantAsync( ["client_id"] = "loopless-frontend", ["code"] = code, ["redirect_uri"] = redirectUri, + // Mirror the offline_access scope requested at the /auth redirect so the SSO + // path also receives a persistent offline refresh token. + ["scope"] = "openid offline_access", }; using var req = new HttpRequestMessage(HttpMethod.Post, BuildTokenUri()) diff --git a/backend/src/Loopless.Infrastructure/Messaging/Consumers/AnalyzeBlockerSentimentConsumer.cs b/backend/src/Loopless.Infrastructure/Messaging/Consumers/AnalyzeBlockerSentimentConsumer.cs new file mode 100644 index 0000000..82472da --- /dev/null +++ b/backend/src/Loopless.Infrastructure/Messaging/Consumers/AnalyzeBlockerSentimentConsumer.cs @@ -0,0 +1,63 @@ +using System.Text.Json; +using Loopless.Application.Interfaces; +using Loopless.Domain.Enums; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using RabbitMQ.Client; + +namespace Loopless.Infrastructure.Messaging.Consumers; + +internal sealed record StandupSubmittedForSentimentEvent(Guid StandupId); + +public sealed class AnalyzeBlockerSentimentConsumer : RabbitMqConsumer +{ + protected override string QueueName => "loopless.standup.sentiment"; + protected override IReadOnlyList RoutingKeys => ["standups.submitted"]; + + private readonly IServiceScopeFactory _scopeFactory; + + public AnalyzeBlockerSentimentConsumer( + ConnectionFactory factory, + IServiceScopeFactory scopeFactory, + ILogger logger) + : base(factory, logger) + { + _scopeFactory = scopeFactory; + } + + protected override async Task HandleMessageAsync(string routingKey, byte[] body, CancellationToken cancellationToken) + { + var e = JsonSerializer.Deserialize(body); + if (e is null) return; + + await using var scope = _scopeFactory.CreateAsyncScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var nlp = scope.ServiceProvider.GetRequiredService(); + var logger = scope.ServiceProvider.GetRequiredService>(); + + var standup = await db.Standups.FirstOrDefaultAsync(s => s.Id == e.StandupId, cancellationToken); + if (standup is null) + { + logger.LogWarning("Standup {StandupId} not found; skipping sentiment analysis.", e.StandupId); + return; + } + + if (string.IsNullOrWhiteSpace(standup.Blockers)) + return; + + Sentiment? sentiment; + try + { + sentiment = nlp.AnalyseSentiment(standup.Blockers); + } + catch (Exception ex) + { + logger.LogError(ex, "NLP sentiment analysis failed for standup {StandupId}; leaving blocker_sentiment null.", e.StandupId); + return; + } + + standup.BlockerSentiment = sentiment; + await db.SaveChangesAsync(cancellationToken); + } +} diff --git a/backend/src/Loopless.Infrastructure/Messaging/Consumers/StandupNotificationConsumer.cs b/backend/src/Loopless.Infrastructure/Messaging/Consumers/StandupNotificationConsumer.cs index af3d189..15b5aed 100644 --- a/backend/src/Loopless.Infrastructure/Messaging/Consumers/StandupNotificationConsumer.cs +++ b/backend/src/Loopless.Infrastructure/Messaging/Consumers/StandupNotificationConsumer.cs @@ -17,7 +17,7 @@ internal sealed record StandupSubmittedEvent( public sealed class StandupNotificationConsumer : RabbitMqConsumer { protected override string QueueName => "loopless.standup.notifications"; - protected override IReadOnlyList RoutingKeys => ["standup.submitted"]; + protected override IReadOnlyList RoutingKeys => ["standups.submitted"]; private readonly IServiceScopeFactory _scopeFactory; diff --git a/backend/src/Loopless.Infrastructure/Persistence/Configurations/ProjectConfiguration.cs b/backend/src/Loopless.Infrastructure/Persistence/Configurations/ProjectConfiguration.cs index 65179be..4aafd21 100644 --- a/backend/src/Loopless.Infrastructure/Persistence/Configurations/ProjectConfiguration.cs +++ b/backend/src/Loopless.Infrastructure/Persistence/Configurations/ProjectConfiguration.cs @@ -16,6 +16,7 @@ public void Configure(EntityTypeBuilder builder) builder.Property(p => p.Title).HasMaxLength(200).IsRequired(); builder.Property(p => p.Description).HasMaxLength(4000); builder.Property(p => p.GitHubRepoUrl).HasMaxLength(500); + builder.Property(p => p.GitHubWebhookSecret).HasMaxLength(256); builder.Property(p => p.Status) .HasConversion() diff --git a/backend/src/Loopless.Infrastructure/Persistence/Configurations/ProjectSummaryConfiguration.cs b/backend/src/Loopless.Infrastructure/Persistence/Configurations/ProjectSummaryConfiguration.cs index 1b1eb8a..0b4d265 100644 --- a/backend/src/Loopless.Infrastructure/Persistence/Configurations/ProjectSummaryConfiguration.cs +++ b/backend/src/Loopless.Infrastructure/Persistence/Configurations/ProjectSummaryConfiguration.cs @@ -1,4 +1,5 @@ using Loopless.Domain.Entities; +using Loopless.Domain.Enums; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata.Builders; @@ -12,9 +13,12 @@ public void Configure(EntityTypeBuilder builder) builder.HasKey(s => s.Id); builder.Property(s => s.ProjectId).IsRequired(); - builder.Property(s => s.Content).HasMaxLength(8000).IsRequired(); - builder.Property(s => s.ModelUsed).HasMaxLength(100).IsRequired(); + builder.Property(s => s.Content).HasMaxLength(8000).IsRequired(false); + builder.Property(s => s.ModelUsed).HasMaxLength(100).IsRequired(false); builder.Property(s => s.GeneratedAt).IsRequired(); + builder.Property(s => s.Status).IsRequired().HasDefaultValue(SummaryStatus.Queued); + builder.Property(s => s.HangfireJobId).HasMaxLength(255).IsRequired(false); + builder.Property(s => s.FailureReason).IsRequired(false); builder.HasOne(s => s.Project) .WithMany() diff --git a/backend/src/Loopless.Infrastructure/Persistence/Migrations/20260610120000_AddGithubWebhookSecret.cs b/backend/src/Loopless.Infrastructure/Persistence/Migrations/20260610120000_AddGithubWebhookSecret.cs new file mode 100644 index 0000000..ab469be --- /dev/null +++ b/backend/src/Loopless.Infrastructure/Persistence/Migrations/20260610120000_AddGithubWebhookSecret.cs @@ -0,0 +1,34 @@ +using Loopless.Infrastructure.Persistence; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Loopless.Infrastructure.Persistence.Migrations +{ + /// + [DbContext(typeof(AppDbContext))] + [Migration("20260610120000_AddGithubWebhookSecret")] + public partial class AddGithubWebhookSecret : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AddColumn( + name: "GitHubWebhookSecret", + table: "projects", + type: "character varying(256)", + maxLength: 256, + nullable: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn( + name: "GitHubWebhookSecret", + table: "projects"); + } + } +} diff --git a/backend/src/Loopless.Infrastructure/Persistence/Migrations/20260610120000_AddSummaryStatusFields.cs b/backend/src/Loopless.Infrastructure/Persistence/Migrations/20260610120000_AddSummaryStatusFields.cs new file mode 100644 index 0000000..6f4f7b4 --- /dev/null +++ b/backend/src/Loopless.Infrastructure/Persistence/Migrations/20260610120000_AddSummaryStatusFields.cs @@ -0,0 +1,92 @@ +using Loopless.Infrastructure.Persistence; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Loopless.Infrastructure.Persistence.Migrations +{ + /// + [DbContext(typeof(AppDbContext))] + [Migration("20260610120000_AddSummaryStatusFields")] + public partial class AddSummaryStatusFields : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn( + name: "ModelUsed", + table: "project_summaries", + type: "character varying(100)", + maxLength: 100, + nullable: true, + oldClrType: typeof(string), + oldType: "character varying(100)", + oldMaxLength: 100); + + migrationBuilder.AlterColumn( + name: "Content", + table: "project_summaries", + type: "character varying(8000)", + maxLength: 8000, + nullable: true, + oldClrType: typeof(string), + oldType: "character varying(8000)", + oldMaxLength: 8000); + + // Existing rows are completed summaries from the synchronous pipeline. + migrationBuilder.AddColumn( + name: "Status", + table: "project_summaries", + type: "integer", + nullable: false, + defaultValue: 2); + + migrationBuilder.AddColumn( + name: "HangfireJobId", + table: "project_summaries", + type: "character varying(255)", + maxLength: 255, + nullable: true); + + migrationBuilder.AddColumn( + name: "FailureReason", + table: "project_summaries", + type: "text", + nullable: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropColumn(name: "Status", table: "project_summaries"); + migrationBuilder.DropColumn(name: "HangfireJobId", table: "project_summaries"); + migrationBuilder.DropColumn(name: "FailureReason", table: "project_summaries"); + + migrationBuilder.AlterColumn( + name: "ModelUsed", + table: "project_summaries", + type: "character varying(100)", + maxLength: 100, + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "character varying(100)", + oldMaxLength: 100, + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "Content", + table: "project_summaries", + type: "character varying(8000)", + maxLength: 8000, + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "character varying(8000)", + oldMaxLength: 8000, + oldNullable: true); + } + } +} diff --git a/backend/src/Loopless.Infrastructure/Persistence/Migrations/20260610130000_MigrateEmbeddingsIndexToHnsw.cs b/backend/src/Loopless.Infrastructure/Persistence/Migrations/20260610130000_MigrateEmbeddingsIndexToHnsw.cs new file mode 100644 index 0000000..d692260 --- /dev/null +++ b/backend/src/Loopless.Infrastructure/Persistence/Migrations/20260610130000_MigrateEmbeddingsIndexToHnsw.cs @@ -0,0 +1,42 @@ +using Loopless.Infrastructure.Persistence; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace Loopless.Infrastructure.Persistence.Migrations +{ + /// + [DbContext(typeof(AppDbContext))] + [Migration("20260610130000_MigrateEmbeddingsIndexToHnsw")] + public partial class MigrateEmbeddingsIndexToHnsw : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql( + "DROP INDEX IF EXISTS ix_embeddings_vector_ivfflat;", + suppressTransaction: true); + + migrationBuilder.Sql( + "CREATE INDEX CONCURRENTLY idx_embeddings_hnsw " + + "ON embeddings USING hnsw (\"Vector\" vector_cosine_ops) " + + "WITH (m = 16, ef_construction = 64);", + suppressTransaction: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql( + "DROP INDEX IF EXISTS idx_embeddings_hnsw;", + suppressTransaction: true); + + migrationBuilder.Sql( + "CREATE INDEX CONCURRENTLY ix_embeddings_vector_ivfflat " + + "ON embeddings USING ivfflat (\"Vector\" vector_cosine_ops) WITH (lists = 100);", + suppressTransaction: true); + } + } +} diff --git a/backend/src/Loopless.Infrastructure/Persistence/Migrations/AppDbContextModelSnapshot.cs b/backend/src/Loopless.Infrastructure/Persistence/Migrations/AppDbContextModelSnapshot.cs index eef8755..0ef0241 100644 --- a/backend/src/Loopless.Infrastructure/Persistence/Migrations/AppDbContextModelSnapshot.cs +++ b/backend/src/Loopless.Infrastructure/Persistence/Migrations/AppDbContextModelSnapshot.cs @@ -427,6 +427,10 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasMaxLength(500) .HasColumnType("character varying(500)"); + b.Property("GitHubWebhookSecret") + .HasMaxLength(256) + .HasColumnType("character varying(256)"); + b.Property("IsDeleted") .ValueGeneratedOnAdd() .HasColumnType("boolean") @@ -607,24 +611,33 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("uuid"); b.Property("Content") - .IsRequired() .HasMaxLength(8000) .HasColumnType("character varying(8000)"); b.Property("CreatedAt") .HasColumnType("timestamp with time zone"); + b.Property("FailureReason") + .HasColumnType("text"); + b.Property("GeneratedAt") .HasColumnType("timestamp with time zone"); + b.Property("HangfireJobId") + .HasMaxLength(255) + .HasColumnType("character varying(255)"); + b.Property("ModelUsed") - .IsRequired() .HasMaxLength(100) .HasColumnType("character varying(100)"); b.Property("ProjectId") .HasColumnType("uuid"); + b.Property("Status") + .HasDefaultValue(0) + .HasColumnType("integer"); + b.Property("UpdatedAt") .HasColumnType("timestamp with time zone"); diff --git a/backend/src/Loopless.Infrastructure/Persistence/Seeds/SeedData.cs b/backend/src/Loopless.Infrastructure/Persistence/Seeds/SeedData.cs deleted file mode 100644 index 4ef00d3..0000000 --- a/backend/src/Loopless.Infrastructure/Persistence/Seeds/SeedData.cs +++ /dev/null @@ -1,176 +0,0 @@ -using Loopless.Application.Interfaces; -using Loopless.Domain.Entities; -using Loopless.Domain.Enums; -using Microsoft.EntityFrameworkCore; -using Pgvector; - -namespace Loopless.Infrastructure.Persistence.Seeds; - -public static class SeedData -{ - private sealed record SeedUser( - string KeycloakId, - string Email, - string Name, - UserRole Role, - SeedFreelancerProfile? FreelancerProfile, - SeedEnterpriseProfile? EnterpriseProfile, - EmbeddingEntityType EmbeddingType, - string EmbeddingText); - - private sealed record SeedFreelancerProfile( - string? Bio, - decimal? HourlyRate, - string? Availability, - List Proficiencies, - string? GithubUsername); - - private sealed record SeedEnterpriseProfile( - string CompanyName, - string? Industry, - List TypicalNeeds); - - public static async Task SeedAsync( - AppDbContext db, - IEmbeddingService embeddings, - CancellationToken cancellationToken = default) - { - var seeds = new List - { - new( - KeycloakId: "seed-freelancer-1", - Email: "freelancer1@loopless.dev", - Name: "Avery Jordan", - Role: UserRole.Freelancer, - FreelancerProfile: new SeedFreelancerProfile( - Bio: "Frontend engineer focused on React and TypeScript.", - HourlyRate: 85m, - Availability: "Full-time", - Proficiencies: ["react", "typescript", "nodejs"], - GithubUsername: "averyj"), - EnterpriseProfile: null, - EmbeddingType: EmbeddingEntityType.FreelancerProfile, - EmbeddingText: "Freelancer skills: React, TypeScript, Node.js"), - new( - KeycloakId: "seed-freelancer-2", - Email: "freelancer2@loopless.dev", - Name: "Mina Patel", - Role: UserRole.Freelancer, - FreelancerProfile: new SeedFreelancerProfile( - Bio: "Data engineer with Python and ML pipelines.", - HourlyRate: 95m, - Availability: "Part-time", - Proficiencies: ["python", "ml", "postgresql"], - GithubUsername: "minap"), - EnterpriseProfile: null, - EmbeddingType: EmbeddingEntityType.FreelancerProfile, - EmbeddingText: "Freelancer skills: Python, Machine Learning, PostgreSQL"), - new( - KeycloakId: "seed-enterprise-1", - Email: "enterprise1@loopless.dev", - Name: "Northwind Labs", - Role: UserRole.Enterprise, - FreelancerProfile: null, - EnterpriseProfile: new SeedEnterpriseProfile( - CompanyName: "Northwind Labs", - Industry: "SaaS", - TypicalNeeds: ["development", "product-management", "data-analytics"]), - EmbeddingType: EmbeddingEntityType.EnterpriseProfile, - EmbeddingText: "Enterprise Northwind Labs needs: Development, Product Management, Data & Analytics"), - new( - KeycloakId: "seed-enterprise-2", - Email: "enterprise2@loopless.dev", - Name: "Contoso Media", - Role: UserRole.Enterprise, - FreelancerProfile: null, - EnterpriseProfile: new SeedEnterpriseProfile( - CompanyName: "Contoso Media", - Industry: "Media", - TypicalNeeds: ["marketing", "design", "creative"]), - EmbeddingType: EmbeddingEntityType.EnterpriseProfile, - EmbeddingText: "Enterprise Contoso Media needs: Marketing, Design, Creative") - }; - - var userIds = new Dictionary(StringComparer.OrdinalIgnoreCase); - - foreach (var seed in seeds) - { - var user = await db.Users - .FirstOrDefaultAsync(u => u.KeycloakId == seed.KeycloakId, cancellationToken); - - if (user is null) - { - user = new User - { - Email = seed.Email, - Name = seed.Name, - Role = seed.Role, - KeycloakId = seed.KeycloakId, - }; - db.Users.Add(user); - } - - userIds[seed.KeycloakId] = user.Id; - - if (seed.FreelancerProfile is not null) - { - var exists = await db.FreelancerProfiles - .AnyAsync(p => p.UserId == user.Id, cancellationToken); - - if (!exists) - { - db.FreelancerProfiles.Add(new FreelancerProfile - { - UserId = user.Id, - Bio = seed.FreelancerProfile.Bio, - HourlyRate = seed.FreelancerProfile.HourlyRate, - Availability = seed.FreelancerProfile.Availability, - Proficiencies = seed.FreelancerProfile.Proficiencies, - GithubUsername = seed.FreelancerProfile.GithubUsername, - }); - } - } - - if (seed.EnterpriseProfile is not null) - { - var exists = await db.EnterpriseProfiles - .AnyAsync(p => p.UserId == user.Id, cancellationToken); - - if (!exists) - { - db.EnterpriseProfiles.Add(new EnterpriseProfile - { - UserId = user.Id, - CompanyName = seed.EnterpriseProfile.CompanyName, - Industry = seed.EnterpriseProfile.Industry, - TypicalNeeds = seed.EnterpriseProfile.TypicalNeeds, - }); - } - } - } - - await db.SaveChangesAsync(cancellationToken); - - foreach (var seed in seeds) - { - var userId = userIds[seed.KeycloakId]; - var embeddingExists = await db.Embeddings - .AnyAsync( - e => e.EntityType == seed.EmbeddingType && e.EntityId == userId, - cancellationToken); - - if (embeddingExists) - continue; - - var vector = await embeddings.GenerateAsync(seed.EmbeddingText, cancellationToken); - db.Embeddings.Add(new Embedding - { - EntityType = seed.EmbeddingType, - EntityId = userId, - Vector = new Vector(vector), - }); - } - - await db.SaveChangesAsync(cancellationToken); - } -} diff --git a/backend/tests/Loopless.UnitTests/Messaging/RecordingMessagingNotifier.cs b/backend/tests/Loopless.UnitTests/Messaging/RecordingMessagingNotifier.cs index 712bedb..6afb634 100644 --- a/backend/tests/Loopless.UnitTests/Messaging/RecordingMessagingNotifier.cs +++ b/backend/tests/Loopless.UnitTests/Messaging/RecordingMessagingNotifier.cs @@ -8,7 +8,7 @@ internal sealed class RecordingMessagingNotifier : IMessagingNotifier public List Sent { get; } = new(); public List<(Guid ConversationId, Guid MessageId, Guid ReaderId, DateTimeOffset ReadAt)> Reads { get; } = new(); - public Task NotifyMessageSentAsync(MessageDto message, CancellationToken cancellationToken = default) + public Task NotifyMessageSentAsync(MessageDto message, Guid recipientId, CancellationToken cancellationToken = default) { Sent.Add(message); return Task.CompletedTask; diff --git a/backend/tests/Loopless.UnitTests/Standups/GetStandupAnalyticsQueryHandlerTests.cs b/backend/tests/Loopless.UnitTests/Standups/GetStandupAnalyticsQueryHandlerTests.cs index 91b3e92..9b53483 100644 --- a/backend/tests/Loopless.UnitTests/Standups/GetStandupAnalyticsQueryHandlerTests.cs +++ b/backend/tests/Loopless.UnitTests/Standups/GetStandupAnalyticsQueryHandlerTests.cs @@ -2,6 +2,7 @@ using Loopless.Application.Features.Standups.GetAnalytics; using Loopless.Domain.Entities; using Loopless.Domain.Enums; +using Loopless.UnitTests.Messaging; using Xunit; namespace Loopless.UnitTests.Standups; @@ -53,7 +54,7 @@ public async Task EmptyWindow_ReturnsZeroCompletion_AndNoBlockers() db.Projects.Add(project); await db.SaveChangesAsync(); - var handler = new GetStandupAnalyticsQueryHandler(db); + var handler = new GetStandupAnalyticsQueryHandler(db, new FakeCurrentUser("k-1")); var result = await handler.Handle(new GetStandupAnalyticsQuery(project.Id, 7), CancellationToken.None); Assert.Equal(project.Id, result.ProjectId); @@ -68,7 +69,10 @@ public async Task EmptyWindow_ReturnsZeroCompletion_AndNoBlockers() public async Task ProjectNotFound_Throws() { await using var db = StandupAnalyticsTestDbContext.CreateInMemory(); - var handler = new GetStandupAnalyticsQueryHandler(db); + db.Users.Add(MakeFreelancer("k-1", "Alice")); + await db.SaveChangesAsync(); + + var handler = new GetStandupAnalyticsQueryHandler(db, new FakeCurrentUser("k-1")); await Assert.ThrowsAsync(() => handler.Handle(new GetStandupAnalyticsQuery(Guid.NewGuid(), 7), CancellationToken.None)); } @@ -99,7 +103,7 @@ public async Task FullWeek_AllWeekdaysCovered_Yields100Percent() } await db.SaveChangesAsync(); - var handler = new GetStandupAnalyticsQueryHandler(db); + var handler = new GetStandupAnalyticsQueryHandler(db, new FakeCurrentUser("k-1")); var result = await handler.Handle(new GetStandupAnalyticsQuery(project.Id, 7), CancellationToken.None); var member = Assert.Single(result.MemberCompletions); @@ -134,7 +138,7 @@ public async Task SaturdayOnlyStandup_DoesNotCountTowardsWeekdayCompletion() "blocked on api")); await db.SaveChangesAsync(); - var handler = new GetStandupAnalyticsQueryHandler(db); + var handler = new GetStandupAnalyticsQueryHandler(db, new FakeCurrentUser("k-1")); var result = await handler.Handle(new GetStandupAnalyticsQuery(project.Id, 7), CancellationToken.None); var member = Assert.Single(result.MemberCompletions); @@ -159,7 +163,7 @@ public async Task BlockerAggregation_DropsStopWords_IsCaseInsensitive() db.Standups.Add(MakeStandup(project.Id, freelancer.Id, now.AddHours(-3), "DATABASE again")); await db.SaveChangesAsync(); - var handler = new GetStandupAnalyticsQueryHandler(db); + var handler = new GetStandupAnalyticsQueryHandler(db, new FakeCurrentUser("k-1")); var result = await handler.Handle(new GetStandupAnalyticsQuery(project.Id, 7), CancellationToken.None); var keywords = result.TopBlockers.ToDictionary(b => b.Keyword, b => b.Count); @@ -183,7 +187,7 @@ public async Task DaysClampedTo90_NegativeFallsBackToDefault() db.Projects.Add(project); await db.SaveChangesAsync(); - var handler = new GetStandupAnalyticsQueryHandler(db); + var handler = new GetStandupAnalyticsQueryHandler(db, new FakeCurrentUser("k-1")); var clamped = await handler.Handle(new GetStandupAnalyticsQuery(project.Id, 365), CancellationToken.None); Assert.Equal(90, clamped.Days); @@ -191,4 +195,22 @@ public async Task DaysClampedTo90_NegativeFallsBackToDefault() var defaulted = await handler.Handle(new GetStandupAnalyticsQuery(project.Id, 0), CancellationToken.None); Assert.Equal(7, defaulted.Days); } + + [Fact] + public async Task NonMember_Throws_Forbidden() + { + await using var db = StandupAnalyticsTestDbContext.CreateInMemory(); + var owner = MakeFreelancer("k-owner", "Owner"); + db.Users.Add(owner); + var outsider = MakeFreelancer("k-outsider", "Outsider"); + db.Users.Add(outsider); + var project = MakeProject(owner.Id); + db.Projects.Add(project); + await db.SaveChangesAsync(); + + // Outsider is neither the owner nor an accepted member -> must be denied (IDOR guard). + var handler = new GetStandupAnalyticsQueryHandler(db, new FakeCurrentUser("k-outsider")); + await Assert.ThrowsAsync(() => + handler.Handle(new GetStandupAnalyticsQuery(project.Id, 7), CancellationToken.None)); + } } diff --git a/backend/tests/Loopless.UnitTests/Standups/StandupAnalyticsTestDbContext.cs b/backend/tests/Loopless.UnitTests/Standups/StandupAnalyticsTestDbContext.cs index f59b4f1..c86e72e 100644 --- a/backend/tests/Loopless.UnitTests/Standups/StandupAnalyticsTestDbContext.cs +++ b/backend/tests/Loopless.UnitTests/Standups/StandupAnalyticsTestDbContext.cs @@ -36,7 +36,6 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Ignore(); modelBuilder.Ignore(); modelBuilder.Ignore(); - modelBuilder.Ignore(); modelBuilder.Ignore(); modelBuilder.Ignore(); modelBuilder.Ignore(); @@ -65,6 +64,14 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) b.HasKey(s => s.Id); b.HasOne(s => s.Freelancer).WithMany().HasForeignKey(s => s.FreelancerId); }); + // Modeled (not ignored) so the handler's owner/member visibility guard can query it. + modelBuilder.Entity(b => + { + b.HasKey(i => i.Id); + b.Ignore(i => i.Project); + b.Ignore(i => i.Enterprise); + b.Ignore(i => i.Freelancer); + }); base.OnModelCreating(modelBuilder); } diff --git a/backend/tests/Loopless.UnitTests/Visibility/ProjectVisibilityGuardTests.cs b/backend/tests/Loopless.UnitTests/Visibility/ProjectVisibilityGuardTests.cs new file mode 100644 index 0000000..b92bfe0 --- /dev/null +++ b/backend/tests/Loopless.UnitTests/Visibility/ProjectVisibilityGuardTests.cs @@ -0,0 +1,146 @@ +using Loopless.Application.Common.Exceptions; +using Loopless.Application.Features.Projects.GetCommits; +using Loopless.Application.Features.Standups.GetAnalytics; +using Loopless.Application.Features.Standups.GetFeed; +using Loopless.Domain.Entities; +using Loopless.Domain.Enums; +using Loopless.UnitTests.Messaging; +using Xunit; + +namespace Loopless.UnitTests.Visibility; + +// Regression tests for FR-4.7 "project visibility invited-only, enforced backend". +// The standup feed, CSV export, and commits endpoints previously returned another +// project's data to ANY authenticated user (no membership check). These assert the +// owner-or-accepted-member guard now blocks non-members and admits owner + members. +public class ProjectVisibilityGuardTests +{ + private static User MakeUser(string keycloakId, UserRole role) + => new() + { + Email = $"{keycloakId}@test.local", + Name = keycloakId, + Role = role, + KeycloakId = keycloakId, + }; + + private static Project MakeProject(Guid ownerId) + => new() { OwnerId = ownerId, Title = "Test Project", Status = ProjectStatus.Active }; + + private sealed record Fixture( + ProjectVisibilityTestDbContext Db, + Project Project, + User Owner, + User Member, + User Outsider); + + // Owner (enterprise) + an accepted freelancer member + an unrelated outsider, with one + // standup and one commit on the project. + private static async Task SeedAsync() + { + var db = ProjectVisibilityTestDbContext.CreateInMemory(); + + var owner = MakeUser("owner", UserRole.Enterprise); + var member = MakeUser("member", UserRole.Freelancer); + var outsider = MakeUser("outsider", UserRole.Freelancer); + db.Users.AddRange(owner, member, outsider); + + var project = MakeProject(owner.Id); + db.Projects.Add(project); + + db.ProjectInvitations.Add(new ProjectInvitation + { + ProjectId = project.Id, + EnterpriseId = owner.Id, + FreelancerId = member.Id, + Status = InvitationStatus.Accepted, + }); + + db.Standups.Add(new Standup + { + ProjectId = project.Id, + FreelancerId = member.Id, + WhatDid = "did", + WhatPlan = "plan", + Blockers = "none", + CreatedAt = DateTimeOffset.UtcNow, + }); + + db.GitHubCommits.Add(new GitHubCommit + { + ProjectId = project.Id, + Sha = "abc123", + Message = "init", + Author = "member", + CommittedAt = DateTimeOffset.UtcNow, + }); + + await db.SaveChangesAsync(); + return new Fixture(db, project, owner, member, outsider); + } + + // ---- Standup feed ------------------------------------------------------- + + [Fact] + public async Task StandupFeed_Outsider_IsForbidden() + { + var f = await SeedAsync(); + var handler = new GetStandupFeedQueryHandler(f.Db, new FakeCurrentUser(f.Outsider.KeycloakId)); + await Assert.ThrowsAsync(() => + handler.Handle(new GetStandupFeedQuery(f.Project.Id, 1, 20), CancellationToken.None)); + } + + [Fact] + public async Task StandupFeed_Owner_AndMember_SeeEntries() + { + var f = await SeedAsync(); + + var asOwner = new GetStandupFeedQueryHandler(f.Db, new FakeCurrentUser(f.Owner.KeycloakId)); + var ownerResult = await asOwner.Handle(new GetStandupFeedQuery(f.Project.Id, 1, 20), CancellationToken.None); + Assert.Equal(1, ownerResult.TotalCount); + + var asMember = new GetStandupFeedQueryHandler(f.Db, new FakeCurrentUser(f.Member.KeycloakId)); + var memberResult = await asMember.Handle(new GetStandupFeedQuery(f.Project.Id, 1, 20), CancellationToken.None); + Assert.Equal(1, memberResult.TotalCount); + } + + // ---- CSV export --------------------------------------------------------- + + [Fact] + public async Task StandupCsvExport_Outsider_IsForbidden() + { + var f = await SeedAsync(); + var handler = new ExportStandupsCsvQueryHandler(f.Db, new FakeCurrentUser(f.Outsider.KeycloakId)); + await Assert.ThrowsAsync(() => + handler.Handle(new ExportStandupsCsvQuery(f.Project.Id), CancellationToken.None)); + } + + [Fact] + public async Task StandupCsvExport_Member_GetsRows() + { + var f = await SeedAsync(); + var handler = new ExportStandupsCsvQueryHandler(f.Db, new FakeCurrentUser(f.Member.KeycloakId)); + var rows = await handler.Handle(new ExportStandupsCsvQuery(f.Project.Id), CancellationToken.None); + Assert.Single(rows); + } + + // ---- Commits ------------------------------------------------------------ + + [Fact] + public async Task Commits_Outsider_IsForbidden() + { + var f = await SeedAsync(); + var handler = new GetProjectCommitsQueryHandler(f.Db, new FakeCurrentUser(f.Outsider.KeycloakId)); + await Assert.ThrowsAsync(() => + handler.Handle(new GetProjectCommitsQuery(f.Project.Id, 1, 20), CancellationToken.None)); + } + + [Fact] + public async Task Commits_Owner_SeesCommits() + { + var f = await SeedAsync(); + var handler = new GetProjectCommitsQueryHandler(f.Db, new FakeCurrentUser(f.Owner.KeycloakId)); + var result = await handler.Handle(new GetProjectCommitsQuery(f.Project.Id, 1, 20), CancellationToken.None); + Assert.Equal(1, result.TotalCount); + } +} diff --git a/backend/tests/Loopless.UnitTests/Visibility/ProjectVisibilityTestDbContext.cs b/backend/tests/Loopless.UnitTests/Visibility/ProjectVisibilityTestDbContext.cs new file mode 100644 index 0000000..0ec6a18 --- /dev/null +++ b/backend/tests/Loopless.UnitTests/Visibility/ProjectVisibilityTestDbContext.cs @@ -0,0 +1,93 @@ +using Loopless.Application.Interfaces; +using Loopless.Domain.Entities; +using Microsoft.EntityFrameworkCore; + +namespace Loopless.UnitTests.Visibility; + +// In-memory IAppDbContext that maps the entities the project-visibility guards touch +// (Users, Projects, ProjectInvitations, Standups, GitHubCommits). The Standups-analytics +// test context ignores ProjectInvitation, so the membership guards need their own. +internal sealed class ProjectVisibilityTestDbContext(DbContextOptions options) + : DbContext(options), IAppDbContext +{ + public DbSet Users => Set(); + public DbSet FreelancerProfiles => Set(); + public DbSet EnterpriseProfiles => Set(); + public DbSet Skills => Set(); + public DbSet Embeddings => Set(); + public DbSet Standups => Set(); + public DbSet Conversations => Set(); + public DbSet Messages => Set(); + public DbSet Projects => Set(); + public DbSet ProjectInvitations => Set(); + public DbSet GitHubCommits => Set(); + public DbSet ProjectTasks => Set(); + public DbSet ProjectTaskAuditLogs => Set(); + public DbSet Notifications => Set(); + public DbSet ProjectSummaries => Set(); + public DbSet AuditTrails => Set(); + public DbSet ProjectFiles => Set(); + public DbSet ContentFlags => Set(); + public DbSet ProjectLinks => Set(); + public DbSet SupportTickets => Set(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Ignore(); + modelBuilder.Ignore(); + modelBuilder.Ignore(); + modelBuilder.Ignore(); + modelBuilder.Ignore(); + modelBuilder.Ignore(); + modelBuilder.Ignore(); + modelBuilder.Ignore(); + modelBuilder.Ignore(); + modelBuilder.Ignore(); + modelBuilder.Ignore(); + modelBuilder.Ignore(); + modelBuilder.Ignore(); + modelBuilder.Ignore(); + modelBuilder.Ignore(); + + modelBuilder.Entity(b => + { + b.HasKey(u => u.Id); + b.Ignore(u => u.FreelancerProfile); + b.Ignore(u => u.EnterpriseProfile); + }); + modelBuilder.Entity(b => + { + b.HasKey(p => p.Id); + b.Ignore(p => p.Owner); + b.Ignore(p => p.Invitations); + }); + modelBuilder.Entity(b => + { + b.HasKey(i => i.Id); + b.Ignore(i => i.Project); + b.Ignore(i => i.Enterprise); + b.Ignore(i => i.Freelancer); + }); + modelBuilder.Entity(b => + { + b.HasKey(s => s.Id); + b.HasOne(s => s.Freelancer).WithMany().HasForeignKey(s => s.FreelancerId); + }); + modelBuilder.Entity(b => + { + b.HasKey(c => c.Id); + b.Ignore(c => c.Project); + }); + + base.OnModelCreating(modelBuilder); + } + + public static ProjectVisibilityTestDbContext CreateInMemory() + { + var options = new DbContextOptionsBuilder() + .UseInMemoryDatabase(Guid.NewGuid().ToString()) + .ConfigureWarnings(w => w.Ignore(Microsoft.EntityFrameworkCore.Diagnostics.InMemoryEventId.TransactionIgnoredWarning)) + .Options; + return new ProjectVisibilityTestDbContext(options); + } +} diff --git a/devops/README.md b/devops/README.md index b6dc9e7..e3867e9 100644 --- a/devops/README.md +++ b/devops/README.md @@ -5,12 +5,11 @@ This folder defines infrastructure, delivery, and observability assets for LIFE- ## Scope - Docker images and Compose environment -- Kubernetes manifests/Helm charts -- Terraform modules and environments +- Kubernetes manifests/Helm charts (all infrastructure via Helm) - Monitoring/logging/alerting configs - CI/CD pipeline scripts and policies -## Suggested Structure +## Structure ```text devops/ @@ -21,27 +20,18 @@ devops/ k8s/ base/ overlays/ - dev/ staging/ - prod/ - terraform/ - modules/ - network/ - database/ - observability/ - envs/ - dev/ - staging/ - prod/ + production/ + helm/ + loopless/ + templates/ PostgreSQL, Redis, MinIO + app services + values.yaml monitoring/ prometheus/ grafana/ loki/ - alertmanager/ scripts/ - bootstrap/ - deploy/ - rollback/ + loadtest/ ``` ## Operational Baseline @@ -60,9 +50,8 @@ Isolated environment for pre-production validation. Auto-deploys from `main` (CP | URL | `https://staging.loopless.app` | | Keycloak | `https://keycloak.staging.loopless.app` (realm `loopless-staging`) | | Namespace | `loopless-staging` | -| Database | PostgreSQL `loopless_staging` | +| Database | PostgreSQL `loopless_staging` (deployed via Helm) | | Kustomize overlay | `devops/k8s/overlays/staging/` | -| Terraform env | `devops/terraform/envs/staging/` | Render manifests: @@ -108,20 +97,6 @@ Both logging stacks are available in Compose: This keeps Grafana/Loki for operational triage while preserving ELK for search-heavy analysis. -## Linux Server Bootstrap (VPS/EC2) - -Production server bootstrap scripts live in `devops/scripts/server/`: - -- `bootstrap-ubuntu.sh` — creates deploy user, installs Docker + Compose, hardens SSH, enables UFW + fail2ban, and installs maintenance cron. -- `install-loopless-service.sh` — installs and enables `loopless-compose.service` systemd unit. - -Example: - -```bash -export SSH_PUBLIC_KEY="ssh-ed25519 AAAA... your-key" -sudo DEPLOY_USER=loopless APP_DIR=/opt/loopless ./devops/scripts/server/bootstrap-ubuntu.sh -sudo DEPLOY_USER=loopless APP_DIR=/opt/loopless ./devops/scripts/server/install-loopless-service.sh -``` ## AIOps Alert Triage Pipeline diff --git a/devops/aiops/package.json b/devops/aiops/package.json new file mode 100644 index 0000000..df01b89 --- /dev/null +++ b/devops/aiops/package.json @@ -0,0 +1,13 @@ +{ + "name": "aiops-triage", + "version": "1.0.0", + "description": "Alertmanager webhook receiver → OpenAI triage → Slack/Discord", + "main": "server.js", + "scripts": { + "start": "node server.js", + "test": "node --test sanitize.test.js" + }, + "engines": { + "node": ">=20" + } +} diff --git a/devops/aiops/sanitize.js b/devops/aiops/sanitize.js new file mode 100644 index 0000000..5592d87 --- /dev/null +++ b/devops/aiops/sanitize.js @@ -0,0 +1,69 @@ +"use strict"; + +// Redaction rules applied to alert text before forwarding to OpenAI. +// Order matters: Bearer pattern must run before raw-JWT pattern to avoid +// double-redacting the same token. +const REDACTION_RULES = [ + { pattern: /postgres:\/\/[^\s"'<>]*/gi, replacement: "[REDACTED_POSTGRES_URI]" }, + { pattern: /redis:\/\/[^\s"'<>]*/gi, replacement: "[REDACTED_REDIS_URI]" }, + { pattern: /amqp:\/\/[^\s"'<>]*/gi, replacement: "[REDACTED_AMQP_URI]" }, + // Bearer token (header value form) + { pattern: /Bearer\s+[A-Za-z0-9\-_]+\.[A-Za-z0-9\-_]+\.[A-Za-z0-9\-_]*/g, replacement: "[REDACTED_BEARER]" }, + // Raw JWT (ey… three-segment form, min 10 chars per segment to avoid false positives) + { pattern: /ey[A-Za-z0-9\-_]{10,}\.[A-Za-z0-9\-_]{10,}\.[A-Za-z0-9\-_]{10,}/g, replacement: "[REDACTED_JWT]" }, + // Email addresses + { pattern: /[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}/g, replacement: "[REDACTED_EMAIL]" }, + // Common secret label patterns: password=, secret=, key=, token= (value up to whitespace) + { pattern: /(?:password|passwd|secret|api_?key|token)\s*=\s*\S+/gi, replacement: "[REDACTED_SECRET]" }, +]; + +function isRfc1918(a, b) { + if (a === 10) return true; + if (a === 172 && b >= 16 && b <= 31) return true; + if (a === 192 && b === 168) return true; + if (a === 127) return true; + if (a === 0) return true; + return false; +} + +function redactPublicIps(text) { + return text.replace( + /\b(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\b/g, + (match, a, b, c, d) => { + const [na, nb, nc, nd] = [Number(a), Number(b), Number(c), Number(d)]; + if (na > 255 || nb > 255 || nc > 255 || nd > 255) return match; + return isRfc1918(na, nb) ? match : "[REDACTED_PUBLIC_IP]"; + } + ); +} + +/** + * Sanitize a string by redacting secrets, credentials, and public IPs. + * Non-string values are returned as-is. + */ +function sanitizeText(text) { + if (typeof text !== "string" || text.length === 0) return text; + let out = text; + for (const { pattern, replacement } of REDACTION_RULES) { + out = out.replace(pattern, replacement); + } + return redactPublicIps(out); +} + +/** + * Build the alert lines sent to OpenAI. + * Only includes: alert index, severity, alertname, sanitized summary/description. + * All other labels and annotations (service, job, runbook_url, etc.) are excluded. + */ +function buildSanitizedAlertLines(alerts) { + return alerts.map((alert, index) => { + const severity = alert.labels?.severity ?? "unknown"; + const alertName = alert.labels?.alertname ?? "UnnamedAlert"; + const summary = sanitizeText(alert.annotations?.summary ?? ""); + const desc = sanitizeText(alert.annotations?.description ?? ""); + const detail = summary || desc || "No details provided."; + return `${index + 1}. [${severity}] ${alertName} - ${detail}`; + }); +} + +module.exports = { sanitizeText, buildSanitizedAlertLines }; diff --git a/devops/aiops/sanitize.test.js b/devops/aiops/sanitize.test.js new file mode 100644 index 0000000..afd136d --- /dev/null +++ b/devops/aiops/sanitize.test.js @@ -0,0 +1,197 @@ +"use strict"; + +const { describe, it } = require("node:test"); +const assert = require("node:assert/strict"); +const { sanitizeText, buildSanitizedAlertLines } = require("./sanitize.js"); + +describe("sanitizeText — connection strings", () => { + it("redacts postgres:// URIs", () => { + const out = sanitizeText("Error: postgres://admin:s3cr3t@db.prod:5432/loopless"); + assert.doesNotMatch(out, /postgres:\/\//); + assert.match(out, /REDACTED_POSTGRES_URI/); + }); + + it("redacts redis:// URIs", () => { + const out = sanitizeText("Cache miss, reconnecting to redis://:password123@cache:6379/0"); + assert.doesNotMatch(out, /redis:\/\//); + assert.match(out, /REDACTED_REDIS_URI/); + }); + + it("redacts amqp:// URIs", () => { + const out = sanitizeText("Publisher connected: amqp://loopless:secret@rabbitmq:5672/vhost"); + assert.doesNotMatch(out, /amqp:\/\//); + assert.match(out, /REDACTED_AMQP_URI/); + }); +}); + +describe("sanitizeText — tokens", () => { + it("redacts Bearer header values", () => { + const token = "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ1c2VyMTIzIn0.SflKxwRJSMeKKF2QT4fwp"; + const out = sanitizeText(`Authorization: Bearer ${token}`); + assert.doesNotMatch(out, /Bearer ey/); + assert.match(out, /REDACTED/); + }); + + it("redacts raw three-segment JWTs", () => { + const token = "eyJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJ1c2VyMTIzIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV"; + const out = sanitizeText(`token=${token}`); + assert.doesNotMatch(out, /eyJhbGci/); + assert.match(out, /REDACTED/); + }); +}); + +describe("sanitizeText — emails", () => { + it("redacts email addresses", () => { + const out = sanitizeText("Alert triggered for user admin@loopless.dev in namespace production"); + assert.doesNotMatch(out, /admin@/); + assert.match(out, /REDACTED_EMAIL/); + }); + + it("redacts emails embedded mid-sentence", () => { + const out = sanitizeText("Owned by enes.shehu@gjirafa.com — see triage notes"); + assert.doesNotMatch(out, /gjirafa\.com/); + assert.match(out, /REDACTED_EMAIL/); + }); +}); + +describe("sanitizeText — IP addresses", () => { + it("redacts public IPv4 addresses", () => { + const out = sanitizeText("Request from 203.0.113.42 reached the service"); + assert.doesNotMatch(out, /203\.0\.113\.42/); + assert.match(out, /REDACTED_PUBLIC_IP/); + }); + + it("preserves 10.x.x.x (RFC-1918)", () => { + const out = sanitizeText("Pod IP: 10.244.1.5"); + assert.match(out, /10\.244\.1\.5/); + }); + + it("preserves 172.16-31.x.x (RFC-1918)", () => { + const out = sanitizeText("Node: 172.20.0.10"); + assert.match(out, /172\.20\.0\.10/); + }); + + it("preserves 192.168.x.x (RFC-1918)", () => { + const out = sanitizeText("Gateway: 192.168.1.1"); + assert.match(out, /192\.168\.1\.1/); + }); + + it("preserves 127.0.0.1 loopback", () => { + const out = sanitizeText("Bound to 127.0.0.1:8080"); + assert.match(out, /127\.0\.0\.1/); + }); + + it("redacts public and preserves private in same string", () => { + const out = sanitizeText("from 203.0.113.42 to 10.0.0.5"); + assert.doesNotMatch(out, /203\.0\.113\.42/); + assert.match(out, /REDACTED_PUBLIC_IP/); + assert.match(out, /10\.0\.0\.5/); + }); +}); + +describe("sanitizeText — secret label patterns", () => { + for (const kw of ["password", "passwd", "secret", "api_key", "apikey", "token"]) { + it(`redacts ${kw}=`, () => { + const out = sanitizeText(`${kw}=supersecretvalue123`); + assert.doesNotMatch(out, /supersecretvalue/); + assert.match(out, /REDACTED_SECRET/); + }); + } +}); + +describe("sanitizeText — edge cases", () => { + it("returns null unchanged", () => { + assert.strictEqual(sanitizeText(null), null); + }); + + it("returns undefined unchanged", () => { + assert.strictEqual(sanitizeText(undefined), undefined); + }); + + it("returns empty string unchanged", () => { + assert.strictEqual(sanitizeText(""), ""); + }); + + it("leaves clean text unmodified", () => { + const clean = "High CPU usage detected on node worker-1. Consider scaling out."; + assert.strictEqual(sanitizeText(clean), clean); + }); +}); + +describe("buildSanitizedAlertLines — OpenAI prompt safety", () => { + it("includes alertname and severity", () => { + const alerts = [ + { + labels: { alertname: "HighMemoryUsage", severity: "critical" }, + annotations: { summary: "Memory above 90%" }, + }, + ]; + const [line] = buildSanitizedAlertLines(alerts); + assert.match(line, /HighMemoryUsage/); + assert.match(line, /critical/); + }); + + it("redacts connection string in description before sending to OpenAI", () => { + const alerts = [ + { + labels: { alertname: "DBDown", severity: "critical" }, + annotations: { + description: "Cannot connect to postgres://admin:pass@db.prod:5432/loopless", + }, + }, + ]; + const [line] = buildSanitizedAlertLines(alerts); + assert.doesNotMatch(line, /postgres:\/\//); + assert.match(line, /REDACTED_POSTGRES_URI/); + }); + + it("excludes service, job, and runbook_url labels from OpenAI prompt", () => { + const alerts = [ + { + labels: { + alertname: "PodCrashLoop", + severity: "warning", + service: "backend", + job: "loopless-api", + }, + annotations: { + summary: "Pod restarted 5 times", + runbook_url: "https://wiki.internal/runbooks/crash-loop", + }, + }, + ]; + const [line] = buildSanitizedAlertLines(alerts); + assert.doesNotMatch(line, /loopless-api/); + assert.doesNotMatch(line, /runbook_url/); + assert.doesNotMatch(line, /wiki\.internal/); + }); + + it("redacts email in summary annotation", () => { + const alerts = [ + { + labels: { alertname: "UserAlert", severity: "info" }, + annotations: { summary: "Alert owner: ops@loopless.dev, check dashboard" }, + }, + ]; + const [line] = buildSanitizedAlertLines(alerts); + assert.doesNotMatch(line, /ops@/); + assert.match(line, /REDACTED_EMAIL/); + }); + + it("falls back to description when summary is absent", () => { + const alerts = [ + { + labels: { alertname: "TestAlert", severity: "warning" }, + annotations: { description: "Disk at 95% on node-1" }, + }, + ]; + const [line] = buildSanitizedAlertLines(alerts); + assert.match(line, /Disk at 95%/); + }); + + it("uses placeholder when both summary and description are absent", () => { + const alerts = [{ labels: { alertname: "Bare", severity: "info" }, annotations: {} }]; + const [line] = buildSanitizedAlertLines(alerts); + assert.match(line, /No details provided/); + }); +}); diff --git a/devops/aiops/server.js b/devops/aiops/server.js index 3f47f5b..975b629 100644 --- a/devops/aiops/server.js +++ b/devops/aiops/server.js @@ -1,4 +1,5 @@ const { createServer } = require("node:http"); +const { buildSanitizedAlertLines } = require("./sanitize.js"); const PORT = Number(process.env.PORT ?? "8085"); const OPENAI_CHAT_ENDPOINT = process.env.OPENAI_CHAT_ENDPOINT ?? "https://api.openai.com/v1/chat/completions"; @@ -41,7 +42,7 @@ async function summarizeAlerts(alerts) { return fallbackSummary(alerts); } - const alertDetails = buildAlertLines(alerts).join("\n"); + const alertDetails = buildSanitizedAlertLines(alerts).join("\n"); const prompt = [ "You are an on-call incident triage assistant for the Loopless platform.", "Summarize these Prometheus firing alerts in 4-6 concise bullet points.", diff --git a/devops/docker/frontend/Dockerfile b/devops/docker/frontend/Dockerfile index 8727257..c2815f9 100644 --- a/devops/docker/frontend/Dockerfile +++ b/devops/docker/frontend/Dockerfile @@ -10,24 +10,21 @@ RUN apk add --no-cache libc6-compat COPY package.json package-lock.json ./ RUN npm ci -# ---- builder: produce the optimized .next output ---- +# ---- builder: produce the optimized standalone .next output ---- FROM node:20-alpine AS builder WORKDIR /app ENV NEXT_TELEMETRY_DISABLED=1 -# NEXT_PUBLIC_* values are inlined into the JS bundle at build time, so they must -# be present BEFORE `next build`. CI passes them via --build-arg. -ARG NEXT_PUBLIC_API_BASE_URL -ARG NEXT_PUBLIC_KEYCLOAK_URL -ENV NEXT_PUBLIC_API_BASE_URL=$NEXT_PUBLIC_API_BASE_URL -ENV NEXT_PUBLIC_KEYCLOAK_URL=$NEXT_PUBLIC_KEYCLOAK_URL +# NOTE: NO NEXT_PUBLIC_* build args. Public config is injected at RUNTIME via window.__ENV +# (see the entrypoint below + lib/clientEnv.ts), so this image is environment-agnostic and +# built once for all environments. COPY --from=deps /app/node_modules ./node_modules COPY . . -# .next/cache holds Next's incremental rebuild cache; next start doesn't -# read it at runtime, so dropping it shaves ~40-80MB off the runner image. +# `output: "standalone"` (next.config.mjs) emits .next/standalone with a minimal server. +# .next/cache is build-only; drop it to keep the artifact small. RUN npm run build && rm -rf .next/cache # ---- runner: minimal production image ---- @@ -42,16 +39,33 @@ ENV HOSTNAME=0.0.0.0 RUN addgroup --system --gid 1001 nextjs \ && adduser --system --uid 1001 nextjs -# Reinstall production-only deps in the runner so devDeps don't bloat the image. -COPY package.json package-lock.json ./ -RUN npm ci --omit=dev && npm cache clean --force - -COPY --from=builder --chown=nextjs:nextjs /app/.next ./.next -COPY --from=builder --chown=nextjs:nextjs /app/public ./public -COPY --from=builder --chown=nextjs:nextjs /app/next.config.mjs ./next.config.mjs +# Standalone bundle ships its own minimal node_modules + server.js — no npm install needed. +COPY --from=builder --chown=nextjs:nextjs /app/.next/standalone ./ +COPY --from=builder --chown=nextjs:nextjs /app/.next/static ./.next/static +COPY --from=builder --chown=nextjs:nextjs /app/public ./public + +# Runtime env injection: write public/__env.js from APP_* at container start, then exec the +# server. Lets one immutable image run in any environment (12-factor config). Generated inline +# so it stays within the frontend/client build context. +RUN cat <<'ENTRYPOINT' > /usr/local/bin/docker-entrypoint.sh +#!/bin/sh +set -e +cat > /app/public/__env.js < plain k8s Secret only + # Restrict /metrics to the in-cluster Prometheus scrape (Host: loopless-backend:8080). + # Snippet directives are disabled on this cluster's ingress, so path-blocking happens + # in-app by Host: the public api host is not allow-listed -> /metrics returns 404 to + # the internet while the in-cluster scrape still works. See Program.cs MapMetrics. + Metrics__AllowedHosts__0: "loopless-backend:8080" # Object storage (in-cluster MinIO). Access/secret keys come from the Secret. Storage__ServiceUrl: "http://minio:9000" Storage__BucketName: "loopless-uploads" @@ -137,8 +142,15 @@ config: Serilog__WriteTo__1__Args__requestUri: "http://logstash:5044" Serilog__WriteTo__1__Args__queueLimitBytes: "" frontend: - NEXT_PUBLIC_API_BASE_URL: "https://api.project-01.gjirafa.dev" - NEXT_PUBLIC_ENVIRONMENT: "production" + # Runtime public config: the container entrypoint writes these into public/__env.js -> + # window.__ENV, read by frontend/client/lib/clientEnv.ts. The image is env-agnostic + # (built once), so changing an environment is a values change + restart, not a rebuild. + APP_API_BASE_URL: "https://api.project-01.gjirafa.dev" + APP_KEYCLOAK_URL: "https://api.project-01.gjirafa.dev/auth" + APP_ENABLE_RBAC: "true" + # PostHog analytics (optional). Leave blank to skip client-side init. + APP_POSTHOG_KEY: "" + APP_POSTHOG_HOST: "" # cert-manager namespaced ACME Issuer (no usable shared ClusterIssuer on this cluster). certIssuer: diff --git a/devops/terraform/.gitignore b/devops/terraform/.gitignore deleted file mode 100644 index f163a60..0000000 --- a/devops/terraform/.gitignore +++ /dev/null @@ -1,17 +0,0 @@ -.terraform/ -.terraform.lock.hcl -*.tfstate -*.tfstate.* -*.tfstate.backup -crash.log -crash.*.log - -terraform.tfvars -*.auto.tfvars -override.tf -override.tf.json -*_override.tf -*_override.tf.json - -.terraformrc -terraform.rc diff --git a/devops/terraform/README.md b/devops/terraform/README.md deleted file mode 100644 index 380bfec..0000000 --- a/devops/terraform/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# Loopless Terraform - -Reusable AWS infrastructure for Loopless. Modules under `modules/`, environment compositions under `envs/`. - -> **Looking for step-by-step apply instructions?** See [`RUNBOOK.md`](RUNBOOK.md) — designed to get you from `git clone` to `terraform apply` in under 15 minutes. - -## Layout - -``` -devops/terraform/ -├── backend.tf # Backend stub (real config passed via -backend-config) -├── versions.tf # Terraform + provider version constraints -├── modules/ -│ ├── network/ # VPC, subnets, SGs, NAT -│ ├── database/ # RDS PostgreSQL 16 + pgvector parameter group -│ ├── cache/ # ElastiCache Redis 7 -│ └── storage/ # S3 bucket for uploads -└── envs/ - ├── staging/ # Small instances, single-AZ - └── production/ # Larger instances, multi-AZ, longer backups -``` - -## One-time bootstrap - -Create the remote-state bucket + lock table per environment (chicken-and-egg with Terraform state itself, so done out-of-band): - -```bash -aws s3api create-bucket --bucket loopless-tfstate-staging --region eu-central-1 \ - --create-bucket-configuration LocationConstraint=eu-central-1 -aws s3api put-bucket-versioning --bucket loopless-tfstate-staging \ - --versioning-configuration Status=Enabled - -aws dynamodb create-table --table-name loopless-tfstate-lock \ - --attribute-definitions AttributeName=LockID,AttributeType=S \ - --key-schema AttributeName=LockID,KeyType=HASH \ - --billing-mode PAY_PER_REQUEST --region eu-central-1 -``` - -Repeat the bucket step for `loopless-tfstate-production`. The lock table is shared. - -## Plan / apply - -```bash -cd envs/staging -terraform init -backend-config=backend-staging.hcl -terraform plan -var-file=terraform.tfvars -terraform apply -var-file=terraform.tfvars -``` - -`terraform.tfvars` is gitignored — copy `terraform.tfvars.example` and fill in `db_master_password` + `bucket_suffix`. - -## Offline validation - -```bash -cd envs/staging -terraform init -backend=false -terraform validate -terraform fmt -recursive ../.. -``` - -## pgvector - -The database module loads the `vector` extension via the `shared_preload_libraries` parameter group. After provisioning, app migrations must still run `CREATE EXTENSION IF NOT EXISTS vector;` against the database. - -## Tagging - -All resources get `{Project = "loopless", Environment = "", ManagedBy = "terraform"}` plus per-resource `Name` tags. diff --git a/devops/terraform/RUNBOOK.md b/devops/terraform/RUNBOOK.md deleted file mode 100644 index 3780740..0000000 --- a/devops/terraform/RUNBOOK.md +++ /dev/null @@ -1,236 +0,0 @@ -# Terraform Apply Runbook - -Operational guide for provisioning Loopless AWS infrastructure with Terraform. Designed so a new operator can go from "git pulled the repo" to "apply complete" in under 15 minutes. - -For module/architecture overview, see [`README.md`](README.md). This document is **procedural** — follow the steps top-to-bottom. - ---- - -## Prerequisites - -| Tool | Version | Install | -|---|---|---| -| Terraform | `~> 1.7` | `brew install terraform` / [terraform.io/downloads](https://terraform.io/downloads) | -| AWS CLI | `>= 2.13` | `brew install awscli` / [aws.amazon.com/cli](https://aws.amazon.com/cli/) | -| jq | any | `brew install jq` (used in helper commands) | - -**AWS credentials** — configure once on the machine running Terraform: - -```bash -aws configure --profile loopless -# Enter access key, secret, region (eu-central-1) -export AWS_PROFILE=loopless -``` - -The IAM principal needs **AdministratorAccess** for initial bootstrap. After first apply, scope it down to: `ec2:*`, `rds:*`, `elasticache:*`, `s3:*`, `iam:*` (for role creation), `dynamodb:*` (state lock), `kms:*` (for SSE keys). - ---- - -## Step 1 — One-time backend bootstrap - -The S3 bucket that stores Terraform state can't be managed by Terraform itself (chicken-and-egg). Create it manually, **once per environment**: - -```bash -# Staging state bucket -aws s3api create-bucket \ - --bucket loopless-tfstate-staging \ - --region eu-central-1 \ - --create-bucket-configuration LocationConstraint=eu-central-1 - -aws s3api put-bucket-versioning \ - --bucket loopless-tfstate-staging \ - --versioning-configuration Status=Enabled - -aws s3api put-bucket-encryption \ - --bucket loopless-tfstate-staging \ - --server-side-encryption-configuration \ - '{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"AES256"}}]}' - -aws s3api put-public-access-block \ - --bucket loopless-tfstate-staging \ - --public-access-block-configuration \ - 'BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true' - -# Shared lock table (one table works for both envs — state keys differ) -aws dynamodb create-table \ - --table-name loopless-tfstate-lock \ - --attribute-definitions AttributeName=LockID,AttributeType=S \ - --key-schema AttributeName=LockID,KeyType=HASH \ - --billing-mode PAY_PER_REQUEST \ - --region eu-central-1 -``` - -Repeat the bucket steps for `loopless-tfstate-production` when you're ready for prod. **Skip this step entirely on subsequent applies.** - ---- - -## Step 2 — Fill in `terraform.tfvars` - -```bash -cd devops/terraform/envs/staging -cp terraform.tfvars.example terraform.tfvars -``` - -Edit `terraform.tfvars`: - -| Variable | Where to get it | Example | -|---|---|---| -| `region` | Already defaulted to `eu-central-1` | `eu-central-1` | -| `db_master_password` | Generate with `openssl rand -base64 32` — **save in your password manager** | `Th1s1sAStr0ngP@ss...` | -| `bucket_suffix` | Last 6 digits of your AWS account ID (run `aws sts get-caller-identity --query Account --output text \| tail -c 7`) | `123456` | - -`terraform.tfvars` is in `.gitignore` — never commit it. - -> **Alternative for `db_master_password`**: pass via env var to avoid writing it to disk: -> ```bash -> export TF_VAR_db_master_password=$(openssl rand -base64 32) -> ``` - ---- - -## Step 3 — `terraform init` - -```bash -# From devops/terraform/envs/staging -terraform init -backend-config=backend-staging.hcl -``` - -Downloads the AWS provider and connects to the remote-state S3 bucket. First run takes ~30s; subsequent inits are cached. - -**Switching environments?** Run `rm -rf .terraform/` first to clear the previous backend cache, then re-init in the new env directory. - ---- - -## Step 4 — `terraform plan` - -```bash -terraform plan -var-file=terraform.tfvars -out=staging.tfplan -``` - -**Review the output before applying.** Expected on a first run: - -- `+ 1` VPC + 4 subnets + IGW + NAT + 2 route tables (~10 network resources) -- `+ 1` RDS instance + subnet group + parameter group -- `+ 1` ElastiCache replication group + subnet group -- `+ 1` S3 bucket + 4 sub-resources (versioning, encryption, public-access-block, lifecycle) - -Total: roughly **20-25 resources** added, zero changed, zero destroyed. - -If you see `~` (modify) or `-` (destroy) on a first run, **stop** and investigate — something is misaligned with what's already in AWS. - ---- - -## Step 5 — `terraform apply` - -```bash -terraform apply staging.tfplan -``` - -Using the saved plan file ensures you apply **exactly** what you reviewed in Step 4. No surprise drift. - -Wall-clock time: -- VPC + subnets: ~30s -- RDS: **10–15 minutes** (slowest) -- ElastiCache: ~7 minutes -- S3: ~10s - -Total: budget **~20 minutes** for an end-to-end first apply. - ---- - -## Step 6 — Capture outputs - -```bash -terraform output -# vpc_id = "vpc-0abc123" -# private_subnet_ids = ["subnet-aaa", "subnet-bbb"] -# db_endpoint = (sensitive) -# redis_primary_endpoint = (sensitive) -# uploads_bucket = "loopless-uploads-staging-123456" - -# Reveal sensitive values -terraform output -raw db_endpoint -terraform output -raw redis_primary_endpoint -``` - -Feed these into your `.env.production` (or push to Azure Key Vault per CP-208): - -```bash -ConnectionStrings__PostgreSQL=Host=$(terraform output -raw db_endpoint | cut -d: -f1);Port=5432;Database=loopless;Username=loopless;Password= -ConnectionStrings__Redis=$(terraform output -raw redis_primary_endpoint):6379 -Storage__BucketName=$(terraform output -raw uploads_bucket) -``` - ---- - -## Step 7 — Post-provision database setup - -Terraform creates the RDS instance with `shared_preload_libraries=vector` in the parameter group, but the extension still needs to be enabled inside the database. The app's EF Core migrations handle this on first start by issuing: - -```sql -CREATE EXTENSION IF NOT EXISTS vector; -``` - -**Verify manually** if migrations haven't run yet: - -```bash -# From a machine inside the VPC (bastion, ECS task, etc.) -psql "host=$(terraform output -raw db_endpoint | cut -d: -f1) \ - port=5432 dbname=loopless user=loopless" \ - -c "CREATE EXTENSION IF NOT EXISTS vector; SELECT extversion FROM pg_extension WHERE extname='vector';" -``` - -You should see a version like `0.7.x`. If you get "shared_preload_libraries does not include vector", the RDS instance needs a reboot (Terraform applies the param group but reboot is required for `shared_preload_libraries`). - ---- - -## Step 8 — Destroy (cleanup) - -```bash -cd devops/terraform/envs/staging -terraform destroy -var-file=terraform.tfvars -``` - -For staging this fully tears down. For production, the RDS instance has `deletion_protection = true` and `skip_final_snapshot = false` — Terraform will refuse to destroy until you: - -1. Disable deletion protection: `terraform apply -var-file=... -var='deletion_protection=false'` (requires propagating the variable through the module) -2. Or do it via console: RDS → modify → Disable deletion protection → apply immediately -3. Then re-run `terraform destroy` — it will take a final snapshot named `loopless-production-db-final` - -The S3 uploads bucket is also retained on destroy (versioning keeps history). Empty + delete manually if you're truly tearing down: - -```bash -aws s3 rb s3://loopless-uploads-staging-123456 --force -``` - ---- - -## Troubleshooting - -| Symptom | Cause | Fix | -|---|---|---| -| `Error: BucketAlreadyExists` on state init | Bucket name globally taken | Use a different account-id suffix | -| `Error: failed to retrieve lock` on init | Lock table missing or wrong region | Re-create the DynamoDB lock table per Step 1 | -| `InvalidParameterValue: shared_preload_libraries` | RDS hasn't rebooted since param group attached | Console → reboot DB instance | -| `terraform plan` shows recreate on existing resources | Backend config mismatch or state corruption | `terraform state list` — verify resources are tracked; never edit state by hand | -| CI workflow fails `terraform validate` | A `.tf` file has a syntax error | Run `terraform fmt -recursive` locally; commit; push | - ---- - -## CI validation - -Every push that touches `devops/terraform/**` triggers [`.github/workflows/terraform.yml`](../../.github/workflows/terraform.yml). It runs: - -- `terraform fmt -check -recursive` — fail on unformatted code -- `terraform init -backend=false` per env -- `terraform validate` per env - -This catches typos and missing variable references **before** anyone runs `apply`. The workflow does NOT have AWS credentials and never calls `plan` or `apply` — strictly static analysis. - ---- - -## Related - -- [CP-205 — Let's Encrypt nginx config](../scripts/init-letsencrypt.sh) — runs against the EC2/server you provision separately -- [CP-206 — Server hardening](../scripts/server/bootstrap-ubuntu.sh) — what to run on the EC2 host -- [CP-208 — Key Vault wiring](../../backend/src/Loopless.Api/Program.cs) — alternative to writing secrets into `.env.production` diff --git a/devops/terraform/backend.tf b/devops/terraform/backend.tf deleted file mode 100644 index 3d26295..0000000 --- a/devops/terraform/backend.tf +++ /dev/null @@ -1,18 +0,0 @@ -# Remote state lives in S3 with a DynamoDB lock table. -# The actual bucket / table / key are passed at init time via: -# terraform init -backend-config=backend-.hcl -# This avoids hardcoding env-specific state locations in shared code. -# -# Bootstrap (one-time, manual; chicken-and-egg with state itself): -# aws s3api create-bucket --bucket loopless-tfstate- --region eu-central-1 \ -# --create-bucket-configuration LocationConstraint=eu-central-1 -# aws s3api put-bucket-versioning --bucket loopless-tfstate- \ -# --versioning-configuration Status=Enabled -# aws dynamodb create-table --table-name loopless-tfstate-lock \ -# --attribute-definitions AttributeName=LockID,AttributeType=S \ -# --key-schema AttributeName=LockID,KeyType=HASH \ -# --billing-mode PAY_PER_REQUEST --region eu-central-1 - -terraform { - backend "s3" {} -} diff --git a/devops/terraform/envs/production/backend-production.hcl b/devops/terraform/envs/production/backend-production.hcl deleted file mode 100644 index b5fe0d8..0000000 --- a/devops/terraform/envs/production/backend-production.hcl +++ /dev/null @@ -1,5 +0,0 @@ -bucket = "loopless-tfstate-production" -key = "production/terraform.tfstate" -region = "eu-central-1" -dynamodb_table = "loopless-tfstate-lock" -encrypt = true diff --git a/devops/terraform/envs/production/backend.tf b/devops/terraform/envs/production/backend.tf deleted file mode 100644 index 3835768..0000000 --- a/devops/terraform/envs/production/backend.tf +++ /dev/null @@ -1,5 +0,0 @@ -# Backend config supplied via: -# terraform init -backend-config=backend-production.hcl -terraform { - backend "s3" {} -} diff --git a/devops/terraform/envs/production/main.tf b/devops/terraform/envs/production/main.tf deleted file mode 100644 index eb08abb..0000000 --- a/devops/terraform/envs/production/main.tf +++ /dev/null @@ -1,94 +0,0 @@ -terraform { - required_version = "~> 1.7" - - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.0" - } - } -} - -provider "aws" { - region = var.region - - default_tags { - tags = local.common_tags - } -} - -locals { - environment = "production" - - common_tags = { - Project = "loopless" - Environment = local.environment - ManagedBy = "terraform" - } -} - -module "network" { - source = "../../modules/network" - - environment = local.environment - vpc_cidr = "10.20.0.0/16" - public_subnet_cidrs = ["10.20.0.0/24", "10.20.1.0/24"] - private_subnet_cidrs = ["10.20.10.0/24", "10.20.11.0/24"] - single_nat_gateway = true - tags = local.common_tags -} - -module "database" { - source = "../../modules/database" - - environment = local.environment - subnet_ids = module.network.private_subnet_ids - vpc_security_group_ids = [module.network.db_security_group_id] - - instance_class = "db.m6g.large" - allocated_storage = 100 - max_allocated_storage = 500 - multi_az = true - backup_retention_period = 30 - deletion_protection = true - skip_final_snapshot = false - performance_insights_enabled = true - master_password = var.db_master_password - - tags = local.common_tags -} - -module "cache" { - source = "../../modules/cache" - - environment = local.environment - subnet_ids = module.network.private_subnet_ids - security_group_ids = [module.network.cache_security_group_id] - - node_type = "cache.m6g.large" - num_cache_clusters = 2 - automatic_failover_enabled = true - transit_encryption_enabled = true - snapshot_retention_limit = 7 - - tags = local.common_tags -} - -module "storage" { - source = "../../modules/storage" - - environment = local.environment - bucket_name = "loopless-uploads-${local.environment}-${var.bucket_suffix}" - noncurrent_version_expiration_days = 90 - - tags = local.common_tags -} - -module "secrets" { - source = "../../modules/secrets" - - environment = local.environment - recovery_window_days = 30 - - tags = local.common_tags -} diff --git a/devops/terraform/envs/production/outputs.tf b/devops/terraform/envs/production/outputs.tf deleted file mode 100644 index a8e4f6b..0000000 --- a/devops/terraform/envs/production/outputs.tf +++ /dev/null @@ -1,34 +0,0 @@ -output "vpc_id" { - value = module.network.vpc_id -} - -output "private_subnet_ids" { - value = module.network.private_subnet_ids -} - -output "db_endpoint" { - description = "RDS endpoint (host:port)." - value = module.database.endpoint - sensitive = true -} - -output "redis_primary_endpoint" { - description = "Redis primary endpoint." - value = module.cache.primary_endpoint - sensitive = true -} - -output "uploads_bucket" { - description = "S3 bucket for user uploads." - value = module.storage.bucket_id -} - -output "secret_arns" { - description = "Map of secret name → ARN. Use in K8s ExternalSecrets manifests." - value = module.secrets.secret_arns -} - -output "secrets_read_policy_arn" { - description = "IAM policy ARN granting read-only access to this env's secrets." - value = module.secrets.read_only_policy_arn -} diff --git a/devops/terraform/envs/production/terraform.tfvars.example b/devops/terraform/envs/production/terraform.tfvars.example deleted file mode 100644 index 3208651..0000000 --- a/devops/terraform/envs/production/terraform.tfvars.example +++ /dev/null @@ -1,4 +0,0 @@ -# Copy to terraform.tfvars (gitignored) and fill in real values. -region = "eu-central-1" -db_master_password = "REPLACE_ME_WITH_STRONG_PASSWORD" -bucket_suffix = "123456" # e.g. last 6 digits of AWS account id diff --git a/devops/terraform/envs/production/variables.tf b/devops/terraform/envs/production/variables.tf deleted file mode 100644 index 589ddbe..0000000 --- a/devops/terraform/envs/production/variables.tf +++ /dev/null @@ -1,16 +0,0 @@ -variable "region" { - description = "AWS region." - type = string - default = "eu-central-1" -} - -variable "db_master_password" { - description = "Master password for the production RDS instance. Pass via TF_VAR_db_master_password." - type = string - sensitive = true -} - -variable "bucket_suffix" { - description = "Suffix to make the uploads bucket name globally unique (e.g. AWS account id last 6 digits)." - type = string -} diff --git a/devops/terraform/envs/staging/backend-staging.hcl b/devops/terraform/envs/staging/backend-staging.hcl deleted file mode 100644 index 3a5632b..0000000 --- a/devops/terraform/envs/staging/backend-staging.hcl +++ /dev/null @@ -1,5 +0,0 @@ -bucket = "loopless-tfstate-staging" -key = "staging/terraform.tfstate" -region = "eu-central-1" -dynamodb_table = "loopless-tfstate-lock" -encrypt = true diff --git a/devops/terraform/envs/staging/backend.tf b/devops/terraform/envs/staging/backend.tf deleted file mode 100644 index a854806..0000000 --- a/devops/terraform/envs/staging/backend.tf +++ /dev/null @@ -1,5 +0,0 @@ -# Backend config supplied via: -# terraform init -backend-config=backend-staging.hcl -terraform { - backend "s3" {} -} diff --git a/devops/terraform/envs/staging/main.tf b/devops/terraform/envs/staging/main.tf deleted file mode 100644 index ca01e78..0000000 --- a/devops/terraform/envs/staging/main.tf +++ /dev/null @@ -1,93 +0,0 @@ -terraform { - required_version = "~> 1.7" - - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.0" - } - } -} - -provider "aws" { - region = var.region - - default_tags { - tags = local.common_tags - } -} - -locals { - environment = "staging" - - common_tags = { - Project = "loopless" - Environment = local.environment - ManagedBy = "terraform" - } -} - -module "network" { - source = "../../modules/network" - - environment = local.environment - vpc_cidr = "10.10.0.0/16" - public_subnet_cidrs = ["10.10.0.0/24", "10.10.1.0/24"] - private_subnet_cidrs = ["10.10.10.0/24", "10.10.11.0/24"] - single_nat_gateway = true - tags = local.common_tags -} - -module "database" { - source = "../../modules/database" - - environment = local.environment - subnet_ids = module.network.private_subnet_ids - vpc_security_group_ids = [module.network.db_security_group_id] - - instance_class = "db.t4g.micro" - allocated_storage = 20 - max_allocated_storage = 100 - multi_az = false - backup_retention_period = 7 - deletion_protection = false - skip_final_snapshot = true - master_password = var.db_master_password - - tags = local.common_tags -} - -module "cache" { - source = "../../modules/cache" - - environment = local.environment - subnet_ids = module.network.private_subnet_ids - security_group_ids = [module.network.cache_security_group_id] - - node_type = "cache.t4g.micro" - num_cache_clusters = 1 - automatic_failover_enabled = false - transit_encryption_enabled = true - snapshot_retention_limit = 1 - - tags = local.common_tags -} - -module "storage" { - source = "../../modules/storage" - - environment = local.environment - bucket_name = "loopless-uploads-${local.environment}-${var.bucket_suffix}" - noncurrent_version_expiration_days = 30 - - tags = local.common_tags -} - -module "secrets" { - source = "../../modules/secrets" - - environment = local.environment - recovery_window_days = 7 - - tags = local.common_tags -} diff --git a/devops/terraform/envs/staging/outputs.tf b/devops/terraform/envs/staging/outputs.tf deleted file mode 100644 index a8e4f6b..0000000 --- a/devops/terraform/envs/staging/outputs.tf +++ /dev/null @@ -1,34 +0,0 @@ -output "vpc_id" { - value = module.network.vpc_id -} - -output "private_subnet_ids" { - value = module.network.private_subnet_ids -} - -output "db_endpoint" { - description = "RDS endpoint (host:port)." - value = module.database.endpoint - sensitive = true -} - -output "redis_primary_endpoint" { - description = "Redis primary endpoint." - value = module.cache.primary_endpoint - sensitive = true -} - -output "uploads_bucket" { - description = "S3 bucket for user uploads." - value = module.storage.bucket_id -} - -output "secret_arns" { - description = "Map of secret name → ARN. Use in K8s ExternalSecrets manifests." - value = module.secrets.secret_arns -} - -output "secrets_read_policy_arn" { - description = "IAM policy ARN granting read-only access to this env's secrets." - value = module.secrets.read_only_policy_arn -} diff --git a/devops/terraform/envs/staging/terraform.tfvars.example b/devops/terraform/envs/staging/terraform.tfvars.example deleted file mode 100644 index 3208651..0000000 --- a/devops/terraform/envs/staging/terraform.tfvars.example +++ /dev/null @@ -1,4 +0,0 @@ -# Copy to terraform.tfvars (gitignored) and fill in real values. -region = "eu-central-1" -db_master_password = "REPLACE_ME_WITH_STRONG_PASSWORD" -bucket_suffix = "123456" # e.g. last 6 digits of AWS account id diff --git a/devops/terraform/envs/staging/variables.tf b/devops/terraform/envs/staging/variables.tf deleted file mode 100644 index 79914fe..0000000 --- a/devops/terraform/envs/staging/variables.tf +++ /dev/null @@ -1,16 +0,0 @@ -variable "region" { - description = "AWS region." - type = string - default = "eu-central-1" -} - -variable "db_master_password" { - description = "Master password for the staging RDS instance. Pass via TF_VAR_db_master_password." - type = string - sensitive = true -} - -variable "bucket_suffix" { - description = "Random suffix to make the uploads bucket name globally unique (e.g. AWS account id last 6 digits)." - type = string -} diff --git a/devops/terraform/modules/cache/main.tf b/devops/terraform/modules/cache/main.tf deleted file mode 100644 index f5b2d23..0000000 --- a/devops/terraform/modules/cache/main.tf +++ /dev/null @@ -1,49 +0,0 @@ -terraform { - required_version = "~> 1.7" - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.0" - } - } -} - -locals { - name = "loopless-${var.environment}-cache" - base_tags = merge(var.tags, { Name = local.name }) -} - -resource "aws_elasticache_subnet_group" "this" { - name = local.name - subnet_ids = var.subnet_ids - tags = local.base_tags -} - -resource "aws_elasticache_replication_group" "this" { - replication_group_id = local.name - description = "Loopless ${var.environment} Redis 7 cluster" - - engine = "redis" - engine_version = "7.1" - node_type = var.node_type - port = 6379 - - num_cache_clusters = var.num_cache_clusters - automatic_failover_enabled = var.automatic_failover_enabled - multi_az_enabled = var.automatic_failover_enabled - - subnet_group_name = aws_elasticache_subnet_group.this.name - security_group_ids = var.security_group_ids - parameter_group_name = "default.redis7" - - at_rest_encryption_enabled = true - transit_encryption_enabled = var.transit_encryption_enabled - - snapshot_retention_limit = var.snapshot_retention_limit - snapshot_window = "04:00-05:00" - maintenance_window = "sun:05:30-sun:06:30" - - apply_immediately = false - - tags = local.base_tags -} diff --git a/devops/terraform/modules/cache/outputs.tf b/devops/terraform/modules/cache/outputs.tf deleted file mode 100644 index 134ed6d..0000000 --- a/devops/terraform/modules/cache/outputs.tf +++ /dev/null @@ -1,19 +0,0 @@ -output "primary_endpoint" { - description = "Primary endpoint address for read/write connections." - value = aws_elasticache_replication_group.this.primary_endpoint_address -} - -output "reader_endpoint" { - description = "Reader endpoint for read-only connections." - value = aws_elasticache_replication_group.this.reader_endpoint_address -} - -output "port" { - description = "Port the cluster listens on." - value = aws_elasticache_replication_group.this.port -} - -output "replication_group_id" { - description = "Replication group identifier." - value = aws_elasticache_replication_group.this.id -} diff --git a/devops/terraform/modules/cache/variables.tf b/devops/terraform/modules/cache/variables.tf deleted file mode 100644 index 9ca47ae..0000000 --- a/devops/terraform/modules/cache/variables.tf +++ /dev/null @@ -1,49 +0,0 @@ -variable "environment" { - description = "Environment name (e.g. staging, production)." - type = string -} - -variable "subnet_ids" { - description = "Private subnet ids for the ElastiCache subnet group." - type = list(string) -} - -variable "security_group_ids" { - description = "Security group ids attached to the cluster." - type = list(string) -} - -variable "node_type" { - description = "ElastiCache node type (e.g. cache.t4g.micro, cache.m6g.large)." - type = string -} - -variable "num_cache_clusters" { - description = "Number of cache nodes in the replication group (min 1; >=2 required for failover)." - type = number - default = 1 -} - -variable "automatic_failover_enabled" { - description = "Enable automatic failover. Requires num_cache_clusters >= 2." - type = bool - default = false -} - -variable "transit_encryption_enabled" { - description = "Enable in-transit TLS encryption." - type = bool - default = true -} - -variable "snapshot_retention_limit" { - description = "Days to retain Redis snapshots." - type = number - default = 1 -} - -variable "tags" { - description = "Common tags." - type = map(string) - default = {} -} diff --git a/devops/terraform/modules/database/main.tf b/devops/terraform/modules/database/main.tf deleted file mode 100644 index 033c089..0000000 --- a/devops/terraform/modules/database/main.tf +++ /dev/null @@ -1,73 +0,0 @@ -terraform { - required_version = "~> 1.7" - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.0" - } - } -} - -locals { - name = "loopless-${var.environment}-db" - base_tags = merge(var.tags, { Name = local.name }) -} - -resource "aws_db_subnet_group" "this" { - name = local.name - subnet_ids = var.subnet_ids - tags = merge(local.base_tags, { Name = "${local.name}-subnet-group" }) -} - -# Preloads pgvector so the extension can be created post-provision via: -# CREATE EXTENSION IF NOT EXISTS vector; -# This is run by the application's EF Core migrations, not Terraform. -resource "aws_db_parameter_group" "this" { - name = local.name - family = "postgres16" - description = "Loopless ${var.environment} PostgreSQL 16 with pgvector preloaded." - - parameter { - name = "shared_preload_libraries" - value = "vector" - apply_method = "pending-reboot" - } - - tags = merge(local.base_tags, { Name = "${local.name}-pg" }) -} - -resource "aws_db_instance" "this" { - identifier = local.name - engine = "postgres" - engine_version = "16" - instance_class = var.instance_class - - allocated_storage = var.allocated_storage - max_allocated_storage = var.max_allocated_storage - storage_type = "gp3" - storage_encrypted = true - - db_name = var.db_name - username = var.master_username - password = var.master_password - port = 5432 - - db_subnet_group_name = aws_db_subnet_group.this.name - vpc_security_group_ids = var.vpc_security_group_ids - parameter_group_name = aws_db_parameter_group.this.name - publicly_accessible = false - - multi_az = var.multi_az - backup_retention_period = var.backup_retention_period - backup_window = "02:00-03:00" - maintenance_window = "sun:03:30-sun:04:30" - - deletion_protection = var.deletion_protection - skip_final_snapshot = var.skip_final_snapshot - final_snapshot_identifier = var.skip_final_snapshot ? null : "${local.name}-final" - apply_immediately = false - - performance_insights_enabled = var.performance_insights_enabled - - tags = local.base_tags -} diff --git a/devops/terraform/modules/database/outputs.tf b/devops/terraform/modules/database/outputs.tf deleted file mode 100644 index 1a468ab..0000000 --- a/devops/terraform/modules/database/outputs.tf +++ /dev/null @@ -1,29 +0,0 @@ -output "endpoint" { - description = "Connection endpoint (host:port)." - value = aws_db_instance.this.endpoint -} - -output "address" { - description = "DNS address of the instance." - value = aws_db_instance.this.address -} - -output "port" { - description = "Port the DB listens on." - value = aws_db_instance.this.port -} - -output "db_name" { - description = "Initial database name." - value = aws_db_instance.this.db_name -} - -output "instance_id" { - description = "RDS instance identifier." - value = aws_db_instance.this.id -} - -output "parameter_group_name" { - description = "Name of the DB parameter group with pgvector preloaded." - value = aws_db_parameter_group.this.name -} diff --git a/devops/terraform/modules/database/variables.tf b/devops/terraform/modules/database/variables.tf deleted file mode 100644 index ad22b8d..0000000 --- a/devops/terraform/modules/database/variables.tf +++ /dev/null @@ -1,85 +0,0 @@ -variable "environment" { - description = "Environment name (e.g. staging, production)." - type = string -} - -variable "subnet_ids" { - description = "Private subnet ids for the DB subnet group. Need at least 2 across different AZs." - type = list(string) -} - -variable "vpc_security_group_ids" { - description = "Security group ids attached to the RDS instance." - type = list(string) -} - -variable "instance_class" { - description = "RDS instance class (e.g. db.t4g.micro, db.m6g.large)." - type = string -} - -variable "allocated_storage" { - description = "Initial allocated storage in GB." - type = number - default = 20 -} - -variable "max_allocated_storage" { - description = "Upper limit for storage autoscaling, in GB." - type = number - default = 100 -} - -variable "db_name" { - description = "Initial database name." - type = string - default = "loopless" -} - -variable "master_username" { - description = "Master username for the DB." - type = string - default = "loopless_admin" -} - -variable "master_password" { - description = "Master password. Pass via TF_VAR_db_master_password or a secret store." - type = string - sensitive = true -} - -variable "backup_retention_period" { - description = "Days of automated backups to retain." - type = number - default = 7 -} - -variable "multi_az" { - description = "Enable Multi-AZ standby replica." - type = bool - default = false -} - -variable "deletion_protection" { - description = "Prevent accidental deletion of the instance." - type = bool - default = true -} - -variable "skip_final_snapshot" { - description = "Skip taking a final snapshot on destroy. Use true for staging only." - type = bool - default = false -} - -variable "performance_insights_enabled" { - description = "Enable RDS Performance Insights." - type = bool - default = false -} - -variable "tags" { - description = "Common tags." - type = map(string) - default = {} -} diff --git a/devops/terraform/modules/network/main.tf b/devops/terraform/modules/network/main.tf deleted file mode 100644 index b03d174..0000000 --- a/devops/terraform/modules/network/main.tf +++ /dev/null @@ -1,154 +0,0 @@ -terraform { - required_version = "~> 1.7" - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.0" - } - } -} - -data "aws_availability_zones" "available" { - state = "available" -} - -locals { - azs = slice(data.aws_availability_zones.available.names, 0, 2) - name = "loopless-${var.environment}" - base_tags = merge(var.tags, { Name = local.name }) -} - -resource "aws_vpc" "this" { - cidr_block = var.vpc_cidr - enable_dns_support = true - enable_dns_hostnames = true - - tags = merge(local.base_tags, { Name = "${local.name}-vpc" }) -} - -resource "aws_internet_gateway" "this" { - vpc_id = aws_vpc.this.id - tags = merge(local.base_tags, { Name = "${local.name}-igw" }) -} - -resource "aws_subnet" "public" { - count = length(var.public_subnet_cidrs) - vpc_id = aws_vpc.this.id - cidr_block = var.public_subnet_cidrs[count.index] - availability_zone = local.azs[count.index] - map_public_ip_on_launch = true - - tags = merge(local.base_tags, { - Name = "${local.name}-public-${local.azs[count.index]}" - Tier = "public" - }) -} - -resource "aws_subnet" "private" { - count = length(var.private_subnet_cidrs) - vpc_id = aws_vpc.this.id - cidr_block = var.private_subnet_cidrs[count.index] - availability_zone = local.azs[count.index] - - tags = merge(local.base_tags, { - Name = "${local.name}-private-${local.azs[count.index]}" - Tier = "private" - }) -} - -resource "aws_eip" "nat" { - count = var.single_nat_gateway ? 1 : length(var.public_subnet_cidrs) - domain = "vpc" - tags = merge(local.base_tags, { Name = "${local.name}-nat-eip-${count.index}" }) -} - -resource "aws_nat_gateway" "this" { - count = var.single_nat_gateway ? 1 : length(var.public_subnet_cidrs) - allocation_id = aws_eip.nat[count.index].id - subnet_id = aws_subnet.public[count.index].id - - tags = merge(local.base_tags, { Name = "${local.name}-nat-${count.index}" }) - - depends_on = [aws_internet_gateway.this] -} - -resource "aws_route_table" "public" { - vpc_id = aws_vpc.this.id - - route { - cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.this.id - } - - tags = merge(local.base_tags, { Name = "${local.name}-rt-public" }) -} - -resource "aws_route_table_association" "public" { - count = length(aws_subnet.public) - subnet_id = aws_subnet.public[count.index].id - route_table_id = aws_route_table.public.id -} - -resource "aws_route_table" "private" { - count = length(aws_subnet.private) - vpc_id = aws_vpc.this.id - - route { - cidr_block = "0.0.0.0/0" - nat_gateway_id = var.single_nat_gateway ? aws_nat_gateway.this[0].id : aws_nat_gateway.this[count.index].id - } - - tags = merge(local.base_tags, { Name = "${local.name}-rt-private-${count.index}" }) -} - -resource "aws_route_table_association" "private" { - count = length(aws_subnet.private) - subnet_id = aws_subnet.private[count.index].id - route_table_id = aws_route_table.private[count.index].id -} - -resource "aws_security_group" "database" { - name = "${local.name}-db" - description = "PostgreSQL access from private subnets" - vpc_id = aws_vpc.this.id - - ingress { - description = "PostgreSQL from private subnets" - from_port = 5432 - to_port = 5432 - protocol = "tcp" - cidr_blocks = var.private_subnet_cidrs - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } - - tags = merge(local.base_tags, { Name = "${local.name}-db-sg" }) -} - -resource "aws_security_group" "cache" { - name = "${local.name}-cache" - description = "Redis access from private subnets" - vpc_id = aws_vpc.this.id - - ingress { - description = "Redis from private subnets" - from_port = 6379 - to_port = 6379 - protocol = "tcp" - cidr_blocks = var.private_subnet_cidrs - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - } - - tags = merge(local.base_tags, { Name = "${local.name}-cache-sg" }) -} diff --git a/devops/terraform/modules/network/outputs.tf b/devops/terraform/modules/network/outputs.tf deleted file mode 100644 index 44f6328..0000000 --- a/devops/terraform/modules/network/outputs.tf +++ /dev/null @@ -1,29 +0,0 @@ -output "vpc_id" { - description = "VPC id." - value = aws_vpc.this.id -} - -output "vpc_cidr" { - description = "VPC CIDR block." - value = aws_vpc.this.cidr_block -} - -output "public_subnet_ids" { - description = "Public subnet ids." - value = aws_subnet.public[*].id -} - -output "private_subnet_ids" { - description = "Private subnet ids." - value = aws_subnet.private[*].id -} - -output "db_security_group_id" { - description = "Security group id allowing PostgreSQL ingress from private subnets." - value = aws_security_group.database.id -} - -output "cache_security_group_id" { - description = "Security group id allowing Redis ingress from private subnets." - value = aws_security_group.cache.id -} diff --git a/devops/terraform/modules/network/variables.tf b/devops/terraform/modules/network/variables.tf deleted file mode 100644 index 2dec48f..0000000 --- a/devops/terraform/modules/network/variables.tf +++ /dev/null @@ -1,44 +0,0 @@ -variable "environment" { - description = "Environment name (e.g. staging, production). Drives resource naming." - type = string -} - -variable "vpc_cidr" { - description = "CIDR block for the VPC." - type = string - default = "10.0.0.0/16" -} - -variable "public_subnet_cidrs" { - description = "CIDR blocks for public subnets, one per AZ. Length must be 2." - type = list(string) - default = ["10.0.0.0/24", "10.0.1.0/24"] - - validation { - condition = length(var.public_subnet_cidrs) == 2 - error_message = "Exactly 2 public subnet CIDRs required (2 AZs)." - } -} - -variable "private_subnet_cidrs" { - description = "CIDR blocks for private subnets, one per AZ. Length must be 2." - type = list(string) - default = ["10.0.10.0/24", "10.0.11.0/24"] - - validation { - condition = length(var.private_subnet_cidrs) == 2 - error_message = "Exactly 2 private subnet CIDRs required (2 AZs)." - } -} - -variable "single_nat_gateway" { - description = "Use one shared NAT gateway instead of one per AZ. Cheaper, less HA." - type = bool - default = true -} - -variable "tags" { - description = "Common tags applied to all resources." - type = map(string) - default = {} -} diff --git a/devops/terraform/modules/secrets/main.tf b/devops/terraform/modules/secrets/main.tf deleted file mode 100644 index c0fb870..0000000 --- a/devops/terraform/modules/secrets/main.tf +++ /dev/null @@ -1,75 +0,0 @@ -terraform { - required_version = "~> 1.7" - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.0" - } - } -} - -locals { - prefix = "loopless/${var.environment}" - base_tags = merge( - var.tags, - { - Module = "secrets" - Environment = var.environment - }, - ) -} - -# AWS Secrets Manager — one secret per logical credential. Values are -# provisioned out-of-band (CI deploy step, sealed-secrets controller, or -# manual rotation via the AWS Console) so Terraform never holds plaintext. -resource "aws_secretsmanager_secret" "this" { - for_each = toset(var.secret_names) - - name = "${local.prefix}/${each.key}" - description = "Loopless ${var.environment} — ${each.key}" - kms_key_id = var.kms_key_id - recovery_window_in_days = var.recovery_window_days - - tags = merge(local.base_tags, { - SecretName = each.key - }) -} - -# IAM policy doc that grants read-only access to *this environment's* secrets. -# Attach to the EKS node IAM role (or use IRSA for per-pod least-privilege). -data "aws_iam_policy_document" "read_only" { - statement { - sid = "ReadEnvironmentSecrets" - effect = "Allow" - - actions = [ - "secretsmanager:GetSecretValue", - "secretsmanager:DescribeSecret", - ] - - resources = [for s in aws_secretsmanager_secret.this : s.arn] - } - - statement { - sid = "DenyCrossEnvironment" - effect = "Deny" - - actions = ["secretsmanager:*"] - - resources = ["arn:aws:secretsmanager:*:*:secret:loopless/*"] - - condition { - test = "StringNotLike" - variable = "secretsmanager:resource/AllowRotationLambdaArn" - values = ["arn:aws:secretsmanager:*:*:secret:${local.prefix}/*"] - } - } -} - -resource "aws_iam_policy" "read_only" { - name = "loopless-${var.environment}-secrets-read" - description = "Read-only access to loopless/${var.environment}/* in Secrets Manager" - policy = data.aws_iam_policy_document.read_only.json - - tags = local.base_tags -} diff --git a/devops/terraform/modules/secrets/outputs.tf b/devops/terraform/modules/secrets/outputs.tf deleted file mode 100644 index f26125a..0000000 --- a/devops/terraform/modules/secrets/outputs.tf +++ /dev/null @@ -1,14 +0,0 @@ -output "secret_arns" { - description = "Map of secret name → ARN. Use these as the ExternalSecrets remoteRef in K8s." - value = { for k, s in aws_secretsmanager_secret.this : k => s.arn } -} - -output "secret_names" { - description = "Map of secret name → full path (loopless//). Use these as the ExternalSecrets remoteRef.key." - value = { for k, s in aws_secretsmanager_secret.this : k => s.name } -} - -output "read_only_policy_arn" { - description = "ARN of the IAM policy that grants read-only access to this environment's secrets. Attach to node IAM role or IRSA service account." - value = aws_iam_policy.read_only.arn -} diff --git a/devops/terraform/modules/secrets/variables.tf b/devops/terraform/modules/secrets/variables.tf deleted file mode 100644 index af2b279..0000000 --- a/devops/terraform/modules/secrets/variables.tf +++ /dev/null @@ -1,40 +0,0 @@ -variable "environment" { - description = "Environment name (e.g. staging, production)" - type = string -} - -variable "secret_names" { - description = "List of secret names to provision in AWS Secrets Manager. Each name becomes loopless//." - type = list(string) - default = [ - "postgres-password", - "redis-auth-token", - "keycloak-admin-password", - "keycloak-client-secret", - "rabbitmq-password", - "openai-api-key", - "github-token", - "github-oauth-client-secret", - "smtp-password", - "discord-webhook-url", - "slack-webhook-url", - ] -} - -variable "recovery_window_days" { - description = "Number of days before a deleted secret is permanently purged (0 = immediate delete, allowed 7-30)." - type = number - default = 7 -} - -variable "kms_key_id" { - description = "Optional KMS key id/ARN for envelope encryption. Defaults to the AWS-managed Secrets Manager key." - type = string - default = null -} - -variable "tags" { - description = "Tags applied to every secret." - type = map(string) - default = {} -} diff --git a/devops/terraform/modules/storage/main.tf b/devops/terraform/modules/storage/main.tf deleted file mode 100644 index 99facd5..0000000 --- a/devops/terraform/modules/storage/main.tf +++ /dev/null @@ -1,67 +0,0 @@ -terraform { - required_version = "~> 1.7" - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.0" - } - } -} - -locals { - base_tags = merge(var.tags, { Name = var.bucket_name }) -} - -resource "aws_s3_bucket" "this" { - bucket = var.bucket_name - tags = local.base_tags -} - -resource "aws_s3_bucket_versioning" "this" { - bucket = aws_s3_bucket.this.id - - versioning_configuration { - status = "Enabled" - } -} - -resource "aws_s3_bucket_server_side_encryption_configuration" "this" { - bucket = aws_s3_bucket.this.id - - rule { - apply_server_side_encryption_by_default { - sse_algorithm = "AES256" - } - bucket_key_enabled = true - } -} - -resource "aws_s3_bucket_public_access_block" "this" { - bucket = aws_s3_bucket.this.id - - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true -} - -resource "aws_s3_bucket_lifecycle_configuration" "this" { - bucket = aws_s3_bucket.this.id - - rule { - id = "expire-noncurrent-versions" - status = "Enabled" - - filter {} - - noncurrent_version_expiration { - noncurrent_days = var.noncurrent_version_expiration_days - } - - abort_incomplete_multipart_upload { - days_after_initiation = 7 - } - } - - depends_on = [aws_s3_bucket_versioning.this] -} diff --git a/devops/terraform/modules/storage/outputs.tf b/devops/terraform/modules/storage/outputs.tf deleted file mode 100644 index 0f4a10f..0000000 --- a/devops/terraform/modules/storage/outputs.tf +++ /dev/null @@ -1,14 +0,0 @@ -output "bucket_id" { - description = "Bucket name (id)." - value = aws_s3_bucket.this.id -} - -output "bucket_arn" { - description = "Bucket ARN." - value = aws_s3_bucket.this.arn -} - -output "bucket_domain_name" { - description = "Bucket regional domain name." - value = aws_s3_bucket.this.bucket_regional_domain_name -} diff --git a/devops/terraform/modules/storage/variables.tf b/devops/terraform/modules/storage/variables.tf deleted file mode 100644 index 8369ff1..0000000 --- a/devops/terraform/modules/storage/variables.tf +++ /dev/null @@ -1,26 +0,0 @@ -variable "environment" { - description = "Environment name (e.g. staging, production)." - type = string -} - -variable "bucket_name" { - description = "Globally unique S3 bucket name. Caller is responsible for uniqueness." - type = string - - validation { - condition = length(var.bucket_name) >= 3 && length(var.bucket_name) <= 63 - error_message = "S3 bucket names must be 3-63 characters." - } -} - -variable "noncurrent_version_expiration_days" { - description = "Days after which noncurrent object versions expire." - type = number - default = 90 -} - -variable "tags" { - description = "Common tags." - type = map(string) - default = {} -} diff --git a/devops/terraform/sandbox/README.md b/devops/terraform/sandbox/README.md deleted file mode 100644 index e5fde7d..0000000 --- a/devops/terraform/sandbox/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# Terraform sandbox — DO-8 live-resource proof - -Minimal, **cheap**, standalone Terraform that provisions **one real AWS resource** (a -hardened S3 bucket) and applies it to a live environment. This is the concrete evidence -for **DO-8** ("Terraform config for at least one real cloud resource applied to a live -environment"). - -It is intentionally separate from the heavy `envs/production` stack (RDS + ElastiCache + -VPC), which is costly and not part of the live Hetzner deployment. This sandbox uses -**local state** so it runs in one shot with no backend bootstrap. - -## What it creates - -| Resource | Cost | -|---|---| -| `aws_s3_bucket` (empty, versioned, SSE-AES256, public access blocked) | ~$0 (storage billed per GB; nothing stored) | -| `random_id` (name suffix) | free | - -## Prerequisites - -- Terraform `>= 1.7` -- AWS credentials configured (`aws configure`, or `AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY` env vars) -- IAM permissions: `s3:CreateBucket`, `s3:PutBucket*`, `s3:DeleteBucket`, `s3:GetBucket*` - -## Apply - -```bash -cd devops/terraform/sandbox -terraform init -terraform plan # review: 4 resources to add -terraform apply # type `yes` -``` - -On success Terraform prints `bucket_name` and `bucket_arn`. Verify it is live: - -```bash -aws s3 ls | grep loopless-iac-demo -# or -aws s3api head-bucket --bucket "$(terraform output -raw bucket_name)" && echo "bucket exists" -``` - -## Tear down (when done — avoids any charge) - -```bash -terraform destroy # type `yes` -``` - -## Notes - -- State (`*.tfstate`) and `terraform.tfvars` are gitignored (`devops/terraform/.gitignore`). -- Override the bucket name prefix or region via `terraform.tfvars` (see `terraform.tfvars.example`). diff --git a/devops/terraform/sandbox/main.tf b/devops/terraform/sandbox/main.tf deleted file mode 100644 index 4649fec..0000000 --- a/devops/terraform/sandbox/main.tf +++ /dev/null @@ -1,51 +0,0 @@ -provider "aws" { - region = var.region -} - -# Random suffix so the (globally unique) bucket name never collides. -resource "random_id" "suffix" { - byte_length = 4 -} - -# DO-8 — one real cloud resource, applied to a live environment. -# An empty S3 bucket costs ~$0 (storage billed per GB; nothing is stored). -# force_destroy = true so `terraform destroy` cleans it up in one step. -resource "aws_s3_bucket" "demo" { - bucket = "${var.bucket_prefix}-${random_id.suffix.hex}" - force_destroy = true - - tags = { - Project = "loopless" - ManagedBy = "terraform" - Purpose = "iac-live-demo" - } -} - -# --- Hardening (DO-9 mindset): never ship an open bucket. --- - -# Block all public access. -resource "aws_s3_bucket_public_access_block" "demo" { - bucket = aws_s3_bucket.demo.id - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true -} - -# Keep object history. -resource "aws_s3_bucket_versioning" "demo" { - bucket = aws_s3_bucket.demo.id - versioning_configuration { - status = "Enabled" - } -} - -# Encrypt at rest. -resource "aws_s3_bucket_server_side_encryption_configuration" "demo" { - bucket = aws_s3_bucket.demo.id - rule { - apply_server_side_encryption_by_default { - sse_algorithm = "AES256" - } - } -} diff --git a/devops/terraform/sandbox/outputs.tf b/devops/terraform/sandbox/outputs.tf deleted file mode 100644 index b55ee70..0000000 --- a/devops/terraform/sandbox/outputs.tf +++ /dev/null @@ -1,14 +0,0 @@ -output "bucket_name" { - description = "Name of the created S3 bucket." - value = aws_s3_bucket.demo.bucket -} - -output "bucket_arn" { - description = "ARN of the created S3 bucket." - value = aws_s3_bucket.demo.arn -} - -output "region" { - description = "Region the bucket was created in." - value = var.region -} diff --git a/devops/terraform/sandbox/terraform.tfvars.example b/devops/terraform/sandbox/terraform.tfvars.example deleted file mode 100644 index 85e4c7e..0000000 --- a/devops/terraform/sandbox/terraform.tfvars.example +++ /dev/null @@ -1,3 +0,0 @@ -# Copy to terraform.tfvars (gitignored) and adjust if needed. -region = "eu-central-1" -bucket_prefix = "loopless-iac-demo" diff --git a/devops/terraform/sandbox/variables.tf b/devops/terraform/sandbox/variables.tf deleted file mode 100644 index 0429c7e..0000000 --- a/devops/terraform/sandbox/variables.tf +++ /dev/null @@ -1,11 +0,0 @@ -variable "region" { - description = "AWS region to create the bucket in." - type = string - default = "eu-central-1" -} - -variable "bucket_prefix" { - description = "Prefix for the bucket name. A random hex suffix is appended for global uniqueness." - type = string - default = "loopless-iac-demo" -} diff --git a/devops/terraform/sandbox/versions.tf b/devops/terraform/sandbox/versions.tf deleted file mode 100644 index f08f663..0000000 --- a/devops/terraform/sandbox/versions.tf +++ /dev/null @@ -1,19 +0,0 @@ -terraform { - required_version = ">= 1.7" - - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.6" - } - } - - # Local state on purpose: this sandbox is the minimal DO-8 "real resource applied - # to a live environment" proof. Local backend keeps it a one-shot `init && apply` - # with zero bootstrap (no S3 state bucket / DynamoDB lock to create first). - # State files are gitignored (see devops/terraform/.gitignore). -} diff --git a/devops/terraform/versions.tf b/devops/terraform/versions.tf deleted file mode 100644 index acff4b1..0000000 --- a/devops/terraform/versions.tf +++ /dev/null @@ -1,14 +0,0 @@ -terraform { - required_version = "~> 1.7" - - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.0" - } - random = { - source = "hashicorp/random" - version = "~> 3.6" - } - } -} diff --git a/docs/admin-isolation.md b/docs/admin-isolation.md new file mode 100644 index 0000000..a7281ba --- /dev/null +++ b/docs/admin-isolation.md @@ -0,0 +1,35 @@ +# Web / Admin Isolation Boundary + +The public site (`web.project-01.gjirafa.dev`) and the internal admin panel +(`admin.project-01.gjirafa.dev`) are served by **one Next.js image** and one +`loopless-frontend` Deployment. They are not separate pods. Separation is enforced at +four layers: + +1. **Edge middleware** (`frontend/client/middleware.ts`) — detects the admin host from the + `Host` / `x-forwarded-host` header (`lib/host.ts` `hostIsAdmin`). On the admin host only + `/admin`, `/login`, `/auth` render; everything else is redirected to `/admin`. +2. **Client route guards** — `AuthGuard` (token presence) wraps `app/(protected)/`; + `AdminGuard` (role check) wraps the admin layout. +3. **API-side RBAC (authoritative)** — every admin endpoint is protected by the Keycloak + `AdminPolicy`. The frontend guards are UX only; authorization is enforced at the API. +4. **Browser origin isolation (production)** — `web.*` and `admin.*` are distinct origins, + so `localStorage` (auth tokens, Zustand persist) is isolated by the browser. There is no + cross-tab state leak between web and admin in production. + +## Dev-only hardening + +In local dev both hosts resolve to `localhost` = the **same origin**, so localStorage is +shared. To prevent cross-contamination there, the persisted keys are host-scoped: + +- `stores/authStore.ts` → `loopless.auth` vs `loopless.auth.admin` +- `lib/auth/token.ts` → `loopless.access_token` vs `loopless.access_token.admin` + +(keyed by `isAdminHost()`). In production this is redundant with origin isolation but makes +the boundary explicit and robust. + +## Why not a separate admin pod? + +A separate admin app/image/Deployment would add a second build, CI/CD path, and shared- +component extraction for marginal benefit: production is already origin-isolated and the API +is the authoritative authorization layer. Revisit only if admin needs independent scaling, +a different security context, or divergent code from `web`. diff --git a/docs/ai-log.md b/docs/ai-log.md index ad4f007..ca40dae 100644 --- a/docs/ai-log.md +++ b/docs/ai-log.md @@ -852,6 +852,8 @@ _Sprint 5 · May 18 – May 29, 2026_ - **Lessons Learned:** The `~> 1.7` constraint on `required_version` correctly accepts the dev box's Terraform 1.14.4 because the pessimistic operator only locks the major version when written with two segments — common point of confusion worth noting. pgvector on RDS is a two-step deal: Terraform preloads the library via `shared_preload_libraries = "vector"` on the parameter group, but `CREATE EXTENSION vector;` still has to run from app migrations after provisioning — left a comment in `modules/database/main.tf` so the next maintainer doesn't assume Terraform handles it end-to-end. Backend config kept env-specific (`backend-staging.hcl` / `backend-production.hcl`) and passed via `-backend-config` rather than hardcoded, so the same module tree serves both envs. Bootstrap of the tfstate bucket + DynamoDB lock table is documented as a one-time manual step in the README — chicken-and-egg with the state itself, not worth a separate Terraform stack at this scale. Staging vs production diff lives entirely in the env `main.tf` files (instance class, multi-AZ, backup retention, failover) — modules themselves stay environment-agnostic. +**Update (2026-06-08):** Per mentor guidance, Terraform implementation has been superseded in favor of Helm-based infrastructure (PostgreSQL, Redis, MinIO all deployed in-cluster via Helm charts). This simplifies stack and removes AWS dependency. Original Terraform code is retained in git history as reference. + --- ### [CP-122] Prometheus Scrape Config + Grafana Dashboards @@ -1313,3 +1315,42 @@ OKR KR3.1 target (≥ 70% A/B) exceeded by large margin. ### Process Change for Next Project Use Claude Code's plan mode at the start of every sprint to generate a dependency-ordered task graph before touching code. The `missing-tasks.md` format (dependency graph + effort estimate + owner + status) created in Sprint 3 eliminated sprint-end scrambles. Adopt this format from sprint 1 next time. + +--- + +## Production Hardening Session — Three-Subdomain Audit (2026-06-10) + +### 2026-06-10 — Backend Swagger/OpenAPI production gating +* **Tool Used**: Claude Code (CLI via workspace execution) +* **Prompt/Input**: Gate Swagger generator + UI + `/swagger`/`/api-docs` routes behind `IsDevelopment()` so they are completely un-routed in Production; make the dev-exception-page boundary explicit. +* **Output Quality**: Required manual edits — gated both the DI registration (`AddSwaggerGen`/`AddEndpointsApiExplorer`) and the middleware (`UseSwagger`/`UseSwaggerUI`). Kept `UseExceptionHandler` always-on (the structured `ValidationExceptionHandler` would be bypassed if swapped for the developer page in dev); dev pages were already absent from prod, so the requirement was met without that swap. +* **Time Saved Estimate**: ~30 min vs manually locating the always-on Swagger block and reasoning about the exception-handler interaction. +* **Lessons Learned**: The naive "use developer page in dev, exception handler in prod" pattern breaks custom `IExceptionHandler` registrations in dev. Gate Swagger, not the exception handler. + +### 2026-06-10 — Frontend runtime env injection (window.__ENV) +* **Tool Used**: Claude Code (CLI via workspace execution) +* **Prompt/Input**: Replace build-time `NEXT_PUBLIC_*` baking with runtime injection so one image runs in any environment. New `lib/clientEnv.ts` accessor, `public/__env.js` written by the container entrypoint, loaded `beforeInteractive` in `app/layout.tsx`; refactor all 10 `process.env.NEXT_PUBLIC_*` call sites (8 files). +* **Output Quality**: Required manual edits — `tsc --noEmit` + ESLint clean on all touched files. Left `lib/host.ts` `ADMIN_HOST` build-time because it is read by the Edge middleware, which cannot see `window.__ENV` or runtime `APP_*`. +* **Time Saved Estimate**: ~2 hr vs manually designing the SSR/CSR/Edge resolution matrix and refactoring every call site. +* **Lessons Learned**: `process.env.NEXT_PUBLIC_X` is statically inlined; a dynamic `process.env[key]` lookup is NOT, which is the trick for reading live values server-side. Edge runtime can't consume runtime injection — keep middleware-facing constants build-time. + +### 2026-06-10 — Dev web/admin shared-state isolation +* **Tool Used**: Claude Code (CLI via workspace execution) +* **Prompt/Input**: Host-scope the persisted auth keys (Zustand `persist` name + `loopless.access_token`) so a localhost dev session can't cross-contaminate web↔admin; document the boundary. +* **Output Quality**: Success out-of-the-box — reused existing `isAdminHost()` from `lib/host.ts`; added `docs/admin-isolation.md`. +* **Time Saved Estimate**: ~20 min. +* **Lessons Learned**: Production already isolates via distinct subdomain origins; the only real gap is dev (shared localhost origin). Scope the fix to that. + +### 2026-06-10 — DevOps: standalone image + runtime config + Trivy gate +* **Tool Used**: Claude Code (CLI via workspace execution) +* **Prompt/Input**: `next.config` `output: "standalone"`; rework the frontend Dockerfile runner to standalone + a non-root inline entrypoint that writes `public/__env.js` from `APP_*`; drop `NEXT_PUBLIC_*` build-args; tighten Trivy image gate to `CRITICAL,HIGH`; add `APP_*` to the Helm frontend ConfigMap. +* **Output Quality**: Required manual edits + iterated — the frontend build context is `frontend/client`, so the entrypoint can't live under `devops/`; generated it inline via a Dockerfile heredoc instead. Did NOT delete `k8s/base/ingress.yaml`: the Kustomize overlays patch `loopless-ingress`, so deleting the base resource would break `kustomize build` (flagged for a follow-up that also strips the overlay patches). +* **Time Saved Estimate**: ~1.5 hr vs hand-writing the standalone runner + runtime-config entrypoint and tracing the Kustomize patch dependency. +* **Lessons Learned**: Build context dictates where COPY-able files can live; heredoc-generated entrypoints sidestep context limits. Always check overlay/patch targets before deleting a base manifest. + +### 2026-06-10 — WIP build/test fixes (summary feature + migrations) +* **Tool Used**: Claude Code (CLI via workspace execution) +* **Prompt/Input**: Diagnose and fix in-progress build/test failures: (1) `ExecuteDeleteAsync` ambiguity in DeleteUserCommandHandler, (2) stale `GitHubSyncJob` enqueue vs the new `SummaryGenerationJob.GenerateAsync` signature, (3) integration tests failing with `column "GitHubWebhookSecret" does not exist`. +* **Output Quality**: Required manual edits + iterated. (1) Qualified the call via the provider-agnostic `EntityFrameworkQueryableExtensions` (both core + relational overloads were in scope via Pgvector). (2) Created the ProjectSummary row + enqueued with the 4-arg signature, matching GenerateProjectSummaryCommandHandler. (3) Root cause: 3 hand-authored WIP migrations lacked `.Designer.cs` companions, so EF never discovered them and the columns were never created — added `[DbContext]`/`[Migration("id")]` attributes directly to each migration class (the file-preserving fix; full regeneration would have destroyed the custom raw-SQL pgvector HNSW index migration). Also updated the standup-analytics unit test for the new ICurrentUser dependency from the IDOR fix and modeled ProjectInvitation in its in-memory test context. +* **Time Saved Estimate**: ~2 hr vs manually tracing EF migration discovery, the bulk-delete extension ambiguity, and the multi-project build chain. +* **Lessons Learned**: EF discovers migrations by the `[Migration]` attribute that lives in the Designer partial — hand-authored migration `.cs` files without a Designer are silently skipped by `MigrateAsync`. For migrations that wrap custom SQL (HNSW `CREATE INDEX CONCURRENTLY`), regeneration isn't an option, so adding the attributes directly is the correct minimal fix. EF Core 10 surfaces `ExecuteDeleteAsync` on both core and relational extension classes — disambiguate when both are referenced. diff --git a/docs/api-endpoints.md b/docs/api-endpoints.md index 69150c9..4890b35 100644 --- a/docs/api-endpoints.md +++ b/docs/api-endpoints.md @@ -15,16 +15,52 @@ Base URL: `/api/v1/` | Auth: JWT via Keycloak ## Projects -| Method | Endpoint | Auth | -| -------------- | ---------------------------------- | --------------------- | -| GET/POST | /api/v1/projects | Yes / Freelancer | -| GET/PUT/DELETE | /api/v1/projects/{id} | Owner/Invited | -| GET | /api/v1/projects/{id}/commits | Owner/Invited | -| POST | /api/v1/projects/{id}/commits/sync | Owner | -| POST/GET | /api/v1/projects/{id}/files | Owner / Owner+Invited | -| POST | /api/v1/projects/{id}/summary | Owner/Invited | -| POST | /api/v1/projects/{id}/invitations | Owner | -| PUT | /api/v1/invitations/{id}/accept | Invited Enterprise | +| Method | Endpoint | Auth | Notes | +| -------------- | ----------------------------------------- | --------------------- | ------------------------------------ | +| GET/POST | /api/v1/projects | Yes / Freelancer | | +| GET/PUT/DELETE | /api/v1/projects/{id} | Owner/Invited | | +| GET | /api/v1/projects/{id}/commits | Owner/Invited | | +| POST | /api/v1/projects/{id}/commits/sync | Owner | | +| POST/GET | /api/v1/projects/{id}/files | Owner / Owner+Invited | | +| POST | /api/v1/projects/{id}/summary | Owner/Invited | Returns 202; see async summary below | +| GET | /api/v1/projects/{id}/summary/status | Owner/Invited | Polls async summary job status | +| POST | /api/v1/projects/{id}/invitations | Owner | | +| PUT | /api/v1/invitations/{id}/accept | Invited Enterprise | | + +### Async Summary Generation + +`POST /api/v1/projects/{id}/summary` enqueues a background RAG job (Hangfire) and returns immediately: + +```json +HTTP 202 Accepted +{ + "jobId": "", + "status": "queued" +} +``` + +Poll for completion with `GET /api/v1/projects/{id}/summary/status`: + +```json +// In progress +{ "jobId": "...", "status": "running" } + +// Completed +{ + "jobId": "...", + "status": "completed", + "summary": { + "id": "...", + "projectId": "...", + "content": "...", + "modelUsed": "gpt-4o-mini", + "generatedAt": "2026-06-10T12:00:00Z" + } +} + +// Failed (after 3 retries with exponential backoff: 30 s, 120 s, 300 s) +{ "jobId": "...", "status": "failed", "failureReason": "..." } +``` ## Standups diff --git a/docs/architecture.md b/docs/architecture.md index 9158eb6..784550b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -158,3 +158,47 @@ Browser | Availability | 99.5% | Uptime Kuma monitors, K8s HPA (min 2 replicas) | | Observability | Full trace | Correlation ID on every request, Serilog → Loki | | Auth brute force | Max 10 req/min on /auth | Stricter rate limit policy on auth endpoints | + +--- + +## Integrations + +### GitHub Commit Sync + +Loopless keeps project commit history in sync with GitHub via two complementary mechanisms. + +#### Webhook (primary) + +When a project has a `github_webhook_secret` configured, GitHub delivers `push` events to: + +``` +POST /api/v1/projects/{id}/github/webhook +``` + +The endpoint: +1. Reads the raw request body and validates the `X-Hub-Signature-256` HMAC-SHA256 header against the per-project `github_webhook_secret`. +2. Returns `401 Unauthorized` on signature mismatch — no detail leaked. +3. Acknowledges non-`push` event types with `200 OK` (no-op). +4. On a valid `push` event, immediately enqueues a `SyncGithubCommitsJob` via Hangfire — commits appear within seconds of the push. + +The endpoint is unauthenticated (no JWT required); the HMAC signature is the sole authentication mechanism. The per-project secret is stored in the `github_webhook_secret` column (VARCHAR 256, nullable) added by the `AddGithubWebhookSecret` migration. + +#### Fallback polling + +A Hangfire recurring job (`github-commit-sync`, `GitHubSyncJob.SyncAllAsync`) polls all projects with a configured `GitHubRepoUrl` every **6 hours** as a catch-all for missed webhooks (webhook not yet configured, transient GitHub delivery failure). On GitHub rate-limit responses (429 or 403 with `X-RateLimit-Remaining: 0`), the job reschedules itself 30 minutes into the future. + +#### Flow summary + +``` +GitHub push + → POST /api/v1/projects/{id}/github/webhook + → HMAC-SHA256 validate (X-Hub-Signature-256 vs project.GitHubWebhookSecret) + → Hangfire enqueue: SyncGithubCommitsJob (immediate) + → GitHubService.FetchCommitsAsync → store new commits + → enqueue SummaryGenerationJob + +Fallback (every 6 h): + GitHubSyncJob.SyncAllAsync + → SyncProjectAsync per project + → same fetch + store + summary pipeline +``` diff --git a/docs/database-schema.md b/docs/database-schema.md index 30093e0..8ffbfd5 100644 --- a/docs/database-schema.md +++ b/docs/database-schema.md @@ -21,7 +21,7 @@ PostgreSQL 16 + `pgvector` extension. Migrations are EF Core code-first (see `ba | `project_task_audit_logs` | Audit trail per task lifecycle change | `task_id FK`, `action`, `actor_id`, `from_status`, `to_status` | | `github_commits` | Cached commits pulled by Hangfire from the linked GitHub repo | `project_id FK`, `sha UK`, `message`, `author_name`, `committed_at` | | `project_summaries` | Versioned AI-generated project summaries (RAG output) | `project_id FK`, `content`, `model_used`, `generated_at` | -| `standups` | Daily async check-ins (3 questions + blocker sentiment) | `project_id FK`, `freelancer_id FK`, `blocker_sentiment` | +| `standups` | Daily async check-ins (3 questions + blocker sentiment) | `project_id FK`, `freelancer_id FK`, `blocker_sentiment` (async — may be `null` for recently submitted standups) | | `conversations` | Direct-message thread between two users | `participant_one FK`, `participant_two FK`, `last_message_at` | | `messages` | Individual message in a conversation | `conversation_id FK`, `sender_id FK`, `content`, `file_url`, `email_notification_sent_at` | | `notifications` | Platform-pushed in-app notifications | `user_id FK`, `title`, `body`, `action_url`, `is_read` | @@ -42,7 +42,7 @@ All tables include `id UUID PK`, `created_at`, `updated_at` (nullable on UPDATE) | `projects` | `freelancer_id`, `status` | btree | Owner-scoped queries + status filters | | `project_invitations` | `(project_id, enterprise_id)`, `(project_id, freelancer_id)`, `freelancer_id`, `status` | btree | Both sides of invitation lookup + queue filter | | `freelancer_profiles` | `proficiencies` | GIN | JSONB skill containment queries | -| `embeddings` | `embedding vector_cosine_ops` | **IVFFlat** (lists=100) | ANN semantic search via `<=>` operator | +| `embeddings` | `"Vector" vector_cosine_ops` | **HNSW** (m=16, ef_construction=64) | ANN semantic search via `<=>` operator. HNSW chosen over IVFFlat for better recall-to-latency at scale with no manual index rebuilds. `m=16` controls the number of bi-directional links per node (higher = better recall, more memory); `ef_construction=64` controls build-time search width (higher = better quality graph, slower build). | | `standups` | `(project_id, created_at)` | btree composite | Timeline pagination | | `messages` | `(conversation_id, created_at)` | btree composite | Chat history pagination | | `notifications` | `(user_id, created_at DESC)`, `(user_id, is_read)` | btree composite | Unread badge + feed | @@ -314,7 +314,7 @@ A relation is in **3NF** when every non-key attribute is: | `project_task_audit_logs` | ✅ | ✅ | ✅ | Append-only | | `github_commits` | ✅ | ✅ | ✅ | Cache table; `sha` unique per repo lifetime | | `project_summaries` | ✅ | ✅ | ✅ | Versioned; older rows are never updated | -| `standups` | ✅ | ✅ | ✅ | `blocker_sentiment` computed by `NlpService`, stored to avoid recomputing in admin analytics | +| `standups` | ✅ | ✅ | ✅ | `blocker_sentiment` populated asynchronously by `AnalyzeBlockerSentimentConsumer` after standup write; `null` until the consumer processes the `standups.submitted` event | | `conversations` | ✅ | ✅ | ✅ | `last_message_at` is **denormalized for conversation-list query performance** — documented exception, kept consistent via a domain event on `Message` insert | | `messages` | ✅ | ✅ | ✅ | `email_notification_sent_at` is denormalized to avoid scanning a separate dispatch log — same documented-exception pattern | | `notifications` | ✅ | ✅ | ✅ | — | diff --git a/docs/release-tags.md b/docs/release-tags.md index 98410df..31625ee 100644 --- a/docs/release-tags.md +++ b/docs/release-tags.md @@ -58,16 +58,14 @@ Tag will point at the commit that contains **all** of: - Full Docker Compose stack (core + observability + automation profiles) including the **MinIO + Mailpit** additions for local S3 + SMTP - Production multi-stage Dockerfiles with non-root + Alpine base + `.dockerignore` - CI / CD pipelines: lint → test → secret-scan (Gitleaks + TruffleHog) → Lighthouse CI → Trivy → build → push GHCR → deploy → notify -- Kubernetes manifests + Kustomize overlays + Helm chart + HPAs (CPU 70%) +- Kubernetes manifests + Kustomize overlays + Helm chart (all infrastructure: PostgreSQL, Redis, MinIO) + HPAs (CPU 70%) - Prometheus + Grafana (4 dashboards, 8 alert rules) + Loki + ELK + Alertmanager + Uptime Kuma -- Terraform AWS modules (cache, database, network, storage) — and **applied to a live AWS resource** per DO-8 - Nginx reverse proxy + Let's Encrypt automation -- Ubuntu bootstrap script: users, SSH hardening, UFW, fail2ban, systemd unit, **cron** (4 jobs: nginx reload, daily Postgres backup, hourly health probe, weekly Docker prune) - AIOps pipeline — Prometheus → Alertmanager → `aiops-triage` (Node.js) → GitHub Models gpt-4o-mini → Discord + Slack webhooks. **Real firing alert captured** in `docs/aiops-demo-evidence.md` - Conventional Commits + release-please semantic versioning + Dependabot (8 ecosystems) - Security: Trivy image scan + Gitleaks + TruffleHog in CI -**Tag command (run once Terraform has been applied to a live AWS resource):** +**Tag command:** ```bash # Ensure working tree is clean + on main diff --git a/docs/security-audit.md b/docs/security-audit.md index 021b6ba..5531c53 100644 --- a/docs/security-audit.md +++ b/docs/security-audit.md @@ -26,6 +26,23 @@ trufflehog filesystem . --only-verified **Result: No verified secrets in git history.** +### AIOps Payload Sanitization + +Alertmanager webhook payloads routed through the `aiops-triage` service can contain database connection strings, JWT tokens, email addresses, and other PII embedded in Prometheus label values or alert annotations. A dedicated sanitization layer (`devops/aiops/sanitize.js`) runs **before** any alert data is forwarded to the OpenAI API. + +Patterns redacted: + +| Pattern | Replacement | +|---------|-------------| +| `postgres://…`, `redis://…`, `amqp://…` connection strings | `[REDACTED_*_URI]` | +| `Bearer ` header values | `[REDACTED_BEARER]` | +| Three-segment JWTs (`ey….ey….…`) | `[REDACTED_JWT]` | +| Email addresses | `[REDACTED_EMAIL]` | +| Non-RFC-1918 public IPv4 addresses | `[REDACTED_PUBLIC_IP]` | +| Label values matching `password=`, `secret=`, `key=`, `token=` | `[REDACTED_SECRET]` | + +Only the alert name, severity label, and sanitized summary/description annotation are forwarded to OpenAI. All other labels and annotations (service, job, runbook URLs, etc.) are excluded from the prompt. See Section 9 for full AIOps data-handling policy. + --- ## 2. Dependency Scan — Trivy @@ -107,17 +124,17 @@ npx @lhci/cli autorun --config=scripts/lighthouse.config.json ## 5. Rate Limiting -Redis-backed fixed-window rate limiting via custom `RedisFixedWindowRateLimiter` (StackExchange.Redis Lua INCR+EXPIRE). Falls back to in-memory when Redis is unavailable (fail-open). +Redis-backed fixed-window rate limiting via custom `RedisFixedWindowRateLimiter` (StackExchange.Redis Lua INCR+EXPIRE). Each policy configures an independent Redis-failure strategy — see **Redis failure behavior** below. ### Policies -| Policy | Limit | Window | Partition | Applied To | -| ------------- | ------ | ------ | ------------------- | ------------------------------------------------------ | -| `global` | 60 req | 1 min | Client IP | All endpoints (chained) | -| `auth` | 10 req | 1 min | Client IP | Auth/token endpoints | -| `ai` | 5 req | 1 min | User ID (JWT `sub`) | `POST /api/v1/projects/{id}/summary` | -| `matching` | 30 req | 1 min | User ID (JWT `sub`) | `/api/v1/matching/discover`, `/api/v1/matching/search` | -| `public-read` | 30 req | 1 min | Client IP | `GET /api/v1/projects/{id}` | +| Policy | Limit | Window | Partition | Applied To | Redis Failure | +| ------------- | ------ | ------ | ------------------- | ------------------------------------------------------ | -------------- | +| `global` | 60 req | 1 min | Client IP | All endpoints (chained) | fail-open | +| `auth` | 10 req | 1 min | Client IP | Auth/token endpoints | **fail-closed** | +| `ai` | 5 req | 1 min | User ID (JWT `sub`) | `POST /api/v1/projects/{id}/summary` | **fail-closed** | +| `matching` | 30 req | 1 min | User ID (JWT `sub`) | `/api/v1/matching/discover`, `/api/v1/matching/search` | fail-open | +| `public-read` | 30 req | 1 min | Client IP | `GET /api/v1/projects/{id}` | fail-open | ### Distribution @@ -125,11 +142,30 @@ Redis key format: `ratelimit:{policy}:{partition}`. TTL = window duration. All A Response on limit exceeded: **HTTP 429 Too Many Requests** with `Retry-After` header. +### Redis failure behavior + +In a multi-pod Kubernetes deployment, per-pod in-memory counters are desynchronized — an undetected Redis outage effectively removes rate limit enforcement across the cluster. To mitigate this for high-risk endpoints: + +- **`auth` and `ai` policies — fail-closed:** When Redis is unreachable the limiter returns **HTTP 503 Service Unavailable** instead of granting the request. This trades availability for security at endpoints where unenforced abuse (credential stuffing, OpenAI cost exhaustion) is highest risk. +- **`global`, `matching`, and `public-read` policies — fail-open:** Request is granted during Redis outages. These policies protect lower-severity surfaces where blocking all traffic would degrade user experience without proportional security benefit. + +When Redis is not configured at all (single-replica dev setup), all policies fall back to in-memory counters regardless of the fail-closed setting. + +### Observability + +A Prometheus counter tracks every Redis fallback event: + +``` +ratelimiter_redis_fallback_total{policy="auth|ai|matching|..."} +``` + +Alert on sustained non-zero rate of this counter to detect Redis degradation before it impacts the security posture of fail-closed policies. The metric is scraped by the existing Prometheus job targeting the backend (`/metrics`) and available in Grafana dashboards. + ### Bypass resistance - Partition is extracted from JWT `sub` for user-scoped policies — IP rotation does not help. - Global IP policy catches unauthenticated probing. -- Fail-open on Redis outage (in-memory fallback per pod) to avoid availability impact. +- Auth and AI endpoints fail-closed during Redis outages — desynchronized pod counters cannot be exploited during cache degradation. --- @@ -259,3 +295,69 @@ Workflow `.github/workflows/security-scan.yml` runs Mondays 03:00 UTC + manual d | A08 | Software & data integrity | ✅ | release-please semver; signed CI artifacts via OIDC GHCR push; Helm chart pinned by digest | | A09 | Logging & monitoring failures | ✅ | Serilog → Loki + ELK; correlation IDs; Prometheus + AlertManager + AIOps triage | | A10 | SSRF | ✅ | Outbound HTTP only to allow-listed hosts (OpenAI, GitHub); no user-supplied URL fetches | + +--- + +## 9. AIOps Data Handling + +### Data flow + +``` +Prometheus → Alertmanager → POST /webhook/alerts (aiops-triage) + │ + sanitize.js redaction layer + │ + OpenAI chat completions API + │ + Slack / Discord webhook +``` + +### What enters the aiops-triage service + +Raw Alertmanager webhook payloads (JSON) containing the full alert struct: +- All Prometheus labels (can include `service`, `job`, `namespace`, `pod`, arbitrary user-defined labels) +- All annotations (can include `summary`, `description`, `runbook_url`, and any custom annotation set by alerting rules) +- Alert metadata (status, startsAt, endsAt, generatorURL) + +These payloads **can** contain sensitive data: connection strings embedded in stack traces, JWT tokens in log annotations, email addresses in `owner` labels, and public IP addresses. + +### Sanitization layer (`devops/aiops/sanitize.js`) + +Before any alert data leaves the service boundary toward OpenAI, `buildSanitizedAlertLines()` performs: + +1. **Connection string redaction** — `postgres://`, `redis://`, `amqp://` URIs replaced with typed placeholders. +2. **Token redaction** — `Bearer ` header form and raw three-segment JWTs (`ey….ey….…`) are replaced. +3. **Email redaction** — RFC-5321 address patterns replaced. +4. **Public IP redaction** — IPv4 addresses outside RFC-1918 ranges (10/8, 172.16-31/12, 192.168/16) and loopback (127/8) replaced. Private/internal IPs are preserved for operational context. +5. **Secret label redaction** — Values following `password=`, `passwd=`, `secret=`, `api_key=`, `apikey=`, `token=` replaced. + +### What is sent to OpenAI + +Only the following fields are included in the prompt, after sanitization: + +| Field | Source | Sanitized | +|-------|--------|-----------| +| Alert index (1, 2, …) | derived | — | +| Severity | `labels.severity` | no (enum value) | +| Alert name | `labels.alertname` | no (metric name) | +| Summary | `annotations.summary` | **yes** | +| Description (fallback) | `annotations.description` | **yes** | + +All other labels, annotations, runbook URLs, generator URLs, and timing metadata are excluded from the OpenAI prompt. + +### Slack / Discord notifications + +The human-readable alert lines sent to Slack/Discord use the original `buildAlertLines()` function which includes service/job labels and runbook URLs — these go to internal operator channels only, not to third-party AI providers. + +### Test coverage + +`devops/aiops/sanitize.test.js` covers all redaction rules with positive (must redact) and negative (must preserve private IPs and clean text) assertions. Run with: + +```bash +cd devops/aiops && node --test sanitize.test.js +``` + +### Residual risk + +- Alerting rules that embed raw stack traces in `annotations.description` may contain fragments not caught by the regex patterns (e.g., base64-encoded secrets). Teams should avoid putting stack traces directly in Alertmanager annotations. +- The sanitization layer applies only to data forwarded to OpenAI. Internal Loki logs and the Slack/Discord payloads are not redacted — access to those channels is already restricted to operators. diff --git a/docs/tasks.md b/docs/tasks.md index a425d95..4337553 100644 --- a/docs/tasks.md +++ b/docs/tasks.md @@ -1505,33 +1505,28 @@ _May 18 – May 29, 2026 · Milestone M5_ --- -#### CP-121 — Terraform VPC + RDS + Redis Modules +#### CP-121 — Infrastructure Provisioning (Helm-based) -- **Status:** `[x]` Done +- **Status:** `[x]` Done (superseded — Terraform replaced with Helm) - **Sprint:** 5 · **Milestone:** M5 - **User Story:** US-8.4 - **Dependencies:** None -**Files to create:** +**Implementation:** Per mentor guidance, infrastructure now deployed entirely via Helm instead of Terraform: -- `devops/terraform/modules/network/main.tf` — VPC, subnets, security groups -- `devops/terraform/modules/database/main.tf` — Managed PostgreSQL (RDS or equivalent) -- `devops/terraform/modules/cache/main.tf` — Managed Redis (ElastiCache or equivalent) -- `devops/terraform/modules/storage/main.tf` — S3-compatible object storage -- `devops/terraform/envs/staging/main.tf` — staging environment composition -- `devops/terraform/envs/production/main.tf` — production environment composition -- `devops/terraform/backend.tf` — remote state (S3 + DynamoDB lock) +- `devops/helm/loopless/templates/postgres.yaml` — PostgreSQL 16 StatefulSet with pgvector, persistent storage +- `devops/helm/loopless/templates/redis.yaml` — Redis 7 Deployment (cache + SignalR backplane) +- `devops/helm/loopless/templates/minio.yaml` — MinIO Deployment with persistent volume (S3-compatible uploads) **Acceptance criteria:** -1. Network module creates VPC with public/private subnets across 2 AZs -2. Database module provisions PostgreSQL 16 with pgvector extension, parameter group, automated backups -3. Cache module provisions Redis 7 with encryption at rest -4. Storage module creates S3 bucket for file uploads with versioning enabled -5. Modules are parameterized: `environment` variable controls naming and sizing -6. Staging uses smaller instance sizes than production -7. `terraform plan` succeeds for both staging and production -8. State stored in remote backend +1. PostgreSQL StatefulSet includes pgvector preload, persistent storage, init scripts for Keycloak DB +2. Redis Deployment runs with no persistence (cache-only, SignalR backplane) +3. MinIO Deployment exposes API (:9000) and console (:9001) with PVC storage +4. All components configured with `storageClassName` for cluster's storage backend +5. Helm values parameterize: image versions, storage sizes, resource limits per environment +6. `helm install loopless devops/helm/loopless` deploys all infra to `project-01` namespace +7. Database, cache, and storage are accessible by backend pods via service DNS --- diff --git a/docs/user-stories.md b/docs/user-stories.md index ac35ad1..c2e6b31 100644 --- a/docs/user-stories.md +++ b/docs/user-stories.md @@ -668,10 +668,10 @@ The score is not static. Re-run RICE on any story when: **So that** environments are reproducible and version-controlled. **Acceptance Criteria:** -- [ ] Terraform modules for: VPC, managed PostgreSQL, Redis, object storage, DNS -- [ ] Modules are parameterized for dev, staging, and prod environments -- [ ] State stored in remote backend (e.g., S3 + DynamoDB lock) -- [ ] `terraform plan` output reviewed in PR before `apply` +- [x] Helm charts provisioning: PostgreSQL, Redis, MinIO (all in-cluster) +- [x] Charts parameterized via `values.yaml` for staging and production environments +- [x] Persistent storage configured with `storageClassName` +- [x] All services accessible by backend pods via Kubernetes service DNS **Sprint:** 5 · **Status:** Planned · **Tasks:** CP-121 diff --git a/frontend/client/app/(protected)/admin/analytics/page.tsx b/frontend/client/app/(protected)/admin/analytics/page.tsx index ee5566c..0f0ddcf 100644 --- a/frontend/client/app/(protected)/admin/analytics/page.tsx +++ b/frontend/client/app/(protected)/admin/analytics/page.tsx @@ -29,12 +29,13 @@ const TrendChart = dynamic( }, ); import { hasRole } from "@/lib/auth/token"; +import { rbacEnabled } from "@/lib/clientEnv"; import type { MetricSummary } from "@/types/analytics"; const RANGE_OPTIONS = [30, 60, 90] as const; type RangeOption = (typeof RANGE_OPTIONS)[number]; -const RBAC_ENABLED = process.env.NEXT_PUBLIC_ENABLE_RBAC === "true"; +const RBAC_ENABLED = rbacEnabled(); function trendProps(metric: MetricSummary) { return { diff --git a/frontend/client/app/(protected)/projects/[id]/page.tsx b/frontend/client/app/(protected)/projects/[id]/page.tsx index 3d0c178..86b6447 100644 --- a/frontend/client/app/(protected)/projects/[id]/page.tsx +++ b/frontend/client/app/(protected)/projects/[id]/page.tsx @@ -350,8 +350,8 @@ export default function ProjectDashboardPage({ transition={{ duration: 0.3, ease: "easeOut" }} className="mt-4" > -
-
+
+

{project.title}

diff --git a/frontend/client/app/layout.tsx b/frontend/client/app/layout.tsx index 91e5895..d30144e 100644 --- a/frontend/client/app/layout.tsx +++ b/frontend/client/app/layout.tsx @@ -1,4 +1,5 @@ import type { Metadata, Viewport } from "next"; +import Script from "next/script"; import { Inter, Space_Grotesk, Instrument_Serif, JetBrains_Mono } from "next/font/google"; import { MotionProvider } from "@/components/providers/MotionProvider"; import { QueryProvider } from "@/components/providers/QueryProvider"; @@ -55,6 +56,9 @@ export default function RootLayout({ children }: { children: React.ReactNode }) className={`${inter.variable} ${spaceGrotesk.variable} ${instrumentSerif.variable} ${jetbrainsMono.variable}`} > + {/* Runtime public config (window.__ENV), written at container start by the Docker + entrypoint so one immutable image runs in any environment. Loaded before app JS. */} +