diff --git a/.env.dev.example b/.env.dev.example index eb58478cd..1ba97c1b1 100644 --- a/.env.dev.example +++ b/.env.dev.example @@ -75,3 +75,6 @@ DATABASE_PASSWORD=postgres-password REDIS_HOST=redis REDIS_PORT=6379 REDIS_PASSWORD= + +# Number of worker processes for Log Stream deliveries (optional, default 2). +#LOG_STREAM_WORKERS=2 diff --git a/.env.example b/.env.example index 3d1936282..a86c71918 100644 --- a/.env.example +++ b/.env.example @@ -68,3 +68,8 @@ DATABASE_PASSWORD=a765b221799be364c53c8a32acccf5dd90d5fc832607bdd14fccaaaa0062ad REDIS_HOST=redis REDIS_PORT=6379 REDIS_PASSWORD= + +# Number of worker processes for Log Stream deliveries (optional, default 2). +# Log shipping is network-bound and serialized per stream, so useful +# concurrency roughly equals your number of active Log Streams. +#LOG_STREAM_WORKERS=2 diff --git a/backend/api/config.py b/backend/api/config.py index a8a092124..64dcbafc0 100644 --- a/backend/api/config.py +++ b/backend/api/config.py @@ -12,6 +12,7 @@ def ready(self): # Connect the post_migrate signal to a custom handler post_migrate.connect(self.validate_licenses_post_migrate, sender=self) + post_migrate.connect(self.init_log_streams_post_migrate, sender=self) def validate_licenses_post_migrate(self, **kwargs): @@ -28,3 +29,11 @@ def validate_licenses_post_migrate(self, **kwargs): activate_license(settings.PHASE_LICENSE) except Exception as e: logging.exception("Failed to activate license: %s", e) + + def init_log_streams_post_migrate(self, **kwargs): + try: + from ee.integrations.logs.streams.jobs import init_log_stream_sweeper + + init_log_stream_sweeper() + except Exception: + logging.exception("Failed to initialise log stream sweeper") diff --git a/backend/api/management/commands/rqworker.py b/backend/api/management/commands/rqworker.py index a9fdf53f4..6ec8bd999 100644 --- a/backend/api/management/commands/rqworker.py +++ b/backend/api/management/commands/rqworker.py @@ -1,9 +1,14 @@ from django.core.management.base import BaseCommand from django.core import management +import logging import multiprocessing -from django.core.management.base import BaseCommand +from multiprocessing.connection import wait +import os +import sys from django_rq.management.commands.rqworker import Command as OriginalRQWorkerCommand +logger = logging.getLogger(__name__) + class Command(BaseCommand): help = "Runs both RQ worker and RQ scheduler in parallel" @@ -50,26 +55,115 @@ def run_scheduler(self): # to a full minute past its due time before being moved to the queue. management.call_command("rqscheduler", "scheduled-jobs", interval=2) + def run_log_streams_workers(self): + """Starts the worker pool for the log-streams queue. + + Log shipping is network-I/O bound and per-stream serialized, so + useful concurrency ~= number of active streams. Sized via the + LOG_STREAM_WORKERS env var. + + Parsed defensively: this runs in a supervised child, and a crash here + (e.g. a typo'd env value) would tear down the whole worker container + — syncs, emails and rotations included, not just log streams. A + zero/negative value would silently starve the queue while the + container looks healthy, so it is clamped to 1. + """ + default_workers = 2 + raw = os.getenv("LOG_STREAM_WORKERS", "") + try: + workers = int(raw) if raw.strip() else default_workers + except ValueError: + logger.warning( + "Invalid LOG_STREAM_WORKERS value %r — using the default (%s)", + raw, + default_workers, + ) + workers = default_workers + if workers < 1: + logger.warning( + "LOG_STREAM_WORKERS=%s would start no delivery workers — clamping to 1", + workers, + ) + workers = 1 + self.stdout.write( + self.style.SUCCESS( + f"Starting log-streams RQ worker pool with {workers} workers..." + ) + ) + + management.call_command("rqworker-pool", "log-streams", num_workers=workers) + + def bootstrap_log_stream_schedule(self): + """(Re-)register the recurring log stream sweep at worker startup. + + The schedule lives only in Redis. If it's lost — a Redis restart, or + rq-scheduler dropping an interval job whose hash expired while the + host was frozen — backend post_migrate wouldn't re-create it until + the next deploy. Worker startup is the natural recovery point; the + registration is idempotent (stable id, cancel-before-schedule). + """ + try: + from ee.integrations.logs.streams.jobs import init_log_stream_sweeper + + init_log_stream_sweeper() + except Exception: + logger.exception("Failed to register log stream sweeper at worker startup") + def handle(self, *args, **options): queue = options["queue"] num_workers = options["num_workers"] - default_workers_process = multiprocessing.Process( - target=self.run_default_workers, - args=( - queue, - num_workers, + self.bootstrap_log_stream_schedule() + + processes = [ + multiprocessing.Process( + name="rqworker-pool-default", + target=self.run_default_workers, + args=( + queue, + num_workers, + ), ), - ) - scheduled_jobs_worker_process = multiprocessing.Process( - target=self.run_scheduled_jobs_worker - ) - scheduler_process = multiprocessing.Process(target=self.run_scheduler) + multiprocessing.Process( + name="rqworker-scheduled-jobs", + target=self.run_scheduled_jobs_worker, + ), + multiprocessing.Process(name="rqscheduler", target=self.run_scheduler), + multiprocessing.Process( + name="rqworker-pool-log-streams", + target=self.run_log_streams_workers, + ), + ] + + for process in processes: + process.start() - default_workers_process.start() - scheduled_jobs_worker_process.start() - scheduler_process.start() + # Supervise the children instead of blindly join()ing them: a dead + # worker or scheduler process used to leave the container "Up" but + # silently degraded (e.g. rq-scheduler dying after a host sleep stops + # every recurring job with no visible failure). `wait()` blocks until + # any child's sentinel fires; exiting non-zero lets the container + # restart policy bring the whole pool back up cleanly. + try: + wait([process.sentinel for process in processes]) + except KeyboardInterrupt: + self._shutdown(processes) + return + + dead = next((p for p in processes if not p.is_alive()), None) + self.stderr.write( + self.style.ERROR( + f"{dead.name if dead else 'a worker process'} exited unexpectedly " + f"(exitcode={dead.exitcode if dead else '?'}); shutting down " + "worker pool for a clean restart" + ) + ) + self._shutdown(processes) + sys.exit(1) - default_workers_process.join() - scheduled_jobs_worker_process.join() - scheduler_process.join() + def _shutdown(self, processes): + for process in processes: + if process.is_alive(): + process.terminate() + for process in processes: + process.join() diff --git a/backend/api/migrations/0132_log_streams.py b/backend/api/migrations/0132_log_streams.py new file mode 100644 index 000000000..51a84cd87 --- /dev/null +++ b/backend/api/migrations/0132_log_streams.py @@ -0,0 +1,73 @@ +# Generated by Django 4.2.30 on 2026-08-02 14:06 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0131_add_queued_sync_status'), + ] + + operations = [ + migrations.AlterField( + model_name='auditevent', + name='resource_type', + field=models.CharField(choices=[('app', 'App'), ('env', 'Environment'), ('role', 'Role'), ('sa', 'ServiceAccount'), ('member', 'OrganisationMember'), ('policy', 'NetworkAccessPolicy'), ('pat', 'UserToken'), ('sa_token', 'ServiceAccountToken'), ('svc_token', 'ServiceToken'), ('invite', 'Invite'), ('team', 'Team'), ('rs', 'RotatingSecret'), ('stream', 'LogStream')], max_length=10), + ), + migrations.AlterField( + model_name='providercredentials', + name='provider', + field=models.CharField(choices=[('cloudflare', 'Cloudflare'), ('aws', 'AWS'), ('aws_assume_role', 'AWS Assume Role'), ('github', 'GitHub'), ('gitlab', 'GitLab'), ('hashicorp_vault', 'Hashicorp Vault'), ('hashicorp_nomad', 'Hashicorp Nomad'), ('railway', 'Railway'), ('vercel', 'Vercel'), ('render', 'Render'), ('azure', 'Azure'), ('openai', 'OpenAI'), ('litellm', 'LiteLLM'), ('datadog', 'Datadog')], max_length=50), + ), + migrations.CreateModel( + name='LogStream', + fields=[ + ('id', models.TextField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=64)), + ('provider', models.CharField(help_text='Log stream adapter id (resolved against the log stream adapter registry).', max_length=50)), + ('sources', models.JSONField(default=list, help_text='Event source ids to ship (resolved against the log stream source registry).')), + ('options', models.JSONField(default=dict)), + ('max_attempts', models.PositiveIntegerField(default=5, help_text='Delivery attempts per chunk before it is recorded as failed and skipped.')), + ('is_active', models.BooleanField(default=True)), + ('health', models.CharField(choices=[('healthy', 'Healthy'), ('degraded', 'Degraded')], default='healthy', max_length=20)), + ('paused_reason', models.TextField(blank=True, default='')), + ('cursors', models.JSONField(default=dict)), + ('ship_job_id', models.TextField(blank=True, null=True)), + ('last_shipped_at', models.DateTimeField(blank=True, null=True)), + ('last_failure_at', models.DateTimeField(blank=True, null=True)), + ('last_failure_reason', models.TextField(blank=True, default='')), + ('created_at', models.DateTimeField(auto_now_add=True, null=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('deleted_at', models.DateTimeField(blank=True, null=True)), + ('authentication', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='log_streams', to='api.providercredentials')), + ('organisation', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='log_streams', to='api.organisation')), + ], + ), + migrations.CreateModel( + name='LogStreamDeliveryEvent', + fields=[ + ('id', models.TextField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('source', models.CharField(max_length=32)), + ('status', models.CharField(choices=[('completed', 'Completed'), ('failed', 'Failed'), ('skipped', 'Skipped')], max_length=16)), + ('event_count', models.PositiveIntegerField(default=0)), + ('payload_bytes', models.PositiveIntegerField(default=0)), + ('attempts', models.PositiveIntegerField(default=0)), + ('cursor_from', models.DateTimeField(blank=True, null=True)), + ('cursor_to', models.DateTimeField(blank=True, null=True)), + ('resolved_at', models.DateTimeField(blank=True, null=True)), + ('job_id', models.TextField(blank=True, null=True)), + ('meta', models.JSONField(null=True)), + ('created_at', models.DateTimeField(auto_now_add=True, null=True)), + ('completed_at', models.DateTimeField(blank=True, null=True)), + ('retried_from', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='retries', to='api.logstreamdeliveryevent')), + ('stream', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='delivery_events', to='api.logstream')), + ], + options={ + 'ordering': ['-created_at'], + 'indexes': [models.Index(fields=['stream', '-created_at'], name='log_stream_delivery_idx')], + }, + ), + ] diff --git a/backend/api/migrations/0133_logstreamdeliveryevent_cursor_from_id_and_more.py b/backend/api/migrations/0133_logstreamdeliveryevent_cursor_from_id_and_more.py new file mode 100644 index 000000000..23819ee17 --- /dev/null +++ b/backend/api/migrations/0133_logstreamdeliveryevent_cursor_from_id_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.30 on 2026-08-03 19:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0132_log_streams'), + ] + + operations = [ + migrations.AddField( + model_name='logstreamdeliveryevent', + name='cursor_from_id', + field=models.TextField(blank=True, default=''), + ), + migrations.AddField( + model_name='logstreamdeliveryevent', + name='cursor_to_id', + field=models.TextField(blank=True, default=''), + ), + ] diff --git a/backend/api/migrations/0134_logstream_drop_job_id.py b/backend/api/migrations/0134_logstream_drop_job_id.py new file mode 100644 index 000000000..332060c16 --- /dev/null +++ b/backend/api/migrations/0134_logstream_drop_job_id.py @@ -0,0 +1,17 @@ +# Generated by Django 5.2.17 on 2026-08-10 10:54 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0133_logstreamdeliveryevent_cursor_from_id_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='logstreamdeliveryevent', + name='job_id', + ), + ] diff --git a/backend/api/migrations/0135_sync_provider_service_choices.py b/backend/api/migrations/0135_sync_provider_service_choices.py new file mode 100644 index 000000000..2b91c0cf4 --- /dev/null +++ b/backend/api/migrations/0135_sync_provider_service_choices.py @@ -0,0 +1,25 @@ +# Choices-only sync for EnvironmentSync.service / ProviderCredentials.provider +# (label recasing + openai/litellm/datadog additions from earlier commits). +# CharField choices are validation-level only — no DB DDL is emitted. + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0134_logstream_drop_job_id'), + ] + + operations = [ + migrations.AlterField( + model_name='environmentsync', + name='service', + field=models.CharField(choices=[('cloudflare_pages', 'Cloudflare Pages'), ('cloudflare_workers', 'Cloudflare Workers'), ('aws_secrets_manager', 'AWS Secrets Manager'), ('github_actions', 'GitHub Actions'), ('github_dependabot', 'GitHub Dependabot'), ('gitlab_ci', 'GitLab CI'), ('hashicorp_vault', 'HashiCorp Vault'), ('hashicorp_nomad', 'HashiCorp Nomad'), ('railway', 'Railway'), ('vercel', 'Vercel'), ('render', 'Render'), ('azure_key_vault', 'Azure Key Vault')], max_length=50), + ), + migrations.AlterField( + model_name='providercredentials', + name='provider', + field=models.CharField(choices=[('cloudflare', 'Cloudflare'), ('aws', 'AWS'), ('aws_assume_role', 'AWS Assume Role'), ('github', 'GitHub'), ('gitlab', 'GitLab'), ('hashicorp_vault', 'HashiCorp Vault'), ('hashicorp_nomad', 'HashiCorp Nomad'), ('railway', 'Railway'), ('vercel', 'Vercel'), ('render', 'Render'), ('azure', 'Azure'), ('openai', 'OpenAI'), ('litellm', 'LiteLLM'), ('datadog', 'Datadog')], max_length=50), + ), + ] diff --git a/backend/api/migrations/0136_logstream_unresolved_idx.py b/backend/api/migrations/0136_logstream_unresolved_idx.py new file mode 100644 index 000000000..31ea09289 --- /dev/null +++ b/backend/api/migrations/0136_logstream_unresolved_idx.py @@ -0,0 +1,27 @@ +# Concurrent index build: LogStreamDeliveryEvent takes continuous writes +# from the ship path, so a plain CREATE INDEX would block them. Mirrors +# migration 0112 (SecretEvent). +# +# AddIndexConcurrently must be the ONLY operation in its (atomic=False) +# migration: if the build fails midway, any earlier operation has already +# committed and re-running the migration would crash on it, wedging the +# deploy. + +from django.contrib.postgres.operations import AddIndexConcurrently +from django.db import migrations, models + + +class Migration(migrations.Migration): + + atomic = False + + dependencies = [ + ('api', '0135_sync_provider_service_choices'), + ] + + operations = [ + AddIndexConcurrently( + model_name='logstreamdeliveryevent', + index=models.Index(condition=models.Q(('resolved_at__isnull', True), ('status__in', ['failed', 'skipped'])), fields=['stream', 'source'], name='log_stream_unresolved_idx'), + ), + ] diff --git a/backend/api/models.py b/backend/api/models.py index e120d8662..0ac78708c 100644 --- a/backend/api/models.py +++ b/backend/api/models.py @@ -1299,6 +1299,7 @@ class AuditEvent(models.Model): INVITE = "invite" TEAM = "team" ROTATING_SECRET = "rs" + LOG_STREAM = "stream" RESOURCE_TYPES = [ (APP, "App"), (ENVIRONMENT, "Environment"), @@ -1312,6 +1313,7 @@ class AuditEvent(models.Model): (INVITE, "Invite"), (TEAM, "Team"), (ROTATING_SECRET, "RotatingSecret"), + (LOG_STREAM, "LogStream"), ] class Meta: @@ -1691,3 +1693,141 @@ class Lockbox(models.Model): created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) expires_at = models.DateTimeField(null=True) allowed_views = models.IntegerField(null=True) + + +class LogStream(models.Model): + """ + Configuration for streaming audit/secret event logs to an external + log management platform (e.g. Datadog). + + Scope: Organisation level. Delivery state (cursors, health, job ids) is + managed exclusively by the log stream engine (ee.integrations.logs.streams). + """ + + HEALTHY = "healthy" + DEGRADED = "degraded" + HEALTH_CHOICES = [ + (HEALTHY, "Healthy"), + (DEGRADED, "Degraded"), + ] + + id = models.TextField(default=uuid4, primary_key=True, editable=False) + organisation = models.ForeignKey( + Organisation, on_delete=models.CASCADE, related_name="log_streams" + ) + name = models.CharField(max_length=64) + provider = models.CharField( + max_length=50, + help_text="Log stream adapter id (resolved against the log stream adapter registry).", + ) + authentication = models.ForeignKey( + ProviderCredentials, + on_delete=models.SET_NULL, + null=True, + related_name="log_streams", + ) + sources = models.JSONField( + default=list, + help_text="Event source ids to ship (resolved against the log stream source registry).", + ) + options = models.JSONField(default=dict) + max_attempts = models.PositiveIntegerField( + default=5, + help_text="Delivery attempts per chunk before it is recorded as failed and skipped.", + ) + is_active = models.BooleanField(default=True) + health = models.CharField(max_length=20, choices=HEALTH_CHOICES, default=HEALTHY) + # Set when the engine pauses the stream (e.g. "auth_error"); empty when + # paused by a user. + paused_reason = models.TextField(blank=True, default="") + # Per-source delivery cursors: {"": {"ts": "", "id": ""}}. + # A cursor only advances after its chunk is accepted by the destination. + cursors = models.JSONField(default=dict) + ship_job_id = models.TextField(null=True, blank=True) + last_shipped_at = models.DateTimeField(null=True, blank=True) + last_failure_at = models.DateTimeField(null=True, blank=True) + last_failure_reason = models.TextField(blank=True, default="") + created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) + updated_at = models.DateTimeField(auto_now=True) + deleted_at = models.DateTimeField(blank=True, null=True) + + def delete(self, *args, **kwargs): + from ee.integrations.logs.streams.engine import cancel_ship_job + + self.is_active = False + self.updated_at = timezone.now() + self.deleted_at = timezone.now() + self.save() + + # A deleted stream's failed/skipped ranges can never be re-shipped, + # and auto-resolve/retention both skip deleted streams — resolve them + # here or they are exempt from retention forever. + self.delivery_events.filter( + status__in=[ + LogStreamDeliveryEvent.FAILED, + LogStreamDeliveryEvent.SKIPPED, + ], + resolved_at__isnull=True, + ).update(resolved_at=timezone.now()) + + cancel_ship_job(self) + + +class LogStreamDeliveryEvent(models.Model): + COMPLETED = "completed" + FAILED = "failed" + SKIPPED = "skipped" + + STATUS_OPTIONS = [ + (COMPLETED, "Completed"), + (FAILED, "Failed"), + (SKIPPED, "Skipped"), + ] + + id = models.TextField(default=uuid4, primary_key=True, editable=False) + stream = models.ForeignKey( + LogStream, on_delete=models.CASCADE, related_name="delivery_events" + ) + source = models.CharField(max_length=32) + # Rows are written once with a terminal status; chunk lifecycle is seconds, + # so there is no queued/in-progress state to track here. + status = models.CharField(max_length=16, choices=STATUS_OPTIONS) + event_count = models.PositiveIntegerField(default=0) + payload_bytes = models.PositiveIntegerField(default=0) + attempts = models.PositiveIntegerField(default=0) + cursor_from = models.DateTimeField(null=True, blank=True) + cursor_to = models.DateTimeField(null=True, blank=True) + # Id bounds of the covered range. Events are ordered by (timestamp, id), + # and chunks can split inside a single timestamp — auto-resolving a + # failed range by timestamp containment alone could match a chunk that + # covered the same timestamps but different events. Empty string means + # "unknown" (legacy rows / open boundaries) and compares leniently. + cursor_from_id = models.TextField(default="", blank=True) + cursor_to_id = models.TextField(default="", blank=True) + # Set on delivery events created by a manual retry of a failed/skipped one. + retried_from = models.ForeignKey( + "self", on_delete=models.SET_NULL, null=True, blank=True, related_name="retries" + ) + # Set on a failed/skipped event once a manual retry covering it succeeds. + resolved_at = models.DateTimeField(null=True, blank=True) + meta = models.JSONField(null=True) + created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) + completed_at = models.DateTimeField(blank=True, null=True) + + class Meta: + indexes = [ + models.Index( + fields=["stream", "-created_at"], + name="log_stream_delivery_idx", + ), + # Unresolved rows are rare but queried constantly (badge count, + # auto-resolve) — partial, so completed rows never enter it. + models.Index( + fields=["stream", "source"], + condition=models.Q( + status__in=["failed", "skipped"], resolved_at__isnull=True + ), + name="log_stream_unresolved_idx", + ), + ] + ordering = ["-created_at"] diff --git a/backend/api/services.py b/backend/api/services.py index 053e7ba6e..c8e568772 100644 --- a/backend/api/services.py +++ b/backend/api/services.py @@ -1,3 +1,27 @@ +# The Datadog site composes into intake/API URLs, so this allowlist doubles +# as an SSRF guard for credential values. Canonical list — credential +# validation and the log stream adapter both use it (the console picker in +# frontend/utils/syncing/datadog.ts mirrors it for display names). +DATADOG_SITES = ( + "datadoghq.com", + "us3.datadoghq.com", + "us5.datadoghq.com", + "datadoghq.eu", + "uk1.datadoghq.com", + "ap1.datadoghq.com", + "ap2.datadoghq.com", + "ddog-gov.com", + "us2.ddog-gov.com", +) + + +def normalize_datadog_site(value): + """Lowercase and strip scheme/slashes so pasted values like + "https://us3.datadoghq.com/" match the allowlist.""" + site = str(value or "").strip().lower() + return site.removeprefix("https://").removeprefix("http://").strip("/") + + class Providers: CLOUDFLARE = { "id": "cloudflare", @@ -110,6 +134,14 @@ class Providers: "auth_scheme": "token", } + DATADOG = { + "id": "datadog", + "name": "Datadog", + "expected_credentials": ["api_key", "site"], + "optional_credentials": ["application_key"], + "auth_scheme": "token", + } + @classmethod def get_provider_choices(cls): return [ diff --git a/backend/api/utils/access/org_resolution.py b/backend/api/utils/access/org_resolution.py index 94e35ff30..09123cbab 100644 --- a/backend/api/utils/access/org_resolution.py +++ b/backend/api/utils/access/org_resolution.py @@ -29,6 +29,8 @@ "policy_id": "NetworkAccessPolicy", "credential_id": "ProviderCredentials", "sync_id": "EnvironmentSync", + "stream_id": "LogStream", + "delivery_event_id": "LogStreamDeliveryEvent", } # Handled by the middleware's dedicated probe (multiple backing models). diff --git a/backend/api/utils/access/permissions.py b/backend/api/utils/access/permissions.py index f6b9fbe86..0d10c27c7 100644 --- a/backend/api/utils/access/permissions.py +++ b/backend/api/utils/access/permissions.py @@ -264,6 +264,18 @@ def role_has_global_access(role): return False # Role is not valid +def user_has_global_access(user, organisation): + """True when the user's role in the organisation has global access.""" + OrganisationMember = apps.get_model("api", "OrganisationMember") + try: + member = OrganisationMember.objects.get( + user=user, organisation=organisation, deleted_at=None + ) + except OrganisationMember.DoesNotExist: + return False + return role_has_global_access(member.role) + + def _check_sa_permission(user, service_account, action, resource): """Permission check for service account operations. diff --git a/backend/api/utils/access/roles.py b/backend/api/utils/access/roles.py index 0a13b4605..0d74db198 100644 --- a/backend/api/utils/access/roles.py +++ b/backend/api/utils/access/roles.py @@ -20,6 +20,7 @@ "SSO": ["create", "read", "update", "delete"], "Teams": ["create", "read", "update", "delete"], "SCIM": ["create", "read", "update", "delete"], + "LogStreams": ["create", "read", "update", "delete"], }, "app_permissions": { "Environments": ["create", "read", "update", "delete"], @@ -58,6 +59,7 @@ "SSO": ["create", "read", "update", "delete"], "Teams": ["create", "read", "update", "delete"], "SCIM": ["create", "read", "update", "delete"], + "LogStreams": ["create", "read", "update", "delete"], }, "app_permissions": { "Environments": ["create", "read", "update", "delete"], @@ -95,6 +97,7 @@ "SSO": [], "Teams": ["create", "read", "update", "delete"], "SCIM": [], + "LogStreams": [], }, "app_permissions": { "Environments": ["read", "create", "update"], @@ -136,6 +139,7 @@ "SSO": [], "Teams": ["read"], "SCIM": [], + "LogStreams": [], }, "app_permissions": { "Environments": ["read", "create", "update"], @@ -173,6 +177,7 @@ "SSO": [], "Teams": [], "SCIM": [], + "LogStreams": [], }, "app_permissions": { "Environments": ["read", "create", "update", "delete"], diff --git a/backend/api/utils/syncing/auth.py b/backend/api/utils/syncing/auth.py index e01d6130f..4262c69ce 100644 --- a/backend/api/utils/syncing/auth.py +++ b/backend/api/utils/syncing/auth.py @@ -35,6 +35,24 @@ def store_oauth_token( return credential +def decrypt_credential_values(credential, keys): + """Decrypt selected fields from an in-memory ProviderCredentials row. + + For callers that need one or two non-sensitive values (e.g. the Datadog + site for a destination link) — avoids decrypting the whole credential + set (API keys included) and the extra row fetch of get_credentials. + """ + pk, sk = get_server_keypair() + values = {} + for key in keys: + encrypted_value = (credential.credentials or {}).get(key) + if encrypted_value is not None: + value = decrypt_asymmetric(encrypted_value, sk.hex(), pk.hex()) + if value is not None: + values[key] = value + return values + + def get_credentials(credential_id): ProviderCredentials = apps.get_model("api", "ProviderCredentials") diff --git a/backend/api/views/audit.py b/backend/api/views/audit.py index e85a1c493..1f1e157f5 100644 --- a/backend/api/views/audit.py +++ b/backend/api/views/audit.py @@ -2,7 +2,7 @@ from api.auth import PhaseTokenAuthentication from api.models import AuditEvent -from api.utils.access.permissions import user_has_permission +from api.utils.access.permissions import role_has_global_access, user_has_permission from api.utils.database import get_approximate_count from api.utils.rest import METHOD_TO_ACTION from api.throttling import PlanBasedRateThrottle @@ -35,18 +35,39 @@ def _get_org(self, request): def initial(self, request, *args, **kwargs): super().initial(request, *args, **kwargs) - account = None - is_sa = False if request.auth["auth_type"] == "User": account = request.auth["org_member"].user + role = request.auth["org_member"].role elif request.auth["auth_type"] == "ServiceAccount": - account = request.auth["service_account"] - is_sa = True + # Service accounts cannot hold global-access roles (enforced at + # SA create/update), so they can never satisfy the org-wide + # guard below — reject with an actionable message instead of an + # unsatisfiable "requires global access". + raise PermissionDenied( + "Audit logs cannot be accessed with a service account token. " + "Use a user token whose role has global access." + ) + else: + # Fail closed: legacy service tokens are environment-scoped and + # have no role/permission model — they must never read + # organisation-wide audit logs. + raise PermissionDenied( + "Audit logs cannot be accessed with a service token." + ) - if account is not None: - org = self._get_org(request) - if not user_has_permission(account, "read", "Logs", org, False, is_sa): - raise PermissionDenied("You don't have permission to view audit logs.") + org = self._get_org(request) + if not user_has_permission(account, "read", "Logs", org, False, False): + raise PermissionDenied("You don't have permission to view audit logs.") + + # This endpoint returns the unscoped org-wide stream (SIEM export / + # backfill). The Console's GraphQL resolver filters events for roles + # without global access to their accessible apps/envs — that scoping + # isn't replicated here, so fail closed for scoped roles instead of + # over-exposing. + if not role_has_global_access(role): + raise PermissionDenied( + "The audit logs API requires a role with global access." + ) def get(self, request, *args, **kwargs): org = self._get_org(request) diff --git a/backend/backend/graphene/mutations/syncing.py b/backend/backend/graphene/mutations/syncing.py index aa40b58fa..7f0411e1d 100644 --- a/backend/backend/graphene/mutations/syncing.py +++ b/backend/backend/graphene/mutations/syncing.py @@ -92,6 +92,33 @@ def mutate(cls, root, info, app_id, env_keys): return InitEnvSync(app=app) +def validate_credential_values(provider_id, credentials): + """Server-side validation of provider-specific credential fields. + + The Datadog site composes into intake/API URLs (an SSRF surface) and a + bad value only surfaces at ship time — enforce the allowlist here, not + just in the console picker. Values arrive encrypted with the server + public key, so validation decrypts the field it checks. + """ + if provider_id != "datadog": + return + from api.services import DATADOG_SITES, normalize_datadog_site + from api.utils.crypto import decrypt_asymmetric, get_server_keypair + + encrypted_site = (credentials or {}).get("site") + if not encrypted_site: + raise GraphQLError("A Datadog site is required") + pk, sk = get_server_keypair() + try: + site = decrypt_asymmetric(encrypted_site, sk.hex(), pk.hex()) + except Exception: + raise GraphQLError("Could not read the Datadog site value") + if normalize_datadog_site(site) not in DATADOG_SITES: + raise GraphQLError( + "Unknown Datadog site. Choose one of the supported Datadog regions." + ) + + class CreateProviderCredentials(graphene.Mutation): class Arguments: org_id = graphene.ID() @@ -113,6 +140,8 @@ def mutate(cls, root, info, org_id, provider, name, credentials): "You don't have permission to create Integration Credentials" ) + validate_credential_values(provider, credentials) + credential = ProviderCredentials.objects.create( organisation=org, name=name, provider=provider, credentials=credentials ) @@ -142,6 +171,8 @@ def mutate(cls, root, info, credential_id, name, credentials): "You don't have permission to update Integration Credentials" ) + validate_credential_values(credential.provider, credentials) + credential.name = name credential.credentials = credentials credential.save() diff --git a/backend/backend/graphene/types.py b/backend/backend/graphene/types.py index 9112c795e..138fa4dd0 100644 --- a/backend/backend/graphene/types.py +++ b/backend/backend/graphene/types.py @@ -988,9 +988,17 @@ class Meta: ) def resolve_sync_count(self, info): - return EnvironmentSync.objects.filter( - authentication_id=self.id, deleted_at=None - ).count() + """Count the sync jobs and the Logstreams that will be disrupted if a third-party credential is deleted.""" + from api.models import LogStream + + return ( + EnvironmentSync.objects.filter( + authentication_id=self.id, deleted_at=None + ).count() + + LogStream.objects.filter( + authentication_id=self.id, deleted_at=None + ).count() + ) def resolve_provider(self, info): return Providers.get_provider_config(self.provider) diff --git a/backend/backend/quotas.py b/backend/backend/quotas.py index 0af55dd5b..21aa98094 100644 --- a/backend/backend/quotas.py +++ b/backend/backend/quotas.py @@ -159,6 +159,11 @@ def can_use_scim(organisation): return organisation.plan == "EN" +def can_use_log_streams(organisation): + """Log Streams require an Enterprise plan.""" + return organisation.plan == "EN" + + def can_use_rotating_secrets(organisation): """Rotating Secrets require a Pro or Enterprise plan (or a valid license).""" if organisation_has_valid_license(organisation): diff --git a/backend/backend/schema.py b/backend/backend/schema.py index 2e4a58a60..9f9f1f167 100644 --- a/backend/backend/schema.py +++ b/backend/backend/schema.py @@ -57,6 +57,32 @@ _ROTATION_AVAILABLE = True except ImportError: pass +_LOG_STREAMS_AVAILABLE = False +try: + from ee.integrations.logs.streams.graphene.types import ( + LogStreamDeliveryHistoryType, + LogStreamProviderType, + LogStreamSourceType, + LogStreamType, + ) + from ee.integrations.logs.streams.graphene.queries import ( + resolve_log_stream_deliveries, + resolve_log_stream_providers, + resolve_log_stream_sources, + resolve_log_streams, + ) + from ee.integrations.logs.streams.graphene.mutations import ( + CreateLogStreamMutation, + DeleteLogStreamMutation, + RetryLogStreamDeliveryMutation, + TestLogStreamConnectionMutation, + ToggleLogStreamMutation, + UpdateLogStreamMutation, + ) + + _LOG_STREAMS_AVAILABLE = True +except ImportError: + pass from backend.graphene.mutations.service_accounts import ( CreateServiceAccountMutation, CreateServiceAccountTokenMutation, @@ -634,6 +660,21 @@ class Query(graphene.ObjectType): source_rotating_secret_id=graphene.ID(required=True), ) + # Log Streams (Enterprise) + if _LOG_STREAMS_AVAILABLE: + log_streams = graphene.List( + LogStreamType, organisation_id=graphene.ID(required=True) + ) + log_stream_deliveries = graphene.Field( + LogStreamDeliveryHistoryType, + stream_id=graphene.ID(required=True), + limit=graphene.Int(required=False), + offset=graphene.Int(required=False), + status=graphene.String(required=False), + ) + log_stream_providers = graphene.List(LogStreamProviderType) + log_stream_sources = graphene.List(LogStreamSourceType) + # -------------------------------------------------------------------- resolve_server_public_key = resolve_server_public_key @@ -695,6 +736,12 @@ class Query(graphene.ObjectType): resolve_openai_projects = resolve_openai_projects resolve_rotation_clone_spec = resolve_rotation_clone_spec + if _LOG_STREAMS_AVAILABLE: + resolve_log_streams = resolve_log_streams + resolve_log_stream_deliveries = resolve_log_stream_deliveries + resolve_log_stream_providers = resolve_log_stream_providers + resolve_log_stream_sources = resolve_log_stream_sources + def resolve_organisations(root, info): memberships = OrganisationMember.objects.filter( user=info.context.user, deleted_at=None @@ -1575,5 +1622,14 @@ class Mutation(graphene.ObjectType): resume_rotating_secret = ResumeRotatingSecretMutation.Field() validate_rotation_credentials = ValidateRotationCredentialsMutation.Field() + # Log Streams (Enterprise) + if _LOG_STREAMS_AVAILABLE: + create_log_stream = CreateLogStreamMutation.Field() + update_log_stream = UpdateLogStreamMutation.Field() + toggle_log_stream = ToggleLogStreamMutation.Field() + delete_log_stream = DeleteLogStreamMutation.Field() + test_log_stream_connection = TestLogStreamConnectionMutation.Field() + retry_log_stream_delivery = RetryLogStreamDeliveryMutation.Field() + schema = graphene.Schema(query=Query, mutation=Mutation) diff --git a/backend/backend/settings.py b/backend/backend/settings.py index 59a3a332f..a4f8cf894 100644 --- a/backend/backend/settings.py +++ b/backend/backend/settings.py @@ -397,6 +397,15 @@ def get_version(): "SSL_OPTIONS": RQ_SSL_OPTIONS, "DB": 0, }, + "log-streams": { + "HOST": REDIS_HOST, + "PORT": REDIS_PORT, + "USERNAME": REDIS_USER, + "PASSWORD": REDIS_PASSWORD, + "SSL": REDIS_SSL, + "SSL_OPTIONS": RQ_SSL_OPTIONS, + "DB": 0, + }, } DYNAMODB = { diff --git a/backend/backend/urls.py b/backend/backend/urls.py index b829b7b1e..a7050d7f1 100644 --- a/backend/backend/urls.py +++ b/backend/backend/urls.py @@ -6,6 +6,8 @@ from api.views.graphql import PrivateGraphQLView from api.views.apps import PublicAppsView, PublicAppDetailView from api.views.environments import PublicEnvironmentsView, PublicEnvironmentDetailView +# PublicAuditLogsView route disabled below pending a performance pass. +# from api.views.audit import PublicAuditLogsView from api.views.secrets import E2EESecretsView, PublicSecretsView from api.views.service_accounts import ( PublicServiceAccountsView, @@ -117,6 +119,11 @@ path("v1/teams//members/", PublicTeamMembersView.as_view()), path("v1/teams//members//", PublicTeamMemberDetailView.as_view()), path("v1/teams//access/", PublicTeamAccessView.as_view()), + # Disabled pending a performance pass: actor_id filtering seq-scans the + # whole (multi-tenant) AuditEvent table, and offset/time-range are + # unbounded. Re-enable with an actor_type guard, keyset pagination and a + # range cap. The view/authz/tests stay in place. + # path("v1/logs/audit/", PublicAuditLogsView.as_view()), path("identities/external/v1/aws/iam/auth/", aws_iam_auth), path("identities/external/v1/azure/entra/auth/", azure_entra_auth), ] diff --git a/backend/ee/integrations/logs/__init__.py b/backend/ee/integrations/logs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/ee/integrations/logs/streams/__init__.py b/backend/ee/integrations/logs/streams/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/ee/integrations/logs/streams/adapters/__init__.py b/backend/ee/integrations/logs/streams/adapters/__init__.py new file mode 100644 index 000000000..99e62d464 --- /dev/null +++ b/backend/ee/integrations/logs/streams/adapters/__init__.py @@ -0,0 +1,24 @@ +"""Log stream destination adapter registry. + +Adding a destination = one adapter module + one entry here. Adapters must +stay dependency-free (plain requests); see base.LogStreamAdapter for the +contract. +""" + +from .datadog import DatadogAdapter + +ADAPTERS = { + adapter.id: adapter + for adapter in (DatadogAdapter(),) +} + + +def get_adapter(adapter_id): + adapter = ADAPTERS.get(adapter_id) + if adapter is None: + raise ValueError(f"Unknown log stream provider '{adapter_id}'") + return adapter + + +def all_adapters(): + return list(ADAPTERS.values()) diff --git a/backend/ee/integrations/logs/streams/adapters/base.py b/backend/ee/integrations/logs/streams/adapters/base.py new file mode 100644 index 000000000..fd171cbce --- /dev/null +++ b/backend/ee/integrations/logs/streams/adapters/base.py @@ -0,0 +1,55 @@ +"""Adapter contract for log stream destinations. + +An adapter turns a chunk of neutral envelopes into one delivery to a specific +destination. The contract is deliberately destination-agnostic: credentials +come from the linked ProviderCredentials record (decrypted by the engine), +`options` is adapter-validated free-form config, and `context` carries stream +metadata (organisation/stream names) for tagging. Adapters that need their own +token exchange (e.g. Microsoft Sentinel via Azure AD) do it inside `ship()`. + +Raise the typed errors from ..exceptions to drive engine behaviour; return a +ShipResult on success. +""" + +from dataclasses import dataclass, field + + +@dataclass +class ShipResult: + status_code: int + duration_ms: int = 0 + meta: dict = field(default_factory=dict) + + +class LogStreamAdapter: + id = None + name = None + # Providers registry id whose credentials this adapter consumes. + credentials_provider = None + # Oldest event timestamp the destination accepts (None = unlimited). + max_event_age = None + # Credential keys destination_url() reads. Lets resolvers decrypt only + # these (polled queries must not decrypt the API key just to build a + # link); empty means destination_url needs the full credential set. + url_credential_keys = () + + def validate_options(self, options): + """Normalise and validate stream options; raise ValueError on bad input.""" + return options or {} + + def destination_url(self, credentials, options): + """Deep link to the shipped logs in the destination's UI, or None.""" + return None + + def ship(self, chunk, credentials, options, context): + raise NotImplementedError + + def test(self, credentials, options, context): + """Verify the credentials and options against the destination. + + Contract: this MUST NOT ingest any data into the destination — use + the provider's key-validation or health endpoint. Only ship a + synthetic event as an explicit last resort for a destination with no + such endpoint. + """ + raise NotImplementedError diff --git a/backend/ee/integrations/logs/streams/adapters/datadog.py b/backend/ee/integrations/logs/streams/adapters/datadog.py new file mode 100644 index 000000000..e96c9cb28 --- /dev/null +++ b/backend/ee/integrations/logs/streams/adapters/datadog.py @@ -0,0 +1,271 @@ +"""Datadog Logs intake adapter. + +Ships chunks to ``https://http-intake.logs.{site}/api/v2/logs`` with the +stream's API key. Uses plain ``requests`` — no Datadog SDK. + +The neutral envelope is remapped onto Datadog standard attributes here so +logs light up native facets with zero pipeline configuration: + +- ``client.address`` -> ``network.client.ip`` +- ``user_agent.original`` -> ``http.useragent`` +- ``user.*`` -> ``usr.{id,name,email}`` +""" + +import gzip +import json +import math +import os +import re +import time +from datetime import datetime, timedelta, timezone +from email.utils import parsedate_to_datetime + +import requests + +from api.services import DATADOG_SITES, normalize_datadog_site + +from ..exceptions import ( + AdapterAuthError, + AdapterPermanentError, + AdapterRateLimitedError, + AdapterTransientError, +) +from .base import LogStreamAdapter, ShipResult + +REQUEST_TIMEOUT = (5, 30) # (connect, read) seconds + +# Per-process connection pool: chunks and retry attempts within one worker +# process reuse the TLS connection to the intake host. Keyed by pid — rq +# forks a work horse per job and pooled sockets must not cross the fork. +_session = None +_session_pid = None + + +def _get_session(): + global _session, _session_pid + pid = os.getpid() + if _session is None or _session_pid != pid: + _session = requests.Session() + _session_pid = pid + return _session + + +def _parse_retry_after(value): + """Retry-After is delay-seconds OR an HTTP-date (RFC 9110 §10.2.3 — + intermediary proxies emit dates). Unparseable values fall back to None, + which lets the engine use its own backoff.""" + if not value: + return None + try: + parsed = float(value) + except (TypeError, ValueError): + parsed = None + if parsed is not None: + # nan/inf/negatives break the sleep arithmetic; 0.0 -> engine backoff. + if not math.isfinite(parsed): + return None + return max(0.0, parsed) + try: + parsed = parsedate_to_datetime(value) + except (TypeError, ValueError): + return None + if parsed is None: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return max(0.0, (parsed - datetime.now(timezone.utc)).total_seconds()) + + +def _tag_value(value): + """Sanitize a value for use in a Datadog tag — tags don't allow spaces or + most punctuation, and an org named "Acme Corp" must not break parsing.""" + sanitized = re.sub(r"[^a-z0-9\-_./:]+", "_", str(value).lower()) + return sanitized.strip("_:")[:200] + + +class DatadogAdapter(LogStreamAdapter): + id = "datadog" + name = "Datadog" + credentials_provider = "datadog" + # Datadog's intake accepts events with timestamps up to 18h in the past; + # older events are silently dropped after a 202. + max_event_age = timedelta(hours=18) + url_credential_keys = ("site",) + + def validate_options(self, options): + options = options or {} + validated = { + "service": str(options.get("service") or "phase-console")[:100], + "tags": str(options.get("tags") or "")[:500], + "gzip": bool(options.get("gzip", True)), + } + return validated + + def test(self, credentials, options, context): + """Validate the API key via Datadog's dedicated key-validation + endpoint — no log data is ingested.""" + self.validate_options(options) + _, site = self._intake_url(credentials) + + try: + response = _get_session().get( + f"https://api.{site}/api/v1/validate", + headers={"DD-API-KEY": credentials.get("api_key") or ""}, + timeout=REQUEST_TIMEOUT, + # Never follow a redirect carrying the DD-API-KEY header — + # requests only strips Authorization on cross-host redirects. + allow_redirects=False, + ) + except requests.RequestException as ex: + raise AdapterTransientError( + f"Could not reach Datadog: {ex}", + user_message="Could not reach Datadog", + ) from ex + + if response.status_code == 200: + return True, {"status_code": 200, "site": site} + if response.status_code in (401, 403): + raise AdapterAuthError( + f"Datadog rejected the credentials ({response.status_code})", + user_message="Datadog rejected the credentials", + status_code=response.status_code, + ) + raise AdapterTransientError( + f"Datadog returned {response.status_code}", + status_code=response.status_code, + ) + + def destination_url(self, credentials, options): + """Deep link to the Datadog Logs explorer filtered to Phase events.""" + site = normalize_datadog_site(credentials.get("site") or "datadoghq.com") + if site not in DATADOG_SITES: + return None + # Bare sites host the app on an app. subdomain; regional sites + # (us3.datadoghq.com etc.) are already the app host. + host = f"app.{site}" if site.count(".") == 1 else site + return f"https://{host}/logs?query=source%3Aphase" + + def _intake_url(self, credentials): + site = normalize_datadog_site(credentials.get("site") or "datadoghq.com") + if site not in DATADOG_SITES: + raise AdapterPermanentError( + f"Unknown Datadog site '{site}'", + user_message=f"Unknown Datadog site '{site}'. Expected one of: " + + ", ".join(DATADOG_SITES), + ) + return f"https://http-intake.logs.{site}/api/v2/logs", site + + def _to_datadog_event(self, envelope, options, context): + event = dict(envelope) + + client = event.pop("client", None) or {} + user_agent = event.pop("user_agent", None) or {} + user = event.pop("user", None) + + if client.get("address"): + event["network"] = {"client": {"ip": client["address"]}} + if user_agent.get("original"): + event["http"] = {"useragent": user_agent["original"]} + if user: + event["usr"] = { + "id": user.get("id", ""), + "name": user.get("name") or user.get("full_name") or "", + "email": user.get("email", ""), + } + + phase = event.get("phase", {}) + event_block = event.get("event", {}) + message = phase.get("description") or "{}.{}".format( + event_block.get("category", "event"), event_block.get("type", "unknown") + ) + + tags = [ + "phase_org:{}".format(_tag_value(context.get("organisation_name", ""))), + "phase_stream:{}".format(_tag_value(context.get("stream_name", ""))), + ] + if options.get("tags"): + tags.append(options["tags"]) + + event["ddsource"] = "phase" + event["service"] = options.get("service") or "phase-console" + event["ddtags"] = ",".join(t for t in tags if t and not t.endswith(":")) + event["message"] = message + return event + + def ship(self, chunk, credentials, options, context): + options = self.validate_options(options) + url, site = self._intake_url(credentials) + api_key = credentials.get("api_key") or "" + + body = json.dumps( + [self._to_datadog_event(envelope, options, context) for envelope in chunk], + default=str, + ).encode("utf-8") + + headers = { + "DD-API-KEY": api_key, + "Content-Type": "application/json", + } + if options["gzip"]: + body = gzip.compress(body) + headers["Content-Encoding"] = "gzip" + + started = time.monotonic() + try: + # allow_redirects=False: following a 3xx would convert the POST + # to a body-less GET (silent data loss behind an intercepting + # proxy) and re-send DD-API-KEY to the redirect target. + response = _get_session().post( + url, + data=body, + headers=headers, + timeout=REQUEST_TIMEOUT, + allow_redirects=False, + ) + except requests.RequestException as ex: + raise AdapterTransientError( + f"Could not reach Datadog intake: {ex}", + user_message="Could not reach the Datadog intake endpoint", + ) from ex + duration_ms = int((time.monotonic() - started) * 1000) + + status = response.status_code + # Datadog's v2 intake acknowledges with 202 only. Anything else 2xx + # (or a redirect) means an intermediary answered, not the intake — + # treating it as delivered would advance the cursor over events + # Datadog never ingested. Retrying is safe: at-least-once, consumers + # dedupe on @event.id. + if status == 202: + return ShipResult( + status_code=status, + duration_ms=duration_ms, + meta={"site": site, "events": len(chunk)}, + ) + if 300 <= status < 400: + raise AdapterTransientError( + f"Datadog intake returned an unexpected redirect ({status})", + user_message="The Datadog intake endpoint returned an unexpected redirect", + status_code=status, + ) + if status in (401, 403): + raise AdapterAuthError( + f"Datadog rejected the credentials ({status})", + user_message="Datadog rejected the credentials", + status_code=status, + ) + if status in (408, 429): + raise AdapterRateLimitedError( + f"Datadog intake throttled the request ({status})", + retry_after=_parse_retry_after(response.headers.get("Retry-After")), + status_code=status, + ) + if status in (400, 413): + raise AdapterPermanentError( + f"Datadog rejected the payload ({status}): {response.text[:200]}", + user_message=f"Datadog rejected the payload ({status})", + status_code=status, + ) + raise AdapterTransientError( + f"Datadog intake returned {status}", + status_code=status, + ) diff --git a/backend/ee/integrations/logs/streams/chunker.py b/backend/ee/integrations/logs/streams/chunker.py new file mode 100644 index 000000000..ee91efec0 --- /dev/null +++ b/backend/ee/integrations/logs/streams/chunker.py @@ -0,0 +1,111 @@ +"""Split serialized envelopes into destination-sized chunks. + +Limits stay comfortably under Datadog's intake caps (1000 events / 5 MB per +payload, 1 MB per event) so that adapter-added reserved fields never push a +chunk over the wire limit. +""" + +import json +from dataclasses import dataclass, field + +CHUNK_MAX_EVENTS = 500 +CHUNK_MAX_BYTES = 2_500_000 +EVENT_MAX_BYTES = 900_000 + + +@dataclass +class Chunk: + events: list = field(default_factory=list) + byte_size: int = 0 + cursor_from: object = None + cursor_to: object = None + # Id bounds of the covered range — (timestamp, id) is the event order, + # so timestamp containment alone can't distinguish two chunks that split + # inside a single timestamp. + cursor_from_id: str = "" + cursor_to_id: str = "" + last_cursor: dict = None + + +def _envelope_size(envelope): + return len(json.dumps(envelope, default=str).encode("utf-8")) + + +def _bound_envelope(envelope): + """Cap a single envelope's size; oversize payloads live in the JSON + metadata fields, so truncate those rather than dropping the event.""" + size = _envelope_size(envelope) + if size <= EVENT_MAX_BYTES: + return envelope, size + + phase = envelope.get("phase", {}) + for key in ("old_values", "new_values"): + if phase.get(key) is not None: + phase[key] = {"truncated": True} + size = _envelope_size(envelope) + if size <= EVENT_MAX_BYTES: + return envelope, size + + if "metadata" in phase.get("resource", {}): + phase["resource"]["metadata"] = {"truncated": True} + size = _envelope_size(envelope) + if size <= EVENT_MAX_BYTES: + return envelope, size + + # Last resort: strip to the identifying core. EVENT_MAX_BYTES leaves + # 100KB of headroom under Datadog's 1MB wire limit for adapter-added + # fields (message/ddtags/remaps), but that guarantee only holds if the + # envelope itself is genuinely bounded — a pathological event must not + # be allowed to permanently 413 its chunk. + slim = { + "schema_version": envelope.get("schema_version"), + "event": envelope.get("event"), + "timestamp": envelope.get("timestamp"), + "actor": envelope.get("actor"), + "phase": { + "organisation": phase.get("organisation"), + "description": str(phase.get("description", ""))[:2000], + "truncated": True, + }, + } + envelope.clear() + envelope.update(slim) + return envelope, _envelope_size(envelope) + + +def chunk_envelopes(entries): + """Group entries into ordered chunks. + + `entries` is a list of dicts: {"envelope": dict, "cursor": dict, + "timestamp": datetime} — one per event, already in (timestamp, id) order. + Each chunk records the timestamp range it covers and the cursor of its + last event, which becomes the stream cursor once the chunk is delivered. + """ + chunks = [] + current = None + + for entry in entries: + envelope, size = _bound_envelope(entry["envelope"]) + + if current is not None and ( + len(current.events) >= CHUNK_MAX_EVENTS + or current.byte_size + size > CHUNK_MAX_BYTES + ): + chunks.append(current) + current = None + + if current is None: + current = Chunk() + current.cursor_from = entry["timestamp"] + current.cursor_from_id = entry["cursor"].get("id", "") + + current.events.append(envelope) + current.byte_size += size + current.cursor_to = entry["timestamp"] + current.cursor_to_id = entry["cursor"].get("id", "") + current.last_cursor = entry["cursor"] + + if current is not None and current.events: + chunks.append(current) + + return chunks diff --git a/backend/ee/integrations/logs/streams/engine.py b/backend/ee/integrations/logs/streams/engine.py new file mode 100644 index 000000000..0b4c6fcd6 --- /dev/null +++ b/backend/ee/integrations/logs/streams/engine.py @@ -0,0 +1,1310 @@ +"""Log stream shipping engine. + +A recurring sweep (rq-scheduler, see jobs.py) enqueues one ship job per +active stream onto the dedicated ``log-streams`` queue. A ship job tails each +of the stream's event sources from its stored cursor, serializes events into +neutral envelopes, chunks them, and delivers chunk-by-chunk through the +stream's adapter. + +Delivery contract (at-least-once): + +- A source cursor advances only after its chunk is accepted (or after the + chunk exhausts its retry budget — the failed range is recorded and remains + manually re-shippable, so the stream never head-of-line blocks). +- Auth failures pause the stream (``paused_reason="auth_error"``); retrying + with dead credentials is pointless. +- Cursors older than the adapter's ``max_event_age`` are floored (the + destination would silently drop the backlog anyway); the skipped range is + recorded as a SKIPPED delivery event. +""" + +import logging +import time +from datetime import timedelta +from uuid import uuid4 + +import django_rq +from django.apps import apps +from django.utils import timezone +from django.utils.dateparse import parse_datetime +from django_rq import job +from rq.exceptions import NoSuchJobError +from rq.job import Job +from rq.timeouts import JobTimeoutException + +from api.utils.syncing.auth import get_credentials +from backend.quotas import can_use_log_streams + +from .adapters import all_adapters, get_adapter +from .chunker import CHUNK_MAX_EVENTS, chunk_envelopes +from .exceptions import ( + AdapterAuthError, + AdapterError, + AdapterPermanentError, + AdapterRateLimitedError, + AdapterTransientError, +) +from .sources import get_source + +logger = logging.getLogger(__name__) + +QUEUE_NAME = "log-streams" +SWEEP_INTERVAL_SECONDS = 30 +SHIP_JOB_TIMEOUT = 3600 +RETRY_JOB_TIMEOUT = 1800 +RETRY_BACKOFF_SECONDS = (5, 15, 60, 120, 300) +RETRY_SLEEP_CAP = 300 +MAX_TAIL_LOOPS = 20 +MAX_ATTEMPTS_CAP = 10 +# Manual retries re-materialize a recorded range in one go. Failed ranges are +# at most one chunk, so this cap only trips on pathological rows — which get +# an honest "range_too_large" failure instead of a silent partial resolve. +RETRY_MAX_EVENTS = 10 * CHUNK_MAX_EVENTS +# One chunk's retry ladder is wall-clock bounded: Retry-After sleeps capped +# at RETRY_SLEEP_CAP per gap could otherwise stretch ten attempts to ~51 +# minutes. +DELIVERY_DEADLINE_SECONDS = 30 * 60 +# Headroom between "we will still try to deliver this" and the destination's +# hard age cutoff. Every chunk is delivered directly after a fresh +# skip-ahead floor (one chunk per tail iteration), so this only has to +# exceed ONE deadline-bounded delivery cycle — otherwise events admitted for +# delivery can age past the cutoff mid-flight and be accepted-then-dropped +# by the destination while recorded as delivered. +SKIP_AHEAD_MARGIN = timedelta(minutes=40) + +# The accepted-then-dropped guard, as arithmetic: one chunk cycle (ship +# path) and one whole manual-retry job must both fit inside the margin with +# request-time slack. Also pinned by tests. +assert DELIVERY_DEADLINE_SECONDS + 300 <= SKIP_AHEAD_MARGIN.total_seconds() +assert RETRY_JOB_TIMEOUT + 300 <= SKIP_AHEAD_MARGIN.total_seconds() +DELIVERY_RETENTION_DAYS = 30 +# Rows outside the ingestion window are unretryable — auto-resolved after a +# grace so the loss is visible on the badge first (skips are born expired). +# Must exceed every adapter's (window - margin + retry lifetime) so a row is +# never resolved while a retry that passed the window check is still running. +EXPIRED_RESOLVE_GRACE = timedelta(hours=24) +assert all( + EXPIRED_RESOLVE_GRACE + >= a.max_event_age - SKIP_AHEAD_MARGIN + timedelta(seconds=RETRY_JOB_TIMEOUT) + for a in all_adapters() + if a.max_event_age +) +# Pending events older than this when a ship job runs indicate the schedule +# stopped firing (host sleep, dead scheduler) — logged for operators; users +# just see the stream's "Delayed" state. +LATE_DELIVERY_WARN_SECONDS = 300 + +# Delivery event statuses (mirror api.models.LogStreamDeliveryEvent) +STATUS_COMPLETED = "completed" +STATUS_FAILED = "failed" +STATUS_SKIPPED = "skipped" + +# _deliver_chunk outcomes +DELIVERED = "delivered" +AUTH_ERROR = "auth_error" +EXHAUSTED = "exhausted" +# The stream was paused, deleted or reconfigured mid-ladder — nothing was +# egressed for this chunk; stop without recording an outcome for it. +ABORTED = "aborted" + +# chunk-loop control +CONTINUE = "continue" +PAUSE = "pause" +# Stop shipping this source without advancing the cursor (e.g. a failure +# record could not be persisted — advancing would silently lose the range). +HALT = "halt" + + +def _queue(): + return django_rq.get_queue(QUEUE_NAME) + + +def _redis(): + return _queue().connection + + +def record_delivery(stream, source_id, status, **fields): + LogStreamDeliveryEvent = apps.get_model("api", "LogStreamDeliveryEvent") + # The out-of-sync badge counts unresolved rows, which must all be + # actionable. Stream-level rows (no source — nothing to re-ship) and + # FAILED rows that record a retry *attempt* (the original row remains the + # open item) are terminal outcomes: store them pre-resolved so they + # inform without inflating the badge forever. + if not source_id or ( + status == STATUS_FAILED and fields.get("retried_from") is not None + ): + fields.setdefault("resolved_at", timezone.now()) + try: + return LogStreamDeliveryEvent.objects.create( + stream=stream, + source=source_id, + status=status, + completed_at=timezone.now(), + **fields, + ) + except Exception: + logger.exception( + "Failed to record log stream delivery event", + extra={"stream_id": stream.id, "source": source_id, "status": status}, + ) + return None + + +def _cursor_timestamp(stream, source_id): + cursor = (stream.cursors or {}).get(source_id) + if cursor and cursor.get("ts"): + ts = cursor["ts"] + return parse_datetime(ts) if isinstance(ts, str) else ts + return stream.created_at or timezone.now() + + +def _get_cursor(stream, source_id): + cursor = (stream.cursors or {}).get(source_id) + if cursor and cursor.get("ts"): + return cursor + # Ship-forward-only: new streams start at their creation time. + start = stream.created_at or timezone.now() + return {"ts": start.isoformat(), "id": ""} + + +def _set_cursor(stream, source_id, cursor): + cursors = dict(stream.cursors or {}) + cursors[source_id] = cursor + stream.cursors = cursors + + +def _mark_healthy(stream): + stream.health = stream.HEALTHY + stream.last_failure_at = None + stream.last_failure_reason = "" + + +def _mark_degraded(stream, reason): + stream.health = stream.DEGRADED + stream.last_failure_at = timezone.now() + stream.last_failure_reason = str(reason)[:1024] + + +# Fields the (per-stream serialized) ship path owns. Lifecycle fields +# (is_active, paused_reason) are deliberately excluded: jobs hold a row +# loaded at job start, and writing lifecycle state from it would revert a +# pause/delete the user made while the job was running. +DELIVERY_STATE_FIELDS = [ + "cursors", + "health", + "last_shipped_at", + "last_failure_at", + "last_failure_reason", + "updated_at", +] + +HEALTH_FIELDS = ["health", "last_failure_at", "last_failure_reason", "updated_at"] + + +def _save_delivery_state(stream): + stream.save(update_fields=DELIVERY_STATE_FIELDS) + + +def _save_health(stream): + stream.save(update_fields=HEALTH_FIELDS) + + +def _pause_stream_row(stream, reason, failure_message): + """Pause + degrade via a targeted queryset update so a stale in-memory + row can never clobber cursors or other concurrently-written fields.""" + now = timezone.now() + LogStream = apps.get_model("api", "LogStream") + LogStream.objects.filter(id=stream.id).update( + is_active=False, + paused_reason=reason, + health=stream.DEGRADED, + last_failure_at=now, + last_failure_reason=str(failure_message)[:1024], + updated_at=now, + ) + # Keep the in-memory row consistent for the rest of the job. + stream.is_active = False + stream.paused_reason = reason + _mark_degraded(stream, failure_message) + + +def _stream_is_shippable(stream): + """Live DB check between delivery attempts: rq cannot stop a started + job, and the job's row predates anything the user did after it started. + + Halts when the stream was paused or deleted, when its configuration + changed (the job still holds the old sources/credentials/options — it + must exit and let the next sweep reload fresh state rather than keep + egressing with stale authority), or when the organisation lost the + Enterprise plan (mirrors quotas.can_use_log_streams, where plan is the + single source of truth).""" + LogStream = apps.get_model("api", "LogStream") + row = ( + LogStream.objects.filter( + id=stream.id, + is_active=True, + deleted_at__isnull=True, + organisation__plan="EN", + ) + .values("sources", "authentication_id", "options", "max_attempts") + .first() + ) + if row is None: + return False + return ( + row["sources"] == stream.sources + and row["authentication_id"] == stream.authentication_id + and row["options"] == stream.options + and row["max_attempts"] == stream.max_attempts + ) + + +def _backoff(attempt): + return RETRY_BACKOFF_SECONDS[min(attempt - 1, len(RETRY_BACKOFF_SECONDS) - 1)] + + +def _deliver_chunk(stream, chunk, adapter, credentials, options, context): + """Run the retry ladder for one chunk. + + Returns (outcome, attempts, info) where outcome is DELIVERED / AUTH_ERROR + / EXHAUSTED and info is the ShipResult (on success) or the last + AdapterError (on failure). + + The ladder is wall-clock bounded by DELIVERY_DEADLINE_SECONDS: sleeps + are honoured only while they fit inside the deadline. Without the bound, + a rate-limited destination sending large Retry-After values could + stretch one chunk past SKIP_AHEAD_MARGIN — its events would cross the + destination's age cutoff mid-ladder and be accepted-then-dropped while + recorded as delivered. + """ + max_attempts = max(1, min(stream.max_attempts or 1, MAX_ATTEMPTS_CAP)) + last_error = None + deadline = time.monotonic() + DELIVERY_DEADLINE_SECONDS + + for attempt in range(1, max_attempts + 1): + # A pause/delete/reconfiguration can land during a backoff sleep — + # re-check right before every egress attempt, not just per chunk. + if not _stream_is_shippable(stream): + return ABORTED, attempt - 1, None + delay = None + try: + result = adapter.ship(chunk.events, credentials, options, context) + return DELIVERED, attempt, result + except AdapterAuthError as ex: + return AUTH_ERROR, attempt, ex + except AdapterRateLimitedError as ex: + last_error = ex + delay = min( + ex.retry_after if ex.retry_after else _backoff(attempt), + RETRY_SLEEP_CAP, + ) + except AdapterPermanentError as ex: + last_error = ex + break + except AdapterTransientError as ex: + last_error = ex + delay = _backoff(attempt) + except JobTimeoutException: + raise + except Exception as ex: + # Adapters must raise typed errors (base.py contract); an untyped + # escape is an adapter bug. Map it onto the transient path so the + # ladder and the failure record still apply — crashing the job + # with the cursor held would head-of-line block the stream on a + # deterministic bug. + logger.exception( + "Log stream adapter raised an untyped exception", + extra={"stream_id": stream.id, "provider": stream.provider}, + ) + last_error = AdapterTransientError( + f"Adapter crashed: {type(ex).__name__}", + user_message="The delivery adapter failed unexpectedly", + ) + delay = _backoff(attempt) + + if attempt >= max_attempts or time.monotonic() + delay > deadline: + break + time.sleep(delay) + + return EXHAUSTED, attempt, last_error + + +def _error_meta(error): + meta = {"error": getattr(error, "user_message", str(error))} + status_code = getattr(error, "status_code", None) + if status_code: + meta["status_code"] = status_code + retry_after = getattr(error, "retry_after", None) + if retry_after: + meta["retry_after"] = retry_after + return meta + + +def _ship_chunk(stream, source_id, chunk, adapter, credentials, options, context): + outcome, attempts, info = _deliver_chunk( + stream, chunk, adapter, credentials, options, context + ) + + if outcome == ABORTED: + # Nothing was egressed and nothing failed — hold the cursor and stop + # the job; the next sweep reloads fresh state. + return PAUSE + + if outcome == DELIVERED: + _set_cursor(stream, source_id, chunk.last_cursor) + stream.last_shipped_at = timezone.now() + _mark_healthy(stream) + _save_delivery_state(stream) + auto_resolved = _resolve_covered_failures(stream, source_id, chunk) + meta = { + "status_code": info.status_code, + "duration_ms": info.duration_ms, + **info.meta, + } + if auto_resolved: + meta["auto_resolved"] = auto_resolved + record_delivery( + stream, + source_id, + STATUS_COMPLETED, + event_count=len(chunk.events), + payload_bytes=chunk.byte_size, + attempts=attempts, + cursor_from=chunk.cursor_from, + cursor_to=chunk.cursor_to, + cursor_from_id=chunk.cursor_from_id, + cursor_to_id=chunk.cursor_to_id, + meta=meta, + ) + return CONTINUE + + if outcome == AUTH_ERROR: + record_delivery( + stream, + source_id, + STATUS_FAILED, + event_count=len(chunk.events), + payload_bytes=chunk.byte_size, + attempts=attempts, + cursor_from=chunk.cursor_from, + cursor_to=chunk.cursor_to, + cursor_from_id=chunk.cursor_from_id, + cursor_to_id=chunk.cursor_to_id, + meta=_error_meta(info), + ) + _pause_stream_row( + stream, "auth_error", getattr(info, "user_message", "authentication failed") + ) + logger.warning( + "Log stream paused after auth failure", + extra={"stream_id": stream.id, "provider": stream.provider}, + ) + return PAUSE + + # EXHAUSTED / permanent: record the failed range, skip past it so newer + # events keep flowing. The range stays in Postgres and can be re-shipped + # from the delivery history. + record = record_delivery( + stream, + source_id, + STATUS_FAILED, + event_count=len(chunk.events), + payload_bytes=chunk.byte_size, + attempts=attempts, + cursor_from=chunk.cursor_from, + cursor_to=chunk.cursor_to, + cursor_from_id=chunk.cursor_from_id, + cursor_to_id=chunk.cursor_to_id, + meta=_error_meta(info), + ) + if record is None: + # The failed range couldn't be recorded — advancing the cursor now + # would lose these events with no re-shippable trace. Hold position; + # the next sweep retries from the same cursor. + _mark_degraded( + stream, "Delivery failed and the failure could not be recorded" + ) + _save_health(stream) + return HALT + _set_cursor(stream, source_id, chunk.last_cursor) + _mark_degraded(stream, getattr(info, "user_message", str(info))) + _save_delivery_state(stream) + return CONTINUE + + +def _resolve_covered_failures(stream, source_id, chunk): + """Mark unresolved failed/skipped delivery rows as resolved when a later + successful ship covers their event range. + + An auth failure holds the cursor, so after the credentials are fixed the + normal sweep re-ships the failed ranges automatically — without this, the + stale failure rows keep the out-of-sync badge up and invite a manual retry + that would double-ship the same events. + + Containment compares (timestamp, id) bounds, not timestamps alone: + chunks can split inside a single timestamp, and a timestamp-only match + would let a chunk resolve a failed row whose events it did NOT deliver. + Empty-string id bounds ("unknown": legacy rows, open floor boundaries) + compare leniently, preserving the timestamp behaviour for them. + """ + if chunk.cursor_from is None or chunk.cursor_to is None: + return 0 + from django.db.models import Q + + starts_inside = Q(cursor_from__gt=chunk.cursor_from) + if chunk.cursor_from_id: + starts_inside |= Q(cursor_from=chunk.cursor_from) & ( + Q(cursor_from_id__gte=chunk.cursor_from_id) | Q(cursor_from_id="") + ) + else: + starts_inside |= Q(cursor_from=chunk.cursor_from) + + ends_inside = Q(cursor_to__lt=chunk.cursor_to) + if chunk.cursor_to_id: + ends_inside |= Q(cursor_to=chunk.cursor_to) & ( + Q(cursor_to_id__lte=chunk.cursor_to_id) | Q(cursor_to_id="") + ) + else: + ends_inside |= Q(cursor_to=chunk.cursor_to) + + LogStreamDeliveryEvent = apps.get_model("api", "LogStreamDeliveryEvent") + try: + return ( + LogStreamDeliveryEvent.objects.filter( + stream=stream, + source=source_id, + status__in=[STATUS_FAILED, STATUS_SKIPPED], + resolved_at__isnull=True, + ) + .filter(starts_inside) + .filter(ends_inside) + .update(resolved_at=timezone.now()) + ) + except Exception: + logger.exception( + "Failed to auto-resolve covered delivery failures", + extra={"stream_id": stream.id, "source": source_id}, + ) + return 0 + + +def _ingestion_floor(adapter, now=None): + """Oldest timestamp the destination still accepts, plus safety margin. + Ranges below it are dropped, so the engine floors/skips past them and + rejects retries under it.""" + return (now or timezone.now()) - adapter.max_event_age + SKIP_AHEAD_MARGIN + + +def _skip_ahead(stream, source_id, source, adapter): + """Floor a stale cursor at the destination's max event age. + + Returns True when shipping may proceed. Returns False when the cursor is + stale but the skipped-range record could not be persisted — proceeding + would ship expired events the destination accepts-then-drops, advancing + the cursor past them with no durable trace of the loss. + """ + if not adapter.max_event_age: + return True + floor = _ingestion_floor(adapter) + cursor_ts = _cursor_timestamp(stream, source_id) + if cursor_ts >= floor: + return True + + cursor = _get_cursor(stream, source_id) + try: + skipped = source.count_before(stream.organisation, cursor, floor) + except Exception: + logger.exception( + "Failed to count skipped events", extra={"stream_id": stream.id} + ) + skipped = None + + if skipped is None or skipped: + meta = { + "reason": "max_event_age_exceeded", + "max_event_age_hours": adapter.max_event_age.total_seconds() / 3600, + } + if skipped is None: + meta["count_unknown"] = True + record = record_delivery( + stream, + source_id, + STATUS_SKIPPED, + event_count=skipped or 0, + cursor_from=cursor_ts, + cursor_to=floor, + cursor_from_id=cursor.get("id", ""), + # The floor is a computed boundary, not an event — open id bound. + cursor_to_id="", + meta=meta, + ) + if record is None: + # No trace of the loss — hold the cursor, halt this source, let + # the next sweep retry the recording. + return False + _set_cursor(stream, source_id, {"ts": floor.isoformat(), "id": ""}) + _save_delivery_state(stream) + return True + + +def _degrade_once(stream, error_code, reason): + """Degrade + record a stream-level failure only on transition — the sweep + fires every 30s and a persistent condition must not write a delivery row + per sweep.""" + if stream.health == stream.DEGRADED and stream.last_failure_reason == reason: + return + record_delivery(stream, "", STATUS_FAILED, meta={"error": error_code}) + _mark_degraded(stream, reason) + _save_health(stream) + + +def _ship_stream(stream): + try: + adapter = get_adapter(stream.provider) + except ValueError: + # A stream whose provider has no registered adapter can never ship — + # pause it visibly instead of crash-looping on every sweep. + record_delivery(stream, "", STATUS_FAILED, meta={"error": "unknown_provider"}) + _pause_stream_row( + stream, + "unknown_provider", + f"No adapter is registered for provider '{stream.provider}'", + ) + logger.error( + "Log stream paused: unknown provider", + extra={"stream_id": stream.id, "provider": stream.provider}, + ) + return + + if not stream.authentication_id: + _degrade_once( + stream, "credentials_missing", "Third-party credentials are missing" + ) + return + + try: + credentials = get_credentials(stream.authentication_id) + except Exception as ex: + _degrade_once( + stream, + f"credentials_unreadable: {type(ex).__name__}", + "Could not decrypt third-party credentials", + ) + return + + try: + options = adapter.validate_options(stream.options) + except Exception: + # Options are validated at create/update, so a failure here is a + # deterministic config error — pause instead of crash-looping. + record_delivery(stream, "", STATUS_FAILED, meta={"error": "invalid_options"}) + _pause_stream_row( + stream, "invalid_options", "The stream's configuration failed validation" + ) + logger.exception( + "Log stream paused: options failed validation", + extra={"stream_id": stream.id, "provider": stream.provider}, + ) + return + context = { + "organisation_name": stream.organisation.name, + "stream_name": stream.name, + } + + for source_id in list(stream.sources or []): + try: + source = get_source(source_id) + except ValueError: + logger.warning( + "Skipping unknown log stream source", + extra={"stream_id": stream.id, "source": source_id}, + ) + continue + + warned_late = False + for _ in range(MAX_TAIL_LOOPS): + # Re-floor every iteration, not just once per source: a slow + # chunk cycle (full retry ladder) can take long enough that the + # next fetch's oldest events have crossed the destination's age + # cutoff — they'd be accepted-and-dropped yet recorded delivered. + if not _skip_ahead(stream, source_id, source, adapter): + break + cursor = _get_cursor(stream, source_id) + events = source.fetch(stream.organisation, cursor, CHUNK_MAX_EVENTS) + if not events: + break + + # Operator signal only: a large pending age on the oldest fetched + # event means the recurring sweep hadn't fired for a while (host + # sleep, scheduler outage) or deliveries were failing long enough + # to back up. Derived from the fetch the loop already performs — + # no extra oldest-pending query per ship job. + if not warned_late: + warned_late = True + delay = int((timezone.now() - events[0].timestamp).total_seconds()) + if delay > LATE_DELIVERY_WARN_SECONDS: + logger.warning( + "Log stream deliveries are running %ss late — the sweep " + "schedule may have stalled or deliveries were backed up", + delay, + extra={"stream_id": stream.id, "source": source_id}, + ) + + entries = [ + { + "envelope": source.serialize(event, stream.organisation), + "cursor": source.cursor_of(event), + "timestamp": event.timestamp, + } + for event in events + ] + + # Ship ONE chunk per iteration. A fetch can byte-split into + # several chunks, and each delivery may leg through a full + # (deadline-bounded) retry ladder — shipping them all from this + # fetch would let the later chunks age past the ingestion floor + # computed above. Looping back re-floors and refetches from the + # advanced cursor instead. + chunks = chunk_envelopes(entries) + + # A console pause/delete/reconfiguration must take effect + # mid-job: rq cannot stop a started job, and this row predates + # the user's action. + if not _stream_is_shippable(stream): + return + result = _ship_chunk( + stream, source_id, chunks[0], adapter, credentials, options, context + ) + if result == PAUSE: + return + if result == HALT: + break + + if len(events) < CHUNK_MAX_EVENTS and len(chunks) == 1: + break + + +@job(QUEUE_NAME, timeout=SHIP_JOB_TIMEOUT) +def ship_log_stream(stream_id): + LogStream = apps.get_model("api", "LogStream") + stream = ( + LogStream.objects.filter(id=stream_id, deleted_at__isnull=True) + .select_related("organisation") + .first() + ) + if stream is None or not stream.is_active: + return + + # Authoritative overlap guard — the sweep's Job.fetch check is only a + # cheap first layer. HTTP delivery must never run under a DB transaction, + # so a Redis lock (TTL > job timeout) serializes ship jobs per stream. + conn = _redis() + lock_key = f"log_streams:ship:{stream.id}" + token = str(uuid4()) + if not conn.set(lock_key, token, nx=True, ex=SHIP_JOB_TIMEOUT + 60): + return + + try: + _ship_stream(stream) + except JobTimeoutException: + # _degrade_once: a deterministic, persistent failure re-runs every + # 30s sweep — record it on transition only, not per run. + _degrade_once(stream, "ship_job_timed_out", "Ship job timed out") + except Exception as ex: + logger.exception( + "Log stream ship job crashed", extra={"stream_id": stream.id} + ) + # Class name only — raw exception strings (driver/SQL internals) are + # user-visible via meta.error and last_failure_reason. + _degrade_once( + stream, + f"ship_job_crashed: {type(ex).__name__}", + f"Delivery failed unexpectedly ({type(ex).__name__}) — check the server logs", + ) + finally: + try: + if conn.get(lock_key) == token.encode(): + conn.delete(lock_key) + except Exception: + pass + + +def _manual_export_hint(source_id): + """Recovery guidance for a range the stream can no longer deliver. The + loss is destination-side only — the events remain queryable in the + Console (organisation audit logs / per-app secret logs). When the public + audit-logs REST route ships (currently disabled in urls.py), org_audit + ranges can point at a bulk export again.""" + return "The events remain available in the Phase Console's logs." + + +@job(QUEUE_NAME, timeout=RETRY_JOB_TIMEOUT) +def retry_delivery(delivery_event_id): + """Manually re-ship the event range covered by a failed/skipped delivery. + + Double-clicks and concurrent API calls can enqueue duplicate jobs before + any worker resolves the row — a per-delivery Redis claim (mirroring the + per-stream ship lock) serializes them so the same range isn't shipped + twice in parallel. + """ + conn = _redis() + claim_key = f"log_streams:retry:{delivery_event_id}" + claim_token = str(uuid4()) + if not conn.set(claim_key, claim_token, nx=True, ex=RETRY_JOB_TIMEOUT + 60): + return + try: + _retry_delivery_locked(delivery_event_id) + finally: + try: + if conn.get(claim_key) == claim_token.encode(): + conn.delete(claim_key) + except Exception: + pass + + +def _retry_delivery_locked(delivery_event_id): + """On success a linked COMPLETED delivery event is written and the + original is marked resolved. Stream cursors and lifecycle fields are + never written — the range is in the past relative to the live cursor, + and a concurrent ship job may have advanced state this job must not + clobber. + """ + LogStream = apps.get_model("api", "LogStream") + LogStreamDeliveryEvent = apps.get_model("api", "LogStreamDeliveryEvent") + original = ( + LogStreamDeliveryEvent.objects.filter(id=delivery_event_id) + .select_related("stream", "stream__organisation") + .first() + ) + if ( + original is None + or original.status not in (STATUS_FAILED, STATUS_SKIPPED) + or original.resolved_at is not None + ): + return + stream = original.stream + if stream.deleted_at is not None or not original.source: + return + # Pause means no egress — manual retries included. The mutation surfaces + # the user-facing error; this covers the direct-enqueue/queued-job path. + if not stream.is_active: + return + if original.cursor_from is None or original.cursor_to is None: + return + + try: + adapter = get_adapter(stream.provider) + source = get_source(original.source) + credentials = get_credentials(stream.authentication_id) + except Exception as ex: + record_delivery( + stream, + original.source, + STATUS_FAILED, + retried_from=original, + meta={"error": f"retry_setup_failed: {type(ex).__name__}"}, + ) + return + + # The destination silently discards events older than max_event_age + # (Datadog 202s, then drops) — shipping an expired range would falsely + # mark it recovered. Reject fully-expired ranges; for partially-expired + # ones, ship the live tail and record the expired head as skipped. + effective_from = original.cursor_from + expired_head = None + if adapter.max_event_age: + floor = _ingestion_floor(adapter) + if original.cursor_to < floor: + record_delivery( + stream, + original.source, + STATUS_FAILED, + retried_from=original, + cursor_from=original.cursor_from, + cursor_to=original.cursor_to, + meta={ + "error": "range_expired", + "detail": ( + "The destination no longer accepts events this old. " + + _manual_export_hint(original.source) + ), + }, + ) + return + if original.cursor_from < floor: + expired_head = (original.cursor_from, floor) + effective_from = floor + + total_events = 0 + total_bytes = 0 + total_attempts = 0 + try: + # Setup runs inside the recording guard too: the user already saw + # "retry queued", so a DB error in fetch_range or a job timeout while + # materializing a large range must still leave a FAILED trace. + options = adapter.validate_options(stream.options) + context = { + "organisation_name": stream.organisation.name, + "stream_name": stream.name, + } + + events = source.fetch_range( + stream.organisation, + effective_from, + original.cursor_to, + limit=RETRY_MAX_EVENTS + 1, + ) + if len(events) > RETRY_MAX_EVENTS: + record_delivery( + stream, + original.source, + STATUS_FAILED, + retried_from=original, + cursor_from=original.cursor_from, + cursor_to=original.cursor_to, + meta={ + "error": "range_too_large", + "detail": ( + f"More than {RETRY_MAX_EVENTS} events in this range. " + + _manual_export_hint(original.source) + ), + }, + ) + return + + entries = [ + { + "envelope": source.serialize(event, stream.organisation), + "cursor": source.cursor_of(event), + "timestamp": event.timestamp, + } + for event in events + ] + chunks = chunk_envelopes(entries) + + for chunk in chunks: + # Same live check as the ship path: a pause/delete/ + # reconfiguration issued while this retry runs must stop egress, + # and rq can't stop a started job. Record the aborted attempt — + # chunks may already have been egressed, and a silent exit would + # leave that duplication unexplained in the history. + if not _stream_is_shippable(stream): + record_delivery( + stream, + original.source, + STATUS_FAILED, + retried_from=original, + event_count=total_events, + payload_bytes=total_bytes, + cursor_from=original.cursor_from, + cursor_to=original.cursor_to, + meta={ + "error": "stream_changed_mid_retry", + "shipped_events": total_events, + }, + ) + return + outcome, attempts, info = _deliver_chunk( + stream, chunk, adapter, credentials, options, context + ) + total_attempts = max(total_attempts, attempts) + if outcome == ABORTED: + # Paused/reconfigured during this chunk's ladder — nothing + # from this chunk was egressed; record what already shipped. + record_delivery( + stream, + original.source, + STATUS_FAILED, + retried_from=original, + event_count=total_events, + payload_bytes=total_bytes, + cursor_from=original.cursor_from, + cursor_to=original.cursor_to, + meta={ + "error": "stream_changed_mid_retry", + "shipped_events": total_events, + }, + ) + return + if outcome != DELIVERED: + record_delivery( + stream, + original.source, + LogStreamDeliveryEvent.FAILED, + retried_from=original, + event_count=len(chunk.events), + payload_bytes=chunk.byte_size, + attempts=attempts, + cursor_from=original.cursor_from, + cursor_to=original.cursor_to, + meta=_error_meta(info), + ) + if outcome == AUTH_ERROR: + _pause_stream_row( + stream, + "auth_error", + getattr(info, "user_message", "authentication failed"), + ) + return + total_events += len(chunk.events) + total_bytes += chunk.byte_size + except JobTimeoutException: + # Without this record, the user saw "retry queued" and nothing would + # ever appear in the history. The original stays unresolved. + record_delivery( + stream, + original.source, + STATUS_FAILED, + retried_from=original, + cursor_from=original.cursor_from, + cursor_to=original.cursor_to, + meta={"error": "retry_job_timed_out"}, + ) + return + except Exception as ex: + logger.exception( + "Log stream delivery retry crashed", + extra={"stream_id": stream.id, "delivery_event_id": delivery_event_id}, + ) + record_delivery( + stream, + original.source, + STATUS_FAILED, + retried_from=original, + cursor_from=original.cursor_from, + cursor_to=original.cursor_to, + meta={"error": f"retry_failed: {type(ex).__name__}"}, + ) + return + + head_record = None + if expired_head: + # The head of the range fell outside the ingestion window before this + # retry ran — record it as its own skipped row so the loss stays + # visible after the original is resolved. + try: + head_count = source.count_before( + stream.organisation, + {"ts": expired_head[0].isoformat(), "id": ""}, + expired_head[1], + ) + except Exception: + head_count = None + head_meta = { + "reason": "max_event_age_exceeded", + "max_event_age_hours": adapter.max_event_age.total_seconds() / 3600, + } + if head_count is None: + head_meta["count_unknown"] = True + head_record = record_delivery( + stream, + original.source, + STATUS_SKIPPED, + retried_from=original, + event_count=head_count or 0, + cursor_from=expired_head[0], + cursor_to=expired_head[1], + cursor_from_id=original.cursor_from_id or "", + # The floor is a computed boundary, not an event — open id bound. + cursor_to_id="", + meta=head_meta, + ) + + record_delivery( + stream, + original.source, + STATUS_COMPLETED, + retried_from=original, + event_count=total_events, + payload_bytes=total_bytes, + attempts=total_attempts, + cursor_from=effective_from, + cursor_to=original.cursor_to, + cursor_from_id="" if expired_head else (original.cursor_from_id or ""), + cursor_to_id=original.cursor_to_id or "", + meta={"manual_retry": True} if events else {"manual_retry": True, "note": "no_events_in_range"}, + ) + if expired_head and head_record is None: + # The lost head has no durable record — keep the original open so the + # loss stays visible. A later retry may re-ship the tail; the + # at-least-once contract tolerates the duplication. + logger.warning( + "Skipped-head record could not be persisted; leaving the original " + "delivery unresolved", + extra={"stream_id": stream.id, "delivery_event_id": delivery_event_id}, + ) + return + original.resolved_at = timezone.now() + original.save(update_fields=["resolved_at"]) + LogStream.objects.filter(id=stream.id).update( + last_shipped_at=timezone.now(), updated_at=timezone.now() + ) + + +def sweep_log_streams(): + """Recurring sweep: enqueue a ship job for every shippable stream.""" + LogStream = apps.get_model("api", "LogStream") + queue = _queue() + + # Streams whose credential row was hard-deleted (the FK is SET_NULL) can + # never ship again — pause them visibly instead of leaving a permanently + # "healthy" stream that silently ships nothing. One-shot: the pause + # removes them from subsequent sweeps. + stranded = LogStream.objects.filter( + is_active=True, deleted_at__isnull=True, authentication__isnull=True + ) + for stream in stranded: + try: + record_delivery( + stream, "", STATUS_FAILED, meta={"error": "credentials_missing"} + ) + _pause_stream_row( + stream, "credentials_missing", "Third-party credentials were deleted" + ) + logger.warning( + "Log stream paused: its third-party credentials were deleted", + extra={"stream_id": stream.id}, + ) + except Exception: + logger.exception( + "Failed to pause credential-less log stream", + extra={"stream_id": stream.id}, + ) + + streams = ( + LogStream.objects.filter( + is_active=True, + deleted_at__isnull=True, + authentication__isnull=False, + ) + .select_related("organisation") + .order_by("created_at") + ) + + starved_streams = 0 + for stream in streams: + try: + if not can_use_log_streams(stream.organisation): + continue + + # Cheap overlap layer: skip if the last ship job is still alive. + # Still *queued* after a full sweep interval means no worker ever + # picked it up — the pool is saturated, not the destination slow. + if stream.ship_job_id: + try: + last_job = Job.fetch(stream.ship_job_id, connection=queue.connection) + if last_job.is_queued: + starved_streams += 1 + continue + if last_job.is_started: + continue + except NoSuchJobError: + pass + + new_job = ship_log_stream.delay(stream.id) + stream.ship_job_id = new_job.get_id() + stream.save(update_fields=["ship_job_id", "updated_at"]) + except Exception: + logger.exception( + "Failed to enqueue log stream ship job", + extra={"stream_id": stream.id}, + ) + + if starved_streams: + logger.warning( + "%s log stream ship job(s) from the previous sweep are still " + "waiting for a worker — the log-streams pool is saturated; " + "raise LOG_STREAM_WORKERS to add delivery capacity", + starved_streams, + ) + + _resolve_expired_failures() + _cleanup_delivery_events() + + +def _resolve_expired_failures(): + """Resolve unresolved failed/skipped rows that can no longer be re-shipped. + + Once a row's whole range is older than its destination's ingestion + window, the retry mutation rejects it as range_expired — leaving it + unresolved would pin the out-of-sync badge forever and exempt the row + from retention indefinitely. The row itself survives (resolved, meta + resolution="expired") as the durable record of the loss until retention + prunes it. Runs every sweep: the unresolved set is empty on healthy + streams and served by the partial index.""" + LogStreamDeliveryEvent = apps.get_model("api", "LogStreamDeliveryEvent") + now = timezone.now() + try: + rows = list( + LogStreamDeliveryEvent.objects.filter( + status__in=[STATUS_FAILED, STATUS_SKIPPED], + resolved_at__isnull=True, + cursor_to__isnull=False, + created_at__lt=now - EXPIRED_RESOLVE_GRACE, + stream__deleted_at__isnull=True, + ) + .exclude(source="") + .select_related("stream") + ) + except Exception: + logger.exception("Failed to load unresolved delivery rows for expiry") + return + + for row in rows: + try: + adapter = get_adapter(row.stream.provider) + except ValueError: + continue + if not adapter.max_event_age: + continue + if row.cursor_to >= _ingestion_floor(adapter, now): + continue + try: + meta = dict(row.meta or {}) + meta["resolution"] = "expired" + # Conditional: never clobber a concurrently-written resolution. + LogStreamDeliveryEvent.objects.filter( + id=row.id, resolved_at__isnull=True + ).update(resolved_at=now, meta=meta) + except Exception: + logger.exception( + "Failed to resolve expired delivery row", + extra={"stream_id": row.stream_id, "delivery_event_id": row.id}, + ) + + +CLEANUP_BATCH_SIZE = 5000 +CLEANUP_MARKER_KEY = "log_streams:cleanup_marker" +# Claim TTL while the prune runs; extended to a day only on success, so a +# crash or job timeout retries within the hour instead of skipping a day +# (a skipped day compounds — the next run has a bigger backlog). +CLEANUP_CLAIM_SECONDS = 3600 + + +def _cleanup_delivery_events(): + """Prune old delivery history ~once a day. Unresolved failed/skipped rows + with an event range are kept — they are the out-of-sync record and stay + re-shippable. Stream-level failure rows (no source) aren't retryable, so + they age out normally.""" + from django.db.models import Q + + try: + conn = _redis() + if not conn.set(CLEANUP_MARKER_KEY, "1", nx=True, ex=CLEANUP_CLAIM_SECONDS): + return + except Exception: + return + + LogStreamDeliveryEvent = apps.get_model("api", "LogStreamDeliveryEvent") + cutoff = timezone.now() - timedelta(days=DELIVERY_RETENTION_DAYS) + # The unresolved-row exemption protects re-shippable ranges, which only + # exist on LIVE streams. Deleted streams' rows age out unconditionally — + # LogStream.delete resolves them, but an in-flight ship job can record + # one more failure after that pass, and nothing else ever resolves rows + # of deleted streams. + prunable = LogStreamDeliveryEvent.objects.filter(created_at__lt=cutoff).exclude( + Q(status__in=[STATUS_FAILED, STATUS_SKIPPED]) + & Q(resolved_at__isnull=True) + & ~Q(source="") + & Q(stream__deleted_at__isnull=True) + ) + try: + # Batched: one unbounded DELETE (plus the retried_from SET_NULL + # collector) can outlive the sweep job's timeout on a large backlog. + # Each batch commits independently, so an interrupted prune keeps its + # progress and the retried claim finishes the remainder. + while True: + batch = list(prunable.values_list("id", flat=True)[:CLEANUP_BATCH_SIZE]) + if not batch: + break + LogStreamDeliveryEvent.objects.filter(id__in=batch).delete() + except Exception: + logger.exception("Failed to prune log stream delivery events") + return + + try: + conn.set(CLEANUP_MARKER_KEY, "1", ex=86400) + except Exception: + pass + + +def lag_for(stream, source_id): + """Delivery delay in seconds: the age of the oldest event still waiting + to ship, 0 when caught up. + + Deliberately NOT cursor distance — after an idle gap, a single fresh + event would make cursor distance spike to the length of the gap for one + sweep interval, reading as a phantom multi-hour stall.""" + try: + source = get_source(source_id) + except ValueError: + return 0 + pending = source.oldest_pending_timestamp( + stream.organisation, _get_cursor(stream, source_id) + ) + if pending is None: + return 0 + return max(0, int((timezone.now() - pending).total_seconds())) + + +def pause(stream, reason=""): + stream.is_active = False + stream.paused_reason = reason + stream.save(update_fields=["is_active", "paused_reason", "updated_at"]) + cancel_ship_job(stream) + + +def resume(stream): + """Reactivate a paused stream; shipping continues from the stored cursor + on the next sweep (subject to the max-event-age floor).""" + stream.is_active = True + stream.paused_reason = "" + stream.save(update_fields=["is_active", "paused_reason", "updated_at"]) + + +def cancel_ship_job(stream): + if not stream.ship_job_id: + return + queue = _queue() + try: + rq_job = Job.fetch(stream.ship_job_id, connection=queue.connection) + if rq_job.is_queued or rq_job.is_started: + rq_job.cancel() + queue.remove(stream.ship_job_id) + except NoSuchJobError: + pass + except Exception: + logger.debug( + "Could not cancel ship job %s", stream.ship_job_id, exc_info=True + ) + + +def test_adapter_connection(provider_id, credential_id, options, organisation): + """Synchronous connection test used by the TestLogStreamConnection + mutation. Returns (ok, message).""" + try: + adapter = get_adapter(provider_id) + except ValueError as ex: + return False, str(ex) + + try: + credentials = get_credentials(credential_id) + except Exception: + return False, "Could not read the selected credentials" + + try: + options = adapter.validate_options(options) + adapter.test( + credentials, + options, + {"organisation_name": organisation.name, "stream_name": "connection-test"}, + ) + return True, "Connection successful" + except AdapterError as ex: + return False, ex.user_message + except Exception: + logger.exception( + "Log stream connection test crashed", + extra={"provider": provider_id, "credential_id": credential_id}, + ) + return False, "The connection test failed unexpectedly" diff --git a/backend/ee/integrations/logs/streams/exceptions.py b/backend/ee/integrations/logs/streams/exceptions.py new file mode 100644 index 000000000..da0fe9b2b --- /dev/null +++ b/backend/ee/integrations/logs/streams/exceptions.py @@ -0,0 +1,36 @@ +"""Typed errors raised by log stream adapters. + +The engine maps each class to a distinct delivery behaviour: + +- AdapterAuthError -> pause the stream (retrying is pointless) +- AdapterRateLimitedError -> sleep (Retry-After if given) and retry +- AdapterTransientError -> exponential backoff and retry +- AdapterPermanentError -> fail the chunk immediately, no retry +""" + + +class AdapterError(Exception): + def __init__(self, message, *, user_message=None, status_code=None): + super().__init__(message) + self.user_message = user_message or message + self.status_code = status_code + + +class AdapterAuthError(AdapterError): + """The destination rejected our credentials (401/403).""" + + +class AdapterRateLimitedError(AdapterError): + """The destination is throttling us (408/429).""" + + def __init__(self, message, *, retry_after=None, **kwargs): + super().__init__(message, **kwargs) + self.retry_after = retry_after + + +class AdapterTransientError(AdapterError): + """A retryable failure: 5xx, timeout, connection error.""" + + +class AdapterPermanentError(AdapterError): + """A non-retryable failure: malformed payload, unknown site, 400/413.""" diff --git a/backend/ee/integrations/logs/streams/graphene/__init__.py b/backend/ee/integrations/logs/streams/graphene/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/ee/integrations/logs/streams/graphene/mutations.py b/backend/ee/integrations/logs/streams/graphene/mutations.py new file mode 100644 index 000000000..e89fbc87e --- /dev/null +++ b/backend/ee/integrations/logs/streams/graphene/mutations.py @@ -0,0 +1,447 @@ +import graphene +from django.utils import timezone +from graphql import GraphQLError + +from api.models import ( + AuditEvent, + LogStream, + LogStreamDeliveryEvent, + Organisation, + ProviderCredentials, +) +from api.utils.access.permissions import user_has_permission +from api.utils.audit_logging import get_actor_info_from_graphql, log_audit_event +from api.utils.rest import get_resolver_request_meta +from backend.quotas import can_use_log_streams + +from .. import engine +from ..adapters import get_adapter +from ..sources import get_source +from .queries import user_has_global_access +from .types import LogStreamType + +PLAN_ERROR = "Log Streams require an Enterprise plan." + + +def _check_plan(org): + if not can_use_log_streams(org): + raise GraphQLError(PLAN_ERROR) + + +def _check_permission(info, action, org): + if not user_has_permission(info.context.user, action, "LogStreams", org): + raise GraphQLError("You don't have permission to manage Log Streams") + # Streams export org-wide activity — a scoped custom role holding + # LogStreams permissions must not configure org-wide egress. + if not user_has_global_access(info.context.user, org): + raise GraphQLError("Managing Log Streams requires a role with global access") + + +def _validate_stream_input(org, provider, credential_id, sources, max_attempts): + try: + adapter = get_adapter(provider) + except ValueError as ex: + raise GraphQLError(str(ex)) + + if not sources: + raise GraphQLError("Select at least one event source to stream") + for source_id in sources: + try: + get_source(source_id) + except ValueError as ex: + raise GraphQLError(str(ex)) + + try: + credential = ProviderCredentials.objects.get( + id=credential_id, deleted_at=None + ) + except ProviderCredentials.DoesNotExist: + raise GraphQLError("The selected credentials don't exist") + if credential.organisation_id != org.id: + raise GraphQLError("The selected credentials don't exist") + if credential.provider != adapter.credentials_provider: + raise GraphQLError( + f"{adapter.name} log streams require {adapter.credentials_provider} credentials" + ) + + max_attempts = max(1, min(int(max_attempts or 5), engine.MAX_ATTEMPTS_CAP)) + + return adapter, credential, max_attempts + + +def _build_options(adapter, service, tags, gzip): + return adapter.validate_options( + {"service": service, "tags": tags, "gzip": True if gzip is None else gzip} + ) + + +def _stream_values(stream): + return { + "name": stream.name, + "provider": stream.provider, + "credential_id": str(stream.authentication_id), + "sources": list(stream.sources or []), + "options": stream.options or {}, + "max_attempts": stream.max_attempts, + } + + +def _audit(info, org, event_type, stream, old_values=None, new_values=None, description=""): + try: + actor_type, actor_id, actor_metadata = get_actor_info_from_graphql(info, org) + ip_address, user_agent = get_resolver_request_meta(info.context) + log_audit_event( + organisation=org, + event_type=event_type, + resource_type=AuditEvent.LOG_STREAM, + resource_id=stream.id, + actor_type=actor_type, + actor_id=actor_id, + actor_metadata=actor_metadata, + resource_metadata={"name": stream.name, "provider": stream.provider}, + old_values=old_values, + new_values=new_values, + description=description, + ip_address=ip_address, + user_agent=user_agent, + ) + except Exception: + pass + + +class CreateLogStreamMutation(graphene.Mutation): + class Arguments: + organisation_id = graphene.ID(required=True) + name = graphene.String(required=True) + provider = graphene.String(required=True) + credential_id = graphene.ID(required=True) + sources = graphene.List(graphene.NonNull(graphene.String), required=True) + service = graphene.String(required=False) + tags = graphene.String(required=False) + gzip = graphene.Boolean(required=False) + max_attempts = graphene.Int(required=False) + + log_stream = graphene.Field(LogStreamType) + + @classmethod + def mutate( + cls, + root, + info, + organisation_id, + name, + provider, + credential_id, + sources, + service=None, + tags=None, + gzip=None, + max_attempts=None, + ): + org = Organisation.objects.get(id=organisation_id) + _check_permission(info, "create", org) + _check_plan(org) + + adapter, credential, max_attempts = _validate_stream_input( + org, provider, credential_id, sources, max_attempts + ) + + name = name.strip() + if not name: + raise GraphQLError("Please enter a name for this Log Stream") + + stream = LogStream.objects.create( + organisation=org, + name=name[:64], + provider=adapter.id, + authentication=credential, + sources=list(dict.fromkeys(sources)), + options=_build_options(adapter, service, tags, gzip), + max_attempts=max_attempts, + ) + + _audit( + info, + org, + AuditEvent.CREATE, + stream, + new_values=_stream_values(stream), + description=f"Created log stream {stream.name}", + ) + + return CreateLogStreamMutation(log_stream=stream) + + +class UpdateLogStreamMutation(graphene.Mutation): + class Arguments: + stream_id = graphene.ID(required=True) + name = graphene.String(required=True) + credential_id = graphene.ID(required=True) + sources = graphene.List(graphene.NonNull(graphene.String), required=True) + service = graphene.String(required=False) + tags = graphene.String(required=False) + gzip = graphene.Boolean(required=False) + max_attempts = graphene.Int(required=False) + + log_stream = graphene.Field(LogStreamType) + + @classmethod + def mutate( + cls, + root, + info, + stream_id, + name, + credential_id, + sources, + service=None, + tags=None, + gzip=None, + max_attempts=None, + ): + stream = LogStream.objects.get(id=stream_id, deleted_at=None) + # Organisation is always derived from the stream, never client input. + org = stream.organisation + _check_permission(info, "update", org) + _check_plan(org) + + adapter, credential, max_attempts = _validate_stream_input( + org, stream.provider, credential_id, sources, max_attempts + ) + + name = name.strip() + if not name: + raise GraphQLError("Please enter a name for this Log Stream") + + old_values = _stream_values(stream) + + stream.name = name[:64] + stream.authentication = credential + stream.sources = list(dict.fromkeys(sources)) + stream.options = _build_options(adapter, service, tags, gzip) + stream.max_attempts = max_attempts + # Config fields only — a full save would write cursors/health/activity + # from this (possibly stale) row and clobber concurrent worker state. + stream.save( + update_fields=[ + "name", + "authentication", + "sources", + "options", + "max_attempts", + "updated_at", + ] + ) + + _audit( + info, + org, + AuditEvent.UPDATE, + stream, + old_values=old_values, + new_values=_stream_values(stream), + description=f"Updated log stream {stream.name}", + ) + + return UpdateLogStreamMutation(log_stream=stream) + + +class ToggleLogStreamMutation(graphene.Mutation): + class Arguments: + stream_id = graphene.ID(required=True) + + log_stream = graphene.Field(LogStreamType) + + @classmethod + def mutate(cls, root, info, stream_id): + stream = LogStream.objects.get(id=stream_id, deleted_at=None) + org = stream.organisation + _check_permission(info, "update", org) + + if stream.is_active: + # Pause is deliberately NOT plan-gated: a downgraded org must be + # able to stop its streams (mirrors delete). Resume re-enables + # egress, so it stays gated below. + engine.pause(stream) + description = f"Paused log stream {stream.name}" + else: + _check_plan(org) + if not stream.authentication_id: + raise GraphQLError( + "This stream has no credentials — select new credentials before resuming" + ) + engine.resume(stream) + description = f"Resumed log stream {stream.name}" + + _audit(info, org, AuditEvent.UPDATE, stream, description=description) + + return ToggleLogStreamMutation(log_stream=stream) + + +class DeleteLogStreamMutation(graphene.Mutation): + class Arguments: + stream_id = graphene.ID(required=True) + + ok = graphene.Boolean() + + @classmethod + def mutate(cls, root, info, stream_id): + stream = LogStream.objects.get(id=stream_id, deleted_at=None) + org = stream.organisation + _check_permission(info, "delete", org) + + stream.delete() + + _audit( + info, + org, + AuditEvent.DELETE, + stream, + old_values=_stream_values(stream), + description=f"Deleted log stream {stream.name}", + ) + + return DeleteLogStreamMutation(ok=True) + + +class TestLogStreamConnectionMutation(graphene.Mutation): + class Arguments: + organisation_id = graphene.ID(required=True) + provider = graphene.String(required=True) + credential_id = graphene.ID(required=True) + service = graphene.String(required=False) + tags = graphene.String(required=False) + gzip = graphene.Boolean(required=False) + + ok = graphene.Boolean() + message = graphene.String() + + @classmethod + def mutate( + cls, + root, + info, + organisation_id, + provider, + credential_id, + service=None, + tags=None, + gzip=None, + ): + org = Organisation.objects.get(id=organisation_id) + if not ( + user_has_permission(info.context.user, "create", "LogStreams", org) + or user_has_permission(info.context.user, "update", "LogStreams", org) + ): + raise GraphQLError("You don't have permission to manage Log Streams") + if not user_has_global_access(info.context.user, org): + raise GraphQLError( + "Managing Log Streams requires a role with global access" + ) + _check_plan(org) + + try: + adapter = get_adapter(provider) + except ValueError as ex: + raise GraphQLError(str(ex)) + + try: + credential = ProviderCredentials.objects.get( + id=credential_id, deleted_at=None + ) + except ProviderCredentials.DoesNotExist: + raise GraphQLError("The selected credentials don't exist") + if credential.organisation_id != org.id: + raise GraphQLError("The selected credentials don't exist") + # Same provider check as create/update — without it, another + # provider's api_key would be decrypted and sent to this adapter's + # destination as part of the test request. + if credential.provider != adapter.credentials_provider: + raise GraphQLError( + f"{adapter.name} log streams require {adapter.credentials_provider} credentials" + ) + + ok, message = engine.test_adapter_connection( + adapter.id, + credential.id, + {"service": service, "tags": tags, "gzip": True if gzip is None else gzip}, + org, + ) + + return TestLogStreamConnectionMutation(ok=ok, message=message) + + +class RetryLogStreamDeliveryMutation(graphene.Mutation): + class Arguments: + delivery_event_id = graphene.ID(required=True) + + ok = graphene.Boolean() + + @classmethod + def mutate(cls, root, info, delivery_event_id): + delivery_event = LogStreamDeliveryEvent.objects.select_related( + "stream", "stream__organisation" + ).get(id=delivery_event_id) + stream = delivery_event.stream + org = stream.organisation + _check_permission(info, "update", org) + _check_plan(org) + + if stream.deleted_at is not None: + raise GraphQLError("This log stream has been deleted") + if not stream.is_active: + raise GraphQLError( + "This stream is paused — resume it before retrying deliveries" + ) + if delivery_event.status not in ( + engine.STATUS_FAILED, + engine.STATUS_SKIPPED, + ): + raise GraphQLError("Only failed or skipped deliveries can be retried") + if delivery_event.resolved_at is not None: + raise GraphQLError("This delivery has already been resolved") + if delivery_event.cursor_from is None or delivery_event.cursor_to is None: + raise GraphQLError("This delivery has no event range to re-ship") + + try: + adapter = get_adapter(stream.provider) + except ValueError as ex: + raise GraphQLError(str(ex)) + # The destination silently discards events older than its ingestion + # window — re-shipping an expired range would falsely mark it + # recovered. (The engine re-checks; this is the user-facing error.) + if adapter.max_event_age and delivery_event.cursor_to < engine._ingestion_floor( + adapter + ): + raise GraphQLError( + "This range is older than the destination's ingestion window — " + "retried events would be silently discarded. " + + engine._manual_export_hint(delivery_event.source) + ) + + # The job holds an authoritative per-delivery Redis claim; this + # pre-check just gives double-clicks a friendly error instead of a + # silently discarded duplicate job. + try: + if engine._redis().exists(f"log_streams:retry:{delivery_event.id}"): + raise GraphQLError( + "A retry for this delivery is already running. If a worker " + "crashed mid-retry, the claim clears automatically within " + "about 30 minutes." + ) + except GraphQLError: + raise + except Exception: + pass + + engine.retry_delivery.delay(delivery_event.id) + + _audit( + info, + org, + AuditEvent.UPDATE, + stream, + description=f"Requested delivery retry for log stream {stream.name}", + ) + + return RetryLogStreamDeliveryMutation(ok=True) diff --git a/backend/ee/integrations/logs/streams/graphene/queries.py b/backend/ee/integrations/logs/streams/graphene/queries.py new file mode 100644 index 000000000..51c23c64c --- /dev/null +++ b/backend/ee/integrations/logs/streams/graphene/queries.py @@ -0,0 +1,95 @@ +from graphql import GraphQLError + +from api.models import LogStream, Organisation +from api.services import Providers +from api.utils.access.permissions import user_has_global_access, user_has_permission +from api.utils.database import get_approximate_count + +from ..adapters import all_adapters +from ..engine import STATUS_COMPLETED, STATUS_FAILED, STATUS_SKIPPED +from ..sources import all_sources +from .types import LogStreamDeliveryHistoryType + +DELIVERY_STATUS_FILTERS = (STATUS_COMPLETED, STATUS_FAILED, STATUS_SKIPPED, "unresolved") + + +# Log streams export the ENTIRE organisation's activity — both sources query +# org-wide, including apps the caller may not be a member of. Custom roles +# can hold LogStreams permissions without global access; such scoped roles +# must not view or configure org-wide egress (mirrors the audit REST +# endpoint's guard). `user_has_global_access` is imported above and re-used +# by the mutations module. + + +def resolve_log_stream_providers(root, info): + return [ + { + "id": adapter.id, + "name": adapter.name, + "credentials_provider": Providers.get_provider_config( + adapter.credentials_provider + ), + "max_event_age_hours": ( + adapter.max_event_age.total_seconds() / 3600 + if adapter.max_event_age + else None + ), + } + for adapter in all_adapters() + ] + + +def resolve_log_stream_sources(root, info): + return [ + {"id": source.id, "name": source.name, "description": source.description} + for source in all_sources() + ] + + +def resolve_log_streams(root, info, organisation_id): + org = Organisation.objects.get(id=organisation_id) + + if not user_has_permission(info.context.user, "read", "LogStreams", org): + return [] + if not user_has_global_access(info.context.user, org): + return [] + + # This query is polled by the console — select_related keeps the nested + # organisation/authentication resolvers from issuing per-stream queries. + return ( + LogStream.objects.filter(organisation=org, deleted_at=None) + .select_related("organisation", "authentication") + .order_by("-created_at") + ) + + +def resolve_log_stream_deliveries( + root, info, stream_id, limit=25, offset=0, status=None +): + stream = LogStream.objects.get(id=stream_id, deleted_at=None) + + if not user_has_permission( + info.context.user, "read", "LogStreams", stream.organisation + ): + raise GraphQLError("You don't have permission to view log stream deliveries") + if not user_has_global_access(info.context.user, stream.organisation): + raise GraphQLError("You don't have permission to view log stream deliveries") + + queryset = stream.delivery_events.all() + if status: + if status not in DELIVERY_STATUS_FILTERS: + raise GraphQLError(f"Invalid delivery status filter '{status}'") + if status == "unresolved": + queryset = queryset.filter( + status__in=[STATUS_FAILED, STATUS_SKIPPED], resolved_at__isnull=True + ) + else: + queryset = queryset.filter(status=status) + + limit = min(max(1, limit), 100) + offset = max(0, offset) + + count = get_approximate_count(queryset) + events = list(queryset[offset : offset + limit]) + + return LogStreamDeliveryHistoryType(events=events, count=count) diff --git a/backend/ee/integrations/logs/streams/graphene/types.py b/backend/ee/integrations/logs/streams/graphene/types.py new file mode 100644 index 000000000..87f925d87 --- /dev/null +++ b/backend/ee/integrations/logs/streams/graphene/types.py @@ -0,0 +1,163 @@ +import graphene +from django.utils import timezone +from graphene import ObjectType +from graphene_django import DjangoObjectType + +from api.models import LogStream, LogStreamDeliveryEvent +from api.services import Providers +from backend.graphene.types import ProviderType + +from ..adapters import get_adapter +from ..engine import STATUS_COMPLETED, STATUS_FAILED, STATUS_SKIPPED, lag_for +from ..sources import SOURCES + + +class LogStreamProviderType(ObjectType): + id = graphene.String(required=True) + name = graphene.String(required=True) + credentials_provider = graphene.Field(ProviderType) + max_event_age_hours = graphene.Float() + + +class LogStreamSourceType(ObjectType): + id = graphene.String(required=True) + name = graphene.String(required=True) + description = graphene.String(required=True) + + +class LogStreamSourceLagType(ObjectType): + source = graphene.String(required=True) + name = graphene.String(required=True) + lag_seconds = graphene.Int(required=True) + + +class LogStreamDeliverySummaryType(ObjectType): + """Delivery counts over the last 24 hours.""" + + completed = graphene.Int(required=True) + failed = graphene.Int(required=True) + + +class LogStreamDeliveryEventType(DjangoObjectType): + class Meta: + model = LogStreamDeliveryEvent + fields = ( + "id", + "source", + "status", + "event_count", + "payload_bytes", + "attempts", + "cursor_from", + "cursor_to", + "retried_from", + "resolved_at", + "meta", + "created_at", + "completed_at", + ) + + +class LogStreamDeliveryHistoryType(ObjectType): + events = graphene.List(LogStreamDeliveryEventType) + count = graphene.Int() + + +class LogStreamType(DjangoObjectType): + provider_info = graphene.Field(LogStreamProviderType) + sources = graphene.List(graphene.NonNull(graphene.String), required=True) + source_lags = graphene.List(graphene.NonNull(LogStreamSourceLagType), required=True) + unresolved_failures = graphene.Int(required=True) + delivery_summary = graphene.Field(LogStreamDeliverySummaryType) + destination_url = graphene.String() + + class Meta: + model = LogStream + # `cursors` and `ship_job_id` are engine-internal — never exposed. + fields = ( + "id", + "name", + "provider", + "authentication", + "sources", + "options", + "max_attempts", + "is_active", + "health", + "paused_reason", + "last_shipped_at", + "last_failure_at", + "last_failure_reason", + "created_at", + "updated_at", + ) + + def resolve_provider_info(self, info): + try: + adapter = get_adapter(self.provider) + except ValueError: + return None + return { + "id": adapter.id, + "name": adapter.name, + "credentials_provider": Providers.get_provider_config( + adapter.credentials_provider + ), + "max_event_age_hours": ( + adapter.max_event_age.total_seconds() / 3600 + if adapter.max_event_age + else None + ), + } + + def resolve_sources(self, info): + return self.sources or [] + + def resolve_destination_url(self, info): + # Never exposes credential values — only the adapter-derived link. + if not self.authentication_id: + return None + try: + from api.utils.syncing.auth import decrypt_credential_values, get_credentials + + adapter = get_adapter(self.provider) + # Polled query: decrypt only the keys the link needs (the + # authentication row is select_related on the list queryset) + # instead of the full credential set including the API key. + if adapter.url_credential_keys: + credentials = decrypt_credential_values( + self.authentication, adapter.url_credential_keys + ) + else: + credentials = get_credentials(self.authentication_id) + return adapter.destination_url(credentials, self.options or {}) + except Exception: + return None + + def resolve_source_lags(self, info): + # Computed for paused streams too: 0 reads as "Up to date", and a + # paused stream with a growing backlog is exactly when the real lag + # (and the approaching ingestion-window cutoff) must stay visible. + return [ + { + "source": source_id, + "name": SOURCES[source_id].name if source_id in SOURCES else source_id, + "lag_seconds": lag_for(self, source_id), + } + for source_id in (self.sources or []) + ] + + def resolve_unresolved_failures(self, info): + return self.delivery_events.filter( + status__in=[STATUS_FAILED, STATUS_SKIPPED], resolved_at__isnull=True + ).count() + + def resolve_delivery_summary(self, info): + from django.db.models import Count, Q + + since = timezone.now() - timezone.timedelta(hours=24) + counts = self.delivery_events.filter(created_at__gte=since).aggregate( + completed=Count("id", filter=Q(status=STATUS_COMPLETED)), + failed=Count("id", filter=Q(status=STATUS_FAILED)), + ) + return {"completed": counts["completed"], "failed": counts["failed"]} diff --git a/backend/ee/integrations/logs/streams/jobs.py b/backend/ee/integrations/logs/streams/jobs.py new file mode 100644 index 000000000..7a50fe34b --- /dev/null +++ b/backend/ee/integrations/logs/streams/jobs.py @@ -0,0 +1,50 @@ +"""Bootstrap for the recurring log stream sweep. + +Registered from a post_migrate hook (api/config.py) on cloud and self-hosted +alike. The job carries a stable id and is cancelled before re-registration, +so repeated migrations replace the schedule instead of accumulating +duplicates. +""" + +import logging +from datetime import timedelta + +import django_rq +from django.utils import timezone + +from .engine import SWEEP_INTERVAL_SECONDS, sweep_log_streams + +logger = logging.getLogger(__name__) + +# NOTE: rq 2.x forbids ":" in job ids. +SWEEP_JOB_ID = "log-streams-sweep" + + +def init_log_stream_sweeper(): + scheduler = django_rq.get_scheduler("scheduled-jobs") + + try: + scheduler.cancel(SWEEP_JOB_ID) + except Exception: + logger.debug("No existing log stream sweep to cancel", exc_info=True) + + scheduler.schedule( + scheduled_time=timezone.now() + timedelta(seconds=15), + func=sweep_log_streams, + interval=SWEEP_INTERVAL_SECONDS, + repeat=None, + # -1 = never expire the job hash. rq-scheduler interval jobs keep + # their schedule metadata ON the job hash; with a short result_ttl, + # any freeze longer than the TTL (host sleep, paused VM, Redis + # failover) expires the hash and the scheduler then permanently + # drops the schedule on its next pass (NoSuchJobError -> cancel). + # The stable id + cancel-before-schedule above keeps this single + # persistent hash from ever accumulating. + result_ttl=-1, + id=SWEEP_JOB_ID, + ) + logger.info( + "Log stream sweeper scheduled every %ss (job id %s)", + SWEEP_INTERVAL_SECONDS, + SWEEP_JOB_ID, + ) diff --git a/backend/ee/integrations/logs/streams/serializers.py b/backend/ee/integrations/logs/streams/serializers.py new file mode 100644 index 000000000..290189125 --- /dev/null +++ b/backend/ee/integrations/logs/streams/serializers.py @@ -0,0 +1,246 @@ +"""Neutral, versioned envelopes for exported log events. + +Attribute names follow OpenTelemetry semantic conventions where one exists +(``client.address``, ``user_agent.original``, ``user.*``); Phase domain data +lives under ``phase.*``. Destination-specific reserved fields and attribute +remapping (e.g. Datadog's ``network.client.ip`` / ``usr.*``) belong in the +adapters, never here. + +Hard rule: ``key``, ``key_digest``, ``value`` and ``comment`` from +SecretEvent are E2EE ciphertext (and the digest is a blind index over the key +name) — they must never appear in an envelope. Secrets are identified by id, +path and their app/environment; the console's global search resolves a bare +secret id back to the exact secret. Envelopes deliberately carry no console +URLs — a baked hostname rots in the destination's records when a self-hosted +console moves or serves multiple origins. +""" + +SCHEMA_VERSION = 1 + +EVENT_TYPE_NAMES = { + "C": "create", + "R": "read", + "U": "update", + "D": "delete", + "A": "access", +} + +ACTOR_TYPE_NAMES = { + "user": "user", + "sa": "service_account", +} + +SECRET_EVENT_VERBS = { + "C": "created", + "R": "read", + "U": "updated", + "D": "deleted", +} + +VERB_LABELS = { + "C": "Created", + "R": "Read", + "U": "Updated", + "D": "Deleted", + "A": "Accessed", +} + +# The DB stores compact resource codes; exports ship readable slugs so SIEM +# queries and facets don't need a Phase-internal decoder ring. +RESOURCE_TYPE_NAMES = { + "app": "app", + "env": "environment", + "role": "role", + "sa": "service_account", + "member": "member", + "policy": "network_access_policy", + "pat": "personal_access_token", + "sa_token": "service_account_token", + "svc_token": "service_token", + "invite": "invite", + "team": "team", + "rs": "rotating_secret", + "stream": "log_stream", +} + +RESOURCE_LABELS = { + "app": "app", + "env": "environment", + "role": "role", + "sa": "service account", + "member": "member", + "policy": "network access policy", + "pat": "personal access token", + "sa_token": "service account token", + "svc_token": "service token", + "invite": "invite", + "team": "team", + "rs": "rotating secret", + "stream": "log stream", +} + + +def _organisation_block(organisation): + return {"id": str(organisation.id), "name": organisation.name} + + +def audit_event_to_envelope(event, organisation): + actor_metadata = event.actor_metadata or {} + + # Call-site descriptions win; otherwise synthesize one so the + # destination's content line is never a bare "org_audit.update". + description = event.description or "" + if not description: + resource_label = RESOURCE_LABELS.get(event.resource_type, event.resource_type) + resource_name = (event.resource_metadata or {}).get("name") + description = ( + f"{VERB_LABELS.get(event.event_type, event.event_type)} {resource_label}" + ) + if resource_name: + description += f" '{resource_name}'" + + envelope = { + "schema_version": SCHEMA_VERSION, + "event": { + "id": str(event.id), + "category": "org_audit", + "type": EVENT_TYPE_NAMES.get(event.event_type, event.event_type), + }, + "timestamp": event.timestamp.isoformat(), + "actor": { + "type": ACTOR_TYPE_NAMES.get(event.actor_type, event.actor_type), + "id": event.actor_id, + "name": actor_metadata.get("email") + or actor_metadata.get("name") + or actor_metadata.get("username") + or "", + }, + "client": {"address": event.ip_address}, + "user_agent": {"original": event.user_agent or ""}, + "phase": { + "organisation": _organisation_block(organisation), + "resource": { + "type": RESOURCE_TYPE_NAMES.get(event.resource_type, event.resource_type), + "id": event.resource_id, + "metadata": event.resource_metadata or {}, + }, + "old_values": event.old_values, + "new_values": event.new_values, + # Capped: the Datadog adapter duplicates this into the top-level + # `message`, so an unbounded description would double its weight + # against the 1MB wire limit. + "description": description[:2000], + }, + } + + if event.actor_type == "user": + envelope["user"] = { + "id": event.actor_id, + "email": actor_metadata.get("email", ""), + "name": actor_metadata.get("username", ""), + } + token = actor_metadata.get("token") + if token: + envelope["actor"]["token"] = { + "id": token.get("id", ""), + "name": token.get("name", ""), + "type": token.get("type", ""), + } + + return envelope + + +def _secret_event_actor(event): + """Resolve the SecretEvent actor FKs into (actor_block, user_block). + + Engine-driven events (e.g. rotations) have no actor at all — mirrored + here as type "phase", matching how the console renders them. + """ + user_block = None + if event.user_id and event.user: + user = event.user.user + actor = { + "type": "user", + "id": str(event.user_id), + "name": getattr(user, "full_name", "") or getattr(user, "email", ""), + } + user_block = { + "id": str(event.user_id), + "email": getattr(user, "email", ""), + "name": getattr(user, "full_name", "") or getattr(user, "email", ""), + } + elif event.service_account_id and event.service_account: + actor = { + "type": "service_account", + "id": str(event.service_account_id), + "name": event.service_account.name, + } + if event.service_account_token_id and event.service_account_token: + actor["token"] = { + "id": str(event.service_account_token_id), + "name": event.service_account_token.name, + "type": "sa_token", + } + elif event.service_token_id and event.service_token: + actor = { + "type": "service_token", + "id": str(event.service_token_id), + "name": event.service_token.name, + } + else: + actor = {"type": "phase", "id": "", "name": "Phase"} + + return actor, user_block + + +def secret_event_to_envelope(event, organisation): + environment = event.environment + app = environment.app + + actor, user_block = _secret_event_actor(event) + + # Human-readable summary — becomes the destination's message/content line. + # The secret name is E2EE and never included; app/env/path locate it. + verb = SECRET_EVENT_VERBS.get(event.event_type, event.event_type) + location = f"{app.name} / {environment.name}" + path = event.path or "/" + if path != "/": + location += f" at {path}" + description = f"Secret {verb} in {location} by {actor['name']}" + + envelope = { + "schema_version": SCHEMA_VERSION, + "event": { + "id": str(event.id), + "category": "secrets", + "type": EVENT_TYPE_NAMES.get(event.event_type, event.event_type), + }, + "timestamp": event.timestamp.isoformat(), + "actor": actor, + "client": {"address": event.ip_address}, + "user_agent": {"original": event.user_agent or ""}, + "phase": { + "organisation": _organisation_block(organisation), + "app": {"id": str(app.id), "name": app.name}, + "environment": { + "id": str(environment.id), + "name": environment.name, + "type": environment.env_type, + }, + "secret": { + "id": str(event.secret_id), + "path": event.path or "/", + "version": event.version, + "type": event.type, + }, + # Capped: the Datadog adapter duplicates this into the top-level + # `message`, so an unbounded description would double its weight + # against the 1MB wire limit. + "description": description[:2000], + }, + } + + if user_block: + envelope["user"] = user_block + + return envelope diff --git a/backend/ee/integrations/logs/streams/sources.py b/backend/ee/integrations/logs/streams/sources.py new file mode 100644 index 000000000..92f815d39 --- /dev/null +++ b/backend/ee/integrations/logs/streams/sources.py @@ -0,0 +1,166 @@ +"""Event source registry for log streams. + +Adding a new streamable event source is a two-step change: + +1. Implement a ``LogSource`` subclass below (fetch/fetch_range/count_before/ + oldest_pending_timestamp/serialize over your event model). +2. Register an instance in ``SOURCES``. + +Everything else — the UI checkbox, per-stream cursors, lag reporting and +delivery history — follows automatically from the registry entry. + +Cursors are ``{"ts": "", "id": ""}`` pairs. Events are totally +ordered by ``(timestamp, id)``; the id tiebreak makes pagination stable when +multiple events share a timestamp (ids are uuid strings, so intra-timestamp +order is arbitrary but consistent). +""" + +from datetime import timedelta + +from django.apps import apps +from django.db.models import Q +from django.utils import timezone +from django.utils.dateparse import parse_datetime + +from .serializers import audit_event_to_envelope, secret_event_to_envelope + +# Events are timestamped inside the writing transaction, so a slow +# transaction can COMMIT an older-timestamped event after a newer one was +# already fetched and shipped — the late event would land permanently behind +# the cursor and never ship. Only events older than this watermark are +# eligible, giving in-flight transactions time to commit first. Well under +# the 30s sweep interval's user-facing latency promise ("about a minute"). +SHIP_WATERMARK_SECONDS = 30 + + +class LogSource: + id = None + name = None + description = None + + def _queryset(self, organisation): + raise NotImplementedError + + def serialize(self, event, organisation): + raise NotImplementedError + + def _cursor_filter(self, cursor): + ts = parse_datetime(cursor["ts"]) if isinstance(cursor["ts"], str) else cursor["ts"] + last_id = cursor.get("id") or "" + # Redundant but load-bearing: Postgres derives no lower scan bound + # across the OR arms. + return (Q(timestamp__gt=ts) | (Q(timestamp=ts) & Q(id__gt=last_id))) & Q( + timestamp__gte=ts + ) + + def fetch(self, organisation, cursor, limit): + """Events strictly after `cursor` but older than the commit-safety + watermark, oldest first.""" + watermark = timezone.now() - timedelta(seconds=SHIP_WATERMARK_SECONDS) + return list( + self._queryset(organisation) + .filter(self._cursor_filter(cursor), timestamp__lt=watermark) + .order_by("timestamp", "id")[:limit] + ) + + def fetch_range(self, organisation, ts_from, ts_to, limit=None): + """Events in the inclusive [ts_from, ts_to] window, oldest first. + + Used by manual delivery retries. The window is the failed chunk's + recorded timestamp range, so boundary events that also landed in a + neighbouring chunk may be re-shipped — acceptable under the + at-least-once delivery contract. `limit` bounds materialization so a + pathologically large range can't exhaust a worker; the caller detects + the cap and fails the retry honestly instead of shipping a subset. + """ + queryset = ( + self._queryset(organisation) + .filter(timestamp__gte=ts_from, timestamp__lte=ts_to) + .order_by("timestamp", "id") + ) + if limit is not None: + queryset = queryset[:limit] + return list(queryset) + + def count_before(self, organisation, cursor, ts_floor): + """Events after `cursor` but older than `ts_floor` (skip-ahead count).""" + return ( + self._queryset(organisation) + .filter(self._cursor_filter(cursor), timestamp__lt=ts_floor) + .count() + ) + + def oldest_pending_timestamp(self, organisation, cursor): + """Timestamp of the oldest event still waiting to ship, or None. + + Drives the delivery-delay metric: its age is how late deliveries + actually are, unlike cursor distance, which spikes misleadingly when + a new event arrives after an idle gap. + """ + return ( + self._queryset(organisation) + .filter(self._cursor_filter(cursor)) + .order_by("timestamp", "id") + .values_list("timestamp", flat=True) + .first() + ) + + def cursor_of(self, event): + return {"ts": event.timestamp.isoformat(), "id": str(event.id)} + + +class OrgAuditLogSource(LogSource): + id = "org_audit" + name = "Organisation audit logs" + description = "Organisation-level activity: members, roles, apps, tokens and teams." + + def _queryset(self, organisation): + AuditEvent = apps.get_model("api", "AuditEvent") + return AuditEvent.objects.filter(organisation=organisation) + + def serialize(self, event, organisation): + return audit_event_to_envelope(event, organisation) + + +class SecretEventLogSource(LogSource): + id = "secrets" + name = "App secret logs" + description = "Create, read, update and delete events for secrets across all apps." + + def _queryset(self, organisation): + SecretEvent = apps.get_model("api", "SecretEvent") + return SecretEvent.objects.filter( + environment__app__organisation=organisation + ).select_related( + "secret", + "environment", + "environment__app", + "user", + "user__user", + "service_token", + "service_account", + "service_account_token", + ) + + def serialize(self, event, organisation): + return secret_event_to_envelope(event, organisation) + + +SOURCES = { + source.id: source + for source in ( + OrgAuditLogSource(), + SecretEventLogSource(), + ) +} + + +def get_source(source_id): + source = SOURCES.get(source_id) + if source is None: + raise ValueError(f"Unknown log stream source '{source_id}'") + return source + + +def all_sources(): + return list(SOURCES.values()) diff --git a/backend/tests/api/views/test_audit_view_authz.py b/backend/tests/api/views/test_audit_view_authz.py new file mode 100644 index 000000000..9c81915c3 --- /dev/null +++ b/backend/tests/api/views/test_audit_view_authz.py @@ -0,0 +1,117 @@ +"""PublicAuditLogsView authorization: only principals with a permission +model (Users, Service Accounts) may read organisation-wide audit logs. + +Legacy service tokens authenticate through the same PhaseTokenAuthentication +class but are environment-scoped and have no role — the view must fail +closed for them instead of silently skipping the Logs:read check. +""" + +from unittest.mock import MagicMock, patch + +import pytest +from rest_framework.exceptions import PermissionDenied +from rest_framework.views import APIView + +from api.views.audit import PublicAuditLogsView + +_V = "api.views.audit" + + +def _request(auth): + request = MagicMock() + request.auth = auth + return request + + +def _view(): + return PublicAuditLogsView() + + +def _initial(view, request): + # Bypass DRF's own initial() machinery (authentication/throttling) — + # under test is only this view's authorization logic. + with patch.object(APIView, "initial"): + view.initial(request) + + +def test_service_tokens_are_rejected(): + org = MagicMock() + request = _request({"auth_type": "Service", "app": MagicMock(organisation=org)}) + + with patch(f"{_V}.user_has_permission") as mock_perm: + with pytest.raises(PermissionDenied, match="service token"): + _initial(_view(), request) + + mock_perm.assert_not_called() + + +def test_user_without_logs_permission_is_rejected(): + org = MagicMock() + org_member = MagicMock() + request = _request( + {"auth_type": "User", "org_member": org_member, "organisation": org} + ) + + with patch(f"{_V}.user_has_permission", return_value=False) as mock_perm, patch( + f"{_V}.role_has_global_access", return_value=True + ): + with pytest.raises(PermissionDenied, match="permission"): + _initial(_view(), request) + + mock_perm.assert_called_once_with( + org_member.user, "read", "Logs", org, False, False + ) + + +def test_service_account_tokens_are_rejected(): + """Service accounts cannot hold global-access roles (enforced at SA + create/update), so they can never satisfy the org-wide guard — reject + with an actionable message instead of an unsatisfiable one.""" + org = MagicMock() + service_account = MagicMock() + request = _request( + { + "auth_type": "ServiceAccount", + "service_account": service_account, + "organisation": org, + } + ) + + with patch(f"{_V}.user_has_permission") as mock_perm: + with pytest.raises(PermissionDenied, match="service account"): + _initial(_view(), request) + + mock_perm.assert_not_called() + + +def test_user_with_global_access_role_is_allowed(): + org = MagicMock() + org_member = MagicMock() + request = _request( + {"auth_type": "User", "org_member": org_member, "organisation": org} + ) + + with patch(f"{_V}.user_has_permission", return_value=True), patch( + f"{_V}.role_has_global_access", return_value=True + ): + _initial(_view(), request) + + +def test_scoped_roles_are_rejected(): + """This endpoint returns the unscoped org-wide stream. The Console's + GraphQL resolver filters events for roles without global access — the + REST view doesn't replicate that scoping, so it must fail closed for + scoped roles (e.g. default Developer/Manager) instead of over-exposing.""" + org = MagicMock() + org_member = MagicMock() + request = _request( + {"auth_type": "User", "org_member": org_member, "organisation": org} + ) + + with patch(f"{_V}.user_has_permission", return_value=True), patch( + f"{_V}.role_has_global_access", return_value=False + ) as mock_global: + with pytest.raises(PermissionDenied, match="global access"): + _initial(_view(), request) + + mock_global.assert_called_once_with(org_member.role) diff --git a/backend/tests/ee/integrations/logs/__init__.py b/backend/tests/ee/integrations/logs/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/tests/ee/integrations/logs/streams/__init__.py b/backend/tests/ee/integrations/logs/streams/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/tests/ee/integrations/logs/streams/adapters/__init__.py b/backend/tests/ee/integrations/logs/streams/adapters/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/tests/ee/integrations/logs/streams/adapters/test_datadog.py b/backend/tests/ee/integrations/logs/streams/adapters/test_datadog.py new file mode 100644 index 000000000..a9c2b7f70 --- /dev/null +++ b/backend/tests/ee/integrations/logs/streams/adapters/test_datadog.py @@ -0,0 +1,313 @@ +"""Datadog adapter: URL/site handling, payload wrapping, status mapping.""" + +import gzip +import json +from contextlib import contextmanager +from unittest.mock import MagicMock, patch + +import pytest +import requests as requests_lib + +from ee.integrations.logs.streams.adapters.datadog import DatadogAdapter +from ee.integrations.logs.streams.exceptions import ( + AdapterAuthError, + AdapterPermanentError, + AdapterRateLimitedError, + AdapterTransientError, +) + +_M = "ee.integrations.logs.streams.adapters.datadog" + + +@contextmanager +def _patch_http(method="post", **mock_kwargs): + """Patch the adapter's pooled requests session; yields the method mock.""" + session = MagicMock() + method_mock = getattr(session, method) + method_mock.configure_mock(**mock_kwargs) + with patch(f"{_M}._get_session", return_value=session): + yield method_mock + + +CREDS = {"api_key": "dd-key-123", "site": "us3.datadoghq.com"} +CONTEXT = {"organisation_name": "Acme Corp", "stream_name": "My Datadog Export"} + +ENVELOPE = { + "schema_version": 1, + "event": {"id": "e1", "category": "secrets", "type": "read"}, + "timestamp": "2026-07-30T12:00:00+00:00", + "actor": {"type": "user", "id": "m1", "name": "dev@example.com"}, + "client": {"address": "203.0.113.7"}, + "user_agent": {"original": "phase-cli/1.18"}, + "user": {"id": "m1", "email": "dev@example.com", "name": "Dev"}, + "phase": {"organisation": {"id": "o1", "name": "Acme"}}, +} + + +def _response(status=202, headers=None, text=""): + response = MagicMock() + response.status_code = status + response.headers = headers or {} + response.text = text + return response + + +def _adapter(): + return DatadogAdapter() + + +def test_intake_url_site_allowlist_and_normalisation(): + adapter = _adapter() + + url, site = adapter._intake_url({"site": "https://us3.datadoghq.com/"}) + assert url == "https://http-intake.logs.us3.datadoghq.com/api/v2/logs" + assert site == "us3.datadoghq.com" + + url, _ = adapter._intake_url({}) + assert url == "https://http-intake.logs.datadoghq.com/api/v2/logs" + + with pytest.raises(AdapterPermanentError, match="Unknown Datadog site"): + adapter._intake_url({"site": "evil.example.com"}) + + +def test_ship_sends_gzipped_payload_with_reserved_fields_and_remaps(): + adapter = _adapter() + + with _patch_http("post", return_value=_response(202)) as mock_post: + result = adapter.ship([ENVELOPE], CREDS, {"tags": "env:prod"}, CONTEXT) + + assert result.status_code == 202 + args, kwargs = mock_post.call_args + assert args[0] == "https://http-intake.logs.us3.datadoghq.com/api/v2/logs" + assert kwargs["headers"]["DD-API-KEY"] == "dd-key-123" + assert kwargs["headers"]["Content-Encoding"] == "gzip" + + payload = json.loads(gzip.decompress(kwargs["data"])) + event = payload[0] + + # Reserved fields set only in the adapter; tag values are sanitized so + # names with spaces don't break Datadog's tag parsing. + assert event["ddsource"] == "phase" + assert event["service"] == "phase-console" + assert "phase_org:acme_corp" in event["ddtags"] + assert "phase_stream:my_datadog_export" in event["ddtags"] + assert "env:prod" in event["ddtags"] + assert event["message"] == "secrets.read" + + # OTel names remapped to Datadog standard attributes + assert event["network"]["client"]["ip"] == "203.0.113.7" + assert event["http"]["useragent"] == "phase-cli/1.18" + assert event["usr"]["email"] == "dev@example.com" + for removed in ("client", "user_agent", "user"): + assert removed not in event + + +def test_ship_without_gzip(): + adapter = _adapter() + + with _patch_http("post", return_value=_response(202)) as mock_post: + adapter.ship([ENVELOPE], CREDS, {"gzip": False}, CONTEXT) + + _, kwargs = mock_post.call_args + assert "Content-Encoding" not in kwargs["headers"] + json.loads(kwargs["data"]) # plain JSON body + + +@pytest.mark.parametrize( + "status,exc", + [ + (401, AdapterAuthError), + (403, AdapterAuthError), + (400, AdapterPermanentError), + (413, AdapterPermanentError), + (500, AdapterTransientError), + (503, AdapterTransientError), + ], +) +def test_ship_maps_http_status_to_typed_errors(status, exc): + adapter = _adapter() + + with _patch_http("post", return_value=_response(status)): + with pytest.raises(exc): + adapter.ship([ENVELOPE], CREDS, {}, CONTEXT) + + +@pytest.mark.parametrize("status", [200, 301, 302, 303, 307]) +def test_ship_accepts_only_202(status): + """Datadog's v2 intake acknowledges with 202 only. A redirect or a + non-202 2xx means an intermediary answered (e.g. a TLS-intercepting + proxy) — accepting it would advance the cursor over events Datadog never + ingested. Retrying is safe: at-least-once, consumers dedupe.""" + adapter = _adapter() + + with _patch_http("post", return_value=_response(status)): + with pytest.raises(AdapterTransientError): + adapter.ship([ENVELOPE], CREDS, {}, CONTEXT) + + +def test_ship_does_not_follow_redirects(): + """Following a 3xx would convert the POST to a body-less GET (silent + data loss) and re-send DD-API-KEY to the redirect target — requests only + strips Authorization on cross-host redirects.""" + adapter = _adapter() + + with _patch_http("post", return_value=_response(202)) as mock_post: + adapter.ship([ENVELOPE], CREDS, {}, CONTEXT) + + assert mock_post.call_args.kwargs["allow_redirects"] is False + + +def test_ship_rate_limited_carries_retry_after(): + adapter = _adapter() + + with _patch_http( + "post", + return_value=_response(429, headers={"Retry-After": "30"}), + ): + with pytest.raises(AdapterRateLimitedError) as excinfo: + adapter.ship([ENVELOPE], CREDS, {}, CONTEXT) + + assert excinfo.value.retry_after == 30.0 + + +def test_ship_connection_error_is_transient(): + adapter = _adapter() + + with _patch_http("post", side_effect=requests_lib.ConnectionError("boom")): + with pytest.raises(AdapterTransientError): + adapter.ship([ENVELOPE], CREDS, {}, CONTEXT) + + +def test_test_validates_key_without_ingesting_data(): + """Connection tests hit Datadog's key-validation endpoint — never the + logs intake, so no garbage events land in the customer's org.""" + adapter = _adapter() + + session = MagicMock() + session.get.return_value = _response(200) + with patch(f"{_M}._get_session", return_value=session): + ok, meta = adapter.test(CREDS, {}, CONTEXT) + + assert ok is True + assert meta["status_code"] == 200 + assert session.get.call_args.args[0] == "https://api.us3.datadoghq.com/api/v1/validate" + assert session.get.call_args.kwargs["headers"]["DD-API-KEY"] == "dd-key-123" + assert session.get.call_args.kwargs["allow_redirects"] is False + session.post.assert_not_called() + + +def test_test_maps_invalid_key_to_auth_error(): + adapter = _adapter() + + with _patch_http("get", return_value=_response(403)): + with pytest.raises(AdapterAuthError, match="credentials"): + adapter.test(CREDS, {}, CONTEXT) + + +def test_auth_error_message_says_credentials_not_key(): + adapter = _adapter() + + with _patch_http("post", return_value=_response(401)): + with pytest.raises(AdapterAuthError) as excinfo: + adapter.ship([ENVELOPE], CREDS, {}, CONTEXT) + + assert "credentials" in excinfo.value.user_message + assert "key" not in excinfo.value.user_message.lower() + + +@pytest.mark.parametrize( + "site,expected_host", + [ + ("datadoghq.com", "app.datadoghq.com"), + ("datadoghq.eu", "app.datadoghq.eu"), + ("us3.datadoghq.com", "us3.datadoghq.com"), + ("ap1.datadoghq.com", "ap1.datadoghq.com"), + ("ddog-gov.com", "app.ddog-gov.com"), + ], +) +def test_destination_url_maps_site_to_app_host(site, expected_host): + adapter = _adapter() + + url = adapter.destination_url({"site": site}, {}) + + assert url == f"https://{expected_host}/logs?query=source%3Aphase" + + +def test_destination_url_rejects_unknown_site(): + adapter = _adapter() + + assert adapter.destination_url({"site": "evil.example.com"}, {}) is None + + +def test_validate_options_defaults_and_bounds(): + adapter = _adapter() + + options = adapter.validate_options({}) + assert options == {"service": "phase-console", "tags": "", "gzip": True} + + options = adapter.validate_options({"service": "x" * 500, "gzip": False}) + assert len(options["service"]) == 100 + assert options["gzip"] is False + + +def test_parse_retry_after_accepts_seconds_dates_and_garbage(): + """RFC 9110 allows delay-seconds OR an HTTP-date (intermediary proxies + emit dates). An unparseable value must yield None — the engine then uses + its own backoff — never raise out of the retry ladder.""" + from ee.integrations.logs.streams.adapters.datadog import _parse_retry_after + + assert _parse_retry_after("2.5") == 2.5 + assert _parse_retry_after(None) is None + assert _parse_retry_after("") is None + assert _parse_retry_after("not-a-date") is None + # Negative values would crash time.sleep; nan/inf poison the engine's + # min()/deadline arithmetic and the meta JSON. Clamp or discard. + assert _parse_retry_after("-1") == 0.0 + assert _parse_retry_after("nan") is None + assert _parse_retry_after("inf") is None + # A past HTTP-date clamps to 0 (falsy -> engine backoff). + assert _parse_retry_after("Wed, 21 Oct 2015 07:28:00 GMT") == 0.0 + future = _parse_retry_after("Fri, 01 Jan 2100 00:00:00 GMT") + assert future is not None and future > 0 + + +def test_ship_survives_http_date_retry_after(): + """Regression: a bare float() on an HTTP-date Retry-After raised + ValueError past the typed error handlers into the job crash path.""" + adapter = _adapter() + response = _response( + 429, headers={"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"} + ) + + with _patch_http("post", return_value=response): + with pytest.raises(AdapterRateLimitedError) as exc_info: + adapter.ship([ENVELOPE], CREDS, {}, CONTEXT) + + assert exc_info.value.retry_after == 0.0 + + +def test_ship_rate_limited_without_retry_after_header(): + adapter = _adapter() + + with _patch_http("post", return_value=_response(429)): + with pytest.raises(AdapterRateLimitedError) as exc_info: + adapter.ship([ENVELOPE], CREDS, {}, CONTEXT) + + assert exc_info.value.retry_after is None + + +def test_intake_and_app_urls_for_uk1_and_us2_fed(): + adapter = _adapter() + + url, _ = adapter._intake_url({"site": "uk1.datadoghq.com"}) + assert url == "https://http-intake.logs.uk1.datadoghq.com/api/v2/logs" + url, _ = adapter._intake_url({"site": "us2.ddog-gov.com"}) + assert url == "https://http-intake.logs.us2.ddog-gov.com/api/v2/logs" + + # Both are regional (two-dot) sites — already app hosts, no app. prefix. + assert adapter.destination_url({"site": "uk1.datadoghq.com"}, {}) == ( + "https://uk1.datadoghq.com/logs?query=source%3Aphase" + ) + assert adapter.destination_url({"site": "us2.ddog-gov.com"}, {}) == ( + "https://us2.ddog-gov.com/logs?query=source%3Aphase" + ) diff --git a/backend/tests/ee/integrations/logs/streams/graphene/__init__.py b/backend/tests/ee/integrations/logs/streams/graphene/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/tests/ee/integrations/logs/streams/graphene/test_mutations.py b/backend/tests/ee/integrations/logs/streams/graphene/test_mutations.py new file mode 100644 index 000000000..349f29da6 --- /dev/null +++ b/backend/tests/ee/integrations/logs/streams/graphene/test_mutations.py @@ -0,0 +1,675 @@ +"""Log stream mutations: permission, plan-gate and input validation guards.""" + +from datetime import timedelta +from unittest.mock import MagicMock, patch + +import pytest +from django.utils import timezone +from graphql import GraphQLError + +from api.models import AuditEvent +from ee.integrations.logs.streams.graphene.mutations import ( + CreateLogStreamMutation, + DeleteLogStreamMutation, + RetryLogStreamDeliveryMutation, + ToggleLogStreamMutation, + UpdateLogStreamMutation, +) +# Aliased so pytest doesn't try to collect the graphene class as a test case. +from ee.integrations.logs.streams.graphene.mutations import ( + TestLogStreamConnectionMutation as ConnectionTestMutation, +) + +_M = "ee.integrations.logs.streams.graphene.mutations" + + +@pytest.fixture(autouse=True) +def _global_access(): + """Log stream operations additionally require a role with global access; + grant it by default so each test exercises its own concern. The denial + path sets .return_value = False explicitly.""" + with patch(f"{_M}.user_has_global_access", return_value=True) as mock_global: + yield mock_global + + +def _info(): + info = MagicMock() + info.context.user.userId = "user-1" + return info + + +def _org(): + org = MagicMock() + org.id = "org-1" + org.plan = "EN" + return org + + +def _adapter(): + adapter = MagicMock() + adapter.id = "datadog" + adapter.name = "Datadog" + adapter.credentials_provider = "datadog" + adapter.max_event_age = timedelta(hours=18) + adapter.validate_options.side_effect = lambda options: options + return adapter + + +def _credential(org_id="org-1", provider="datadog"): + credential = MagicMock() + credential.id = "cred-1" + credential.organisation_id = org_id + credential.provider = provider + return credential + + +def _create(org, **overrides): + kwargs = dict( + organisation_id=org.id, + name="Datadog prod", + provider="datadog", + credential_id="cred-1", + sources=["org_audit"], + ) + kwargs.update(overrides) + return CreateLogStreamMutation.mutate(None, _info(), **kwargs) + + +def test_create_blocked_without_permission(): + org = _org() + + with patch(f"{_M}.Organisation") as MockOrg, patch( + f"{_M}.user_has_permission", return_value=False + ): + MockOrg.objects.get.return_value = org + + with pytest.raises(GraphQLError, match="permission"): + _create(org) + + +def test_create_blocked_without_global_access(_global_access): + """A scoped custom role can hold LogStreams permissions, but streams + export org-wide activity — configuring one requires global access.""" + _global_access.return_value = False + org = _org() + + with patch(f"{_M}.Organisation") as MockOrg, patch( + f"{_M}.user_has_permission", return_value=True + ): + MockOrg.objects.get.return_value = org + + with pytest.raises(GraphQLError, match="global access"): + _create(org) + + +def test_create_blocked_on_non_enterprise_plan(): + org = _org() + org.plan = "PR" + + with patch(f"{_M}.Organisation") as MockOrg, patch( + f"{_M}.user_has_permission", return_value=True + ), patch(f"{_M}.can_use_log_streams", return_value=False): + MockOrg.objects.get.return_value = org + + with pytest.raises(GraphQLError, match="Enterprise"): + _create(org) + + +def test_create_rejects_cross_org_credential(): + org = _org() + + with patch(f"{_M}.Organisation") as MockOrg, patch( + f"{_M}.user_has_permission", return_value=True + ), patch(f"{_M}.can_use_log_streams", return_value=True), patch( + f"{_M}.get_adapter", return_value=_adapter() + ), patch( + f"{_M}.get_source" + ), patch( + f"{_M}.ProviderCredentials" + ) as MockCreds, patch( + f"{_M}.LogStream" + ) as MockStream: + MockOrg.objects.get.return_value = org + MockCreds.DoesNotExist = Exception + MockCreds.objects.get.return_value = _credential(org_id="other-org") + + with pytest.raises(GraphQLError, match="don't exist"): + _create(org) + + MockStream.objects.create.assert_not_called() + + +def test_create_rejects_wrong_provider_credential(): + org = _org() + + with patch(f"{_M}.Organisation") as MockOrg, patch( + f"{_M}.user_has_permission", return_value=True + ), patch(f"{_M}.can_use_log_streams", return_value=True), patch( + f"{_M}.get_adapter", return_value=_adapter() + ), patch( + f"{_M}.get_source" + ), patch( + f"{_M}.ProviderCredentials" + ) as MockCreds: + MockOrg.objects.get.return_value = org + MockCreds.DoesNotExist = Exception + MockCreds.objects.get.return_value = _credential(provider="aws") + + with pytest.raises(GraphQLError, match="require datadog credentials"): + _create(org) + + +def test_create_rejects_empty_and_unknown_sources(): + org = _org() + + with patch(f"{_M}.Organisation") as MockOrg, patch( + f"{_M}.user_has_permission", return_value=True + ), patch(f"{_M}.can_use_log_streams", return_value=True), patch( + f"{_M}.get_adapter", return_value=_adapter() + ), patch( + f"{_M}.get_source", side_effect=ValueError("Unknown log stream source 'bogus'") + ): + MockOrg.objects.get.return_value = org + + with pytest.raises(GraphQLError, match="at least one"): + _create(org, sources=[]) + + with pytest.raises(GraphQLError, match="Unknown log stream source"): + _create(org, sources=["bogus"]) + + +def test_create_clamps_max_attempts_and_dedupes_sources(): + org = _org() + + with patch(f"{_M}.Organisation") as MockOrg, patch( + f"{_M}.user_has_permission", return_value=True + ), patch(f"{_M}.can_use_log_streams", return_value=True), patch( + f"{_M}.get_adapter", return_value=_adapter() + ), patch( + f"{_M}.get_source" + ), patch( + f"{_M}.ProviderCredentials" + ) as MockCreds, patch( + f"{_M}.LogStream" + ) as MockStream, patch( + f"{_M}.log_audit_event" + ): + MockOrg.objects.get.return_value = org + MockCreds.DoesNotExist = Exception + MockCreds.objects.get.return_value = _credential() + + _create(org, sources=["org_audit", "org_audit", "secrets"], max_attempts=99) + + kwargs = MockStream.objects.create.call_args.kwargs + assert kwargs["max_attempts"] == 10 + assert kwargs["sources"] == ["org_audit", "secrets"] + + +def test_toggle_pauses_via_engine(): + stream = MagicMock() + stream.is_active = True + stream.organisation = _org() + + with patch(f"{_M}.LogStream") as MockStream, patch( + f"{_M}.user_has_permission", return_value=True + ), patch(f"{_M}.can_use_log_streams", return_value=True), patch( + f"{_M}.engine.pause" + ) as mock_pause, patch( + f"{_M}.engine.resume" + ) as mock_resume, patch( + f"{_M}.log_audit_event" + ): + MockStream.objects.get.return_value = stream + + ToggleLogStreamMutation.mutate(None, _info(), stream_id="stream-1") + + mock_pause.assert_called_once_with(stream) + mock_resume.assert_not_called() + + +def test_toggle_pause_allowed_on_downgraded_plan(): + """Pause (like delete) is deliberately not plan-gated: a downgraded org + must be able to stop its streams even though the engine already halts + egress for non-Enterprise plans.""" + stream = MagicMock() + stream.is_active = True + stream.organisation = _org() + stream.organisation.plan = "PR" + + with patch(f"{_M}.LogStream") as MockStream, patch( + f"{_M}.user_has_permission", return_value=True + ), patch(f"{_M}.engine.pause") as mock_pause, patch( + f"{_M}.log_audit_event" + ): + MockStream.objects.get.return_value = stream + + ToggleLogStreamMutation.mutate(None, _info(), stream_id="stream-1") + + mock_pause.assert_called_once_with(stream) + + +def test_toggle_resume_blocked_on_downgraded_plan(): + """Resume re-enables egress, so it stays plan-gated.""" + stream = MagicMock() + stream.is_active = False + stream.authentication_id = "cred-1" + stream.organisation = _org() + stream.organisation.plan = "PR" + + with patch(f"{_M}.LogStream") as MockStream, patch( + f"{_M}.user_has_permission", return_value=True + ), patch(f"{_M}.engine.resume") as mock_resume: + MockStream.objects.get.return_value = stream + + with pytest.raises(GraphQLError, match="Enterprise"): + ToggleLogStreamMutation.mutate(None, _info(), stream_id="stream-1") + + mock_resume.assert_not_called() + + +def test_toggle_resume_requires_credentials(): + """A stream stranded by credential deletion can't resume until new + credentials are selected — resuming would just re-pause on the next + sweep.""" + stream = MagicMock() + stream.is_active = False + stream.authentication_id = None + stream.organisation = _org() + + with patch(f"{_M}.LogStream") as MockStream, patch( + f"{_M}.user_has_permission", return_value=True + ), patch(f"{_M}.can_use_log_streams", return_value=True), patch( + f"{_M}.engine.resume" + ) as mock_resume: + MockStream.objects.get.return_value = stream + + with pytest.raises(GraphQLError, match="no credentials"): + ToggleLogStreamMutation.mutate(None, _info(), stream_id="stream-1") + + mock_resume.assert_not_called() + + +def test_test_connection_rejects_wrong_provider_credential(): + """Without the provider check, another provider's api_key would be + decrypted and sent to this adapter's destination.""" + org = _org() + + with patch(f"{_M}.Organisation") as MockOrg, patch( + f"{_M}.user_has_permission", return_value=True + ), patch(f"{_M}.can_use_log_streams", return_value=True), patch( + f"{_M}.get_adapter", return_value=_adapter() + ), patch( + f"{_M}.ProviderCredentials" + ) as MockCreds, patch( + f"{_M}.engine.test_adapter_connection" + ) as mock_test: + MockOrg.objects.get.return_value = org + MockCreds.DoesNotExist = Exception + MockCreds.objects.get.return_value = _credential(provider="render") + + with pytest.raises(GraphQLError, match="require datadog credentials"): + ConnectionTestMutation.mutate( + None, + _info(), + organisation_id=org.id, + provider="datadog", + credential_id="cred-1", + ) + + mock_test.assert_not_called() + + +def test_update_saves_only_config_fields(): + """A full save would write cursors/health/activity from the mutation's + (possibly stale) row and clobber concurrent worker state.""" + stream = MagicMock() + stream.organisation = _org() + stream.provider = "datadog" + + with patch(f"{_M}.LogStream") as MockStream, patch( + f"{_M}.user_has_permission", return_value=True + ), patch(f"{_M}.can_use_log_streams", return_value=True), patch( + f"{_M}.get_adapter", return_value=_adapter() + ), patch( + f"{_M}.get_source" + ), patch( + f"{_M}.ProviderCredentials" + ) as MockCreds, patch( + f"{_M}.log_audit_event" + ): + MockStream.objects.get.return_value = stream + MockCreds.DoesNotExist = Exception + MockCreds.objects.get.return_value = _credential() + + UpdateLogStreamMutation.mutate( + None, + _info(), + stream_id="stream-1", + name="Renamed", + credential_id="cred-1", + sources=["org_audit"], + ) + + update_fields = stream.save.call_args.kwargs["update_fields"] + assert "cursors" not in update_fields + assert "is_active" not in update_fields + assert "health" not in update_fields + assert set(update_fields) == { + "name", + "authentication", + "sources", + "options", + "max_attempts", + "updated_at", + } + + +def test_retry_rejects_completed_and_resolved_deliveries(): + stream = MagicMock() + stream.organisation = _org() + stream.deleted_at = None + + delivery = MagicMock() + delivery.stream = stream + + with patch(f"{_M}.LogStreamDeliveryEvent") as MockDelivery, patch( + f"{_M}.user_has_permission", return_value=True + ), patch(f"{_M}.can_use_log_streams", return_value=True): + ( + MockDelivery.objects.select_related.return_value.get.return_value + ) = delivery + + delivery.status = "completed" + delivery.resolved_at = None + with pytest.raises(GraphQLError, match="Only failed or skipped"): + RetryLogStreamDeliveryMutation.mutate( + None, _info(), delivery_event_id="d-1" + ) + + delivery.status = "failed" + delivery.resolved_at = "2026-07-30T12:00:00Z" + with pytest.raises(GraphQLError, match="already been resolved"): + RetryLogStreamDeliveryMutation.mutate( + None, _info(), delivery_event_id="d-1" + ) + + +def _fresh_delivery(stream): + now = timezone.now() + delivery = MagicMock() + delivery.id = "d-1" + delivery.stream = stream + delivery.status = "failed" + delivery.resolved_at = None + delivery.cursor_from = now - timedelta(hours=2) + delivery.cursor_to = now - timedelta(hours=1) + return delivery + + +def test_retry_enqueues_engine_job(): + stream = MagicMock() + stream.organisation = _org() + stream.deleted_at = None + stream.provider = "datadog" + + delivery = _fresh_delivery(stream) + + with patch(f"{_M}.LogStreamDeliveryEvent") as MockDelivery, patch( + f"{_M}.user_has_permission", return_value=True + ), patch(f"{_M}.can_use_log_streams", return_value=True), patch( + f"{_M}.get_adapter", return_value=_adapter() + ), patch( + f"{_M}.engine.retry_delivery" + ) as mock_retry, patch( + f"{_M}.log_audit_event" + ): + ( + MockDelivery.objects.select_related.return_value.get.return_value + ) = delivery + + result = RetryLogStreamDeliveryMutation.mutate( + None, _info(), delivery_event_id="d-1" + ) + + assert result.ok is True + mock_retry.delay.assert_called_once_with("d-1") + + +def test_retry_rejects_expired_range(): + """The destination silently discards events older than its ingestion + window — the mutation rejects the retry with actionable guidance.""" + stream = MagicMock() + stream.organisation = _org() + stream.deleted_at = None + stream.provider = "datadog" + + delivery = _fresh_delivery(stream) + delivery.cursor_from = timezone.now() - timedelta(hours=30) + delivery.cursor_to = timezone.now() - timedelta(hours=20) + + with patch(f"{_M}.LogStreamDeliveryEvent") as MockDelivery, patch( + f"{_M}.user_has_permission", return_value=True + ), patch(f"{_M}.can_use_log_streams", return_value=True), patch( + f"{_M}.get_adapter", return_value=_adapter() + ), patch( + f"{_M}.engine.retry_delivery" + ) as mock_retry: + ( + MockDelivery.objects.select_related.return_value.get.return_value + ) = delivery + + with pytest.raises(GraphQLError, match="ingestion window"): + RetryLogStreamDeliveryMutation.mutate( + None, _info(), delivery_event_id="d-1" + ) + + mock_retry.delay.assert_not_called() + + +def test_retry_rejects_paused_streams(): + """Pause means no egress — manual retries included.""" + stream = MagicMock() + stream.organisation = _org() + stream.deleted_at = None + stream.is_active = False + + delivery = _fresh_delivery(stream) + + with patch(f"{_M}.LogStreamDeliveryEvent") as MockDelivery, patch( + f"{_M}.user_has_permission", return_value=True + ), patch(f"{_M}.can_use_log_streams", return_value=True), patch( + f"{_M}.engine.retry_delivery" + ) as mock_retry: + ( + MockDelivery.objects.select_related.return_value.get.return_value + ) = delivery + + with pytest.raises(GraphQLError, match="paused"): + RetryLogStreamDeliveryMutation.mutate( + None, _info(), delivery_event_id="d-1" + ) + + mock_retry.delay.assert_not_called() + + +def _stream_row(org): + stream = MagicMock() + stream.organisation = org + stream.name = "Datadog prod" + stream.provider = "datadog" + stream.deleted_at = None + stream.is_active = True + stream.sources = ["org_audit"] + stream.options = {} + return stream + + +def test_id_keyed_mutations_blocked_without_permission(): + """Update/toggle/delete/retry derive the org from the object — the + caller's LogStreams RBAC check must still run against that org.""" + org = _org() + stream = _stream_row(org) + delivery = MagicMock() + delivery.stream = stream + + with patch(f"{_M}.LogStream") as MockStream, patch( + f"{_M}.LogStreamDeliveryEvent" + ) as MockDelivery, patch( + f"{_M}.user_has_permission", return_value=False + ), patch( + f"{_M}.engine.pause" + ) as mock_pause, patch( + f"{_M}.engine.retry_delivery" + ) as mock_retry: + MockStream.objects.get.return_value = stream + ( + MockDelivery.objects.select_related.return_value.get.return_value + ) = delivery + + with pytest.raises(GraphQLError, match="permission"): + UpdateLogStreamMutation.mutate( + None, + _info(), + stream_id="s-1", + name="x", + credential_id="c-1", + sources=["org_audit"], + ) + with pytest.raises(GraphQLError, match="permission"): + ToggleLogStreamMutation.mutate(None, _info(), stream_id="s-1") + with pytest.raises(GraphQLError, match="permission"): + DeleteLogStreamMutation.mutate(None, _info(), stream_id="s-1") + with pytest.raises(GraphQLError, match="permission"): + RetryLogStreamDeliveryMutation.mutate( + None, _info(), delivery_event_id="d-1" + ) + + stream.delete.assert_not_called() + stream.save.assert_not_called() + mock_pause.assert_not_called() + mock_retry.delay.assert_not_called() + + +def test_id_keyed_mutations_blocked_without_global_access(_global_access): + """A scoped custom role can hold LogStreams permissions — org-wide + egress management still requires global access on every mutation, not + just create.""" + _global_access.return_value = False + org = _org() + stream = _stream_row(org) + delivery = MagicMock() + delivery.stream = stream + + with patch(f"{_M}.LogStream") as MockStream, patch( + f"{_M}.LogStreamDeliveryEvent" + ) as MockDelivery, patch( + f"{_M}.user_has_permission", return_value=True + ), patch( + f"{_M}.engine.pause" + ) as mock_pause, patch( + f"{_M}.engine.retry_delivery" + ) as mock_retry: + MockStream.objects.get.return_value = stream + ( + MockDelivery.objects.select_related.return_value.get.return_value + ) = delivery + + with pytest.raises(GraphQLError, match="global access"): + UpdateLogStreamMutation.mutate( + None, + _info(), + stream_id="s-1", + name="x", + credential_id="c-1", + sources=["org_audit"], + ) + with pytest.raises(GraphQLError, match="global access"): + ToggleLogStreamMutation.mutate(None, _info(), stream_id="s-1") + with pytest.raises(GraphQLError, match="global access"): + DeleteLogStreamMutation.mutate(None, _info(), stream_id="s-1") + with pytest.raises(GraphQLError, match="global access"): + RetryLogStreamDeliveryMutation.mutate( + None, _info(), delivery_event_id="d-1" + ) + + stream.delete.assert_not_called() + stream.save.assert_not_called() + mock_pause.assert_not_called() + mock_retry.delay.assert_not_called() + + +def test_test_connection_blocked_without_permission(): + org = _org() + + with patch(f"{_M}.Organisation") as MockOrg, patch( + f"{_M}.user_has_permission", return_value=False + ), patch(f"{_M}.engine.test_adapter_connection") as mock_test: + MockOrg.objects.get.return_value = org + + with pytest.raises(GraphQLError, match="permission"): + ConnectionTestMutation.mutate( + None, + _info(), + organisation_id=org.id, + provider="datadog", + credential_id="cred-1", + ) + + mock_test.assert_not_called() + + +def test_delete_soft_deletes_and_audits(): + """Delete goes through the model's soft delete (which also cancels the + in-flight ship job) and records an audit event with the stream's config + as old_values. Deliberately NOT plan-gated: a downgraded org must be + able to remove its streams.""" + org = _org() + org.plan = "FR" + stream = _stream_row(org) + + with patch(f"{_M}.LogStream") as MockStream, patch( + f"{_M}.user_has_permission", return_value=True + ), patch( + f"{_M}.get_actor_info_from_graphql", + return_value=("user", "user-1", {}), + ), patch( + f"{_M}.get_resolver_request_meta", return_value=("203.0.113.7", "ua") + ), patch( + f"{_M}.log_audit_event" + ) as mock_audit: + MockStream.objects.get.return_value = stream + + result = DeleteLogStreamMutation.mutate(None, _info(), stream_id="s-1") + + assert result.ok is True + stream.delete.assert_called_once_with() + audit_kwargs = mock_audit.call_args.kwargs + assert audit_kwargs["event_type"] == AuditEvent.DELETE + assert audit_kwargs["resource_type"] == AuditEvent.LOG_STREAM + assert audit_kwargs["old_values"]["name"] == "Datadog prod" + + +def test_retry_rejects_rows_without_event_range(): + stream = MagicMock() + stream.organisation = _org() + stream.deleted_at = None + + delivery = _fresh_delivery(stream) + delivery.cursor_from = None + delivery.cursor_to = None + + with patch(f"{_M}.LogStreamDeliveryEvent") as MockDelivery, patch( + f"{_M}.user_has_permission", return_value=True + ), patch(f"{_M}.can_use_log_streams", return_value=True), patch( + f"{_M}.engine.retry_delivery" + ) as mock_retry: + ( + MockDelivery.objects.select_related.return_value.get.return_value + ) = delivery + + with pytest.raises(GraphQLError, match="no event range"): + RetryLogStreamDeliveryMutation.mutate( + None, _info(), delivery_event_id="d-1" + ) + + mock_retry.delay.assert_not_called() diff --git a/backend/tests/ee/integrations/logs/streams/graphene/test_queries.py b/backend/tests/ee/integrations/logs/streams/graphene/test_queries.py new file mode 100644 index 000000000..f8840908b --- /dev/null +++ b/backend/tests/ee/integrations/logs/streams/graphene/test_queries.py @@ -0,0 +1,38 @@ +"""Log stream queries: permission + global-access gating. + +Streams export org-wide activity, so read access additionally requires a +role with global access — a scoped custom role holding LogStreams:read must +not see org-wide egress configuration or delivery history. +""" + +from unittest.mock import MagicMock, patch + +import pytest +from graphql import GraphQLError + +from ee.integrations.logs.streams.graphene import queries + +_Q = "ee.integrations.logs.streams.graphene.queries" + + +def test_resolve_log_streams_requires_global_access(): + org = MagicMock() + + with patch(f"{_Q}.Organisation") as MockOrg, patch( + f"{_Q}.user_has_permission", return_value=True + ), patch(f"{_Q}.user_has_global_access", return_value=False): + MockOrg.objects.get.return_value = org + + assert queries.resolve_log_streams(None, MagicMock(), "org-1") == [] + + +def test_resolve_deliveries_requires_global_access(): + stream = MagicMock() + + with patch(f"{_Q}.LogStream") as MockStream, patch( + f"{_Q}.user_has_permission", return_value=True + ), patch(f"{_Q}.user_has_global_access", return_value=False): + MockStream.objects.get.return_value = stream + + with pytest.raises(GraphQLError, match="permission"): + queries.resolve_log_stream_deliveries(None, MagicMock(), "s-1") diff --git a/backend/tests/ee/integrations/logs/streams/test_chunker.py b/backend/tests/ee/integrations/logs/streams/test_chunker.py new file mode 100644 index 000000000..1db2cba99 --- /dev/null +++ b/backend/tests/ee/integrations/logs/streams/test_chunker.py @@ -0,0 +1,103 @@ +"""Chunking of serialized envelopes into destination-sized batches.""" + +from datetime import datetime, timedelta, timezone +from unittest.mock import patch + +from ee.integrations.logs.streams import chunker +from ee.integrations.logs.streams.chunker import chunk_envelopes + + +def _entries(n, start=None): + start = start or datetime(2026, 1, 1, tzinfo=timezone.utc) + return [ + { + "envelope": {"event": {"id": f"e{i}"}, "phase": {}}, + "cursor": {"ts": (start + timedelta(seconds=i)).isoformat(), "id": f"e{i}"}, + "timestamp": start + timedelta(seconds=i), + } + for i in range(n) + ] + + +def test_splits_on_event_count(): + with patch.object(chunker, "CHUNK_MAX_EVENTS", 2): + chunks = chunk_envelopes(_entries(5)) + + assert [len(c.events) for c in chunks] == [2, 2, 1] + # Each chunk's cursor markers cover exactly its own events. + assert chunks[0].last_cursor["id"] == "e1" + assert chunks[1].last_cursor["id"] == "e3" + assert chunks[2].last_cursor["id"] == "e4" + assert chunks[0].cursor_from < chunks[0].cursor_to + assert chunks[2].cursor_from == chunks[2].cursor_to + + +def test_splits_on_byte_size(): + entries = _entries(3) + for entry in entries: + entry["envelope"]["phase"]["description"] = "x" * 100 + + # Each envelope is >100 bytes, so a 150-byte cap forces one per chunk. + with patch.object(chunker, "CHUNK_MAX_BYTES", 150): + chunks = chunk_envelopes(entries) + + assert [len(c.events) for c in chunks] == [1, 1, 1] + assert all(c.byte_size > 0 for c in chunks) + + +def test_oversized_event_gets_json_metadata_truncated(): + entries = _entries(1) + entries[0]["envelope"]["phase"]["old_values"] = {"blob": "y" * 500} + entries[0]["envelope"]["phase"]["new_values"] = {"blob": "z" * 500} + + with patch.object(chunker, "EVENT_MAX_BYTES", 200): + chunks = chunk_envelopes(entries) + + envelope = chunks[0].events[0] + assert envelope["phase"]["old_values"] == {"truncated": True} + assert envelope["phase"]["new_values"] == {"truncated": True} + + +def test_empty_input_returns_no_chunks(): + assert chunk_envelopes([]) == [] + + +def test_chunks_record_id_bounds(): + """(timestamp, id) is the event order — auto-resolve containment needs + the id bounds to tell apart chunks that split inside one timestamp.""" + chunks = chunk_envelopes(_entries(3)) + + assert chunks[0].cursor_from_id == "e0" + assert chunks[0].cursor_to_id == "e2" + + +def test_pathological_envelope_is_clamped_below_event_limit(): + """After metadata truncation fails to shrink an envelope, it must be + stripped to an identifying core — EVENT_MAX_BYTES' headroom under the + destination's wire limit only holds for genuinely bounded envelopes, and + an oversize event would otherwise permanently 413 its chunk.""" + huge = "x" * (chunker.EVENT_MAX_BYTES + 100) + entry = { + "envelope": { + "schema_version": 1, + "event": {"id": "e0", "category": "org_audit", "type": "update"}, + "timestamp": "2026-01-01T00:00:00+00:00", + "actor": {"type": "user", "id": "a1", "name": "A"}, + "user_agent": {"original": huge}, + "phase": { + "organisation": {"id": "o1", "name": "acme"}, + "description": "Updated something", + }, + }, + "cursor": {"ts": "2026-01-01T00:00:00+00:00", "id": "e0"}, + "timestamp": datetime(2026, 1, 1, tzinfo=timezone.utc), + } + + chunks = chunk_envelopes([entry]) + + event = chunks[0].events[0] + assert chunks[0].byte_size <= chunker.EVENT_MAX_BYTES + assert event["phase"]["truncated"] is True + # The identifying core survives. + assert event["event"]["id"] == "e0" + assert event["phase"]["organisation"]["name"] == "acme" diff --git a/backend/tests/ee/integrations/logs/streams/test_engine.py b/backend/tests/ee/integrations/logs/streams/test_engine.py new file mode 100644 index 000000000..4919c2265 --- /dev/null +++ b/backend/tests/ee/integrations/logs/streams/test_engine.py @@ -0,0 +1,1407 @@ +"""Shipping engine: retry ladder, cursor semantics, pause/skip behaviour. + +The at-least-once contract under test: +- cursors advance only after a successful ship (or deliberately, past an + exhausted chunk / a stale skip-ahead range), +- auth errors pause the stream instead of burning retries, +- manual retries resolve the original delivery record without touching the + live cursor. +""" + +from datetime import datetime, timedelta, timezone as dt_timezone +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest +from django.utils import timezone + +from ee.integrations.logs.streams import engine +from ee.integrations.logs.streams.chunker import Chunk +from ee.integrations.logs.streams.adapters.base import ShipResult +from ee.integrations.logs.streams.exceptions import ( + AdapterAuthError, + AdapterPermanentError, + AdapterRateLimitedError, + AdapterTransientError, +) + +_E = "ee.integrations.logs.streams.engine" + +_TS = datetime(2026, 7, 30, 12, 0, 0, tzinfo=dt_timezone.utc) + + +# Bound at test-module import, BEFORE the autouse fixture patches the module +# attribute — lets liveness-semantics tests exercise the real implementation. +_real_stream_is_shippable = engine._stream_is_shippable + + +@pytest.fixture(autouse=True) +def _shippable(): + """Delivery attempts re-check stream liveness/config against the DB; + default it to shippable so each test exercises its own concern. Liveness + tests override the return value or call the real implementation.""" + with patch(f"{_E}._stream_is_shippable", return_value=True) as mock_shippable: + yield mock_shippable + + +def _stream(**overrides): + stream = MagicMock() + stream.id = "stream-1" + stream.name = "test-stream" + stream.provider = "datadog" + stream.HEALTHY = "healthy" + stream.DEGRADED = "degraded" + stream.health = "healthy" + stream.cursors = {} + stream.sources = ["org_audit"] + stream.max_attempts = 3 + stream.is_active = True + stream.paused_reason = "" + stream.authentication_id = "cred-1" + stream.organisation = MagicMock() + stream.organisation.name = "Acme" + stream.created_at = _TS + stream.deleted_at = None + for key, value in overrides.items(): + setattr(stream, key, value) + return stream + + +def _chunk(): + return Chunk( + events=[{"event": {"id": "e1"}}, {"event": {"id": "e2"}}], + byte_size=256, + cursor_from=_TS, + cursor_to=_TS + timedelta(seconds=5), + cursor_from_id="e1", + cursor_to_id="e2", + last_cursor={"ts": (_TS + timedelta(seconds=5)).isoformat(), "id": "e2"}, + ) + + +def _adapter(ship=None, max_event_age=timedelta(hours=18)): + adapter = MagicMock() + adapter.max_event_age = max_event_age + if ship is not None: + adapter.ship = ship + return adapter + + +# --------------------------------------------------------------------------- +# record_delivery — badge policy +# --------------------------------------------------------------------------- + + +def test_record_delivery_pre_resolves_informational_rows(): + """The out-of-sync badge counts unresolved rows, which must all be + actionable. Stream-level rows and FAILED retry-attempt children (the + original stays the open item) are terminal outcomes — stored resolved.""" + delivery_model = MagicMock() + stream = _stream() + + with patch(f"{_E}.apps.get_model", return_value=delivery_model): + engine.record_delivery(stream, "", engine.STATUS_FAILED, meta={}) + assert delivery_model.objects.create.call_args.kwargs["resolved_at"] is not None + + engine.record_delivery( + stream, + "org_audit", + engine.STATUS_FAILED, + retried_from=MagicMock(), + meta={}, + ) + assert delivery_model.objects.create.call_args.kwargs["resolved_at"] is not None + + # A ranged live failure is the actionable record — stays unresolved. + engine.record_delivery(stream, "org_audit", engine.STATUS_FAILED, meta={}) + assert "resolved_at" not in delivery_model.objects.create.call_args.kwargs + + # The skipped head of a partial-expiry retry is the durable loss + # record — stays unresolved even though it is a child row. + engine.record_delivery( + stream, + "org_audit", + engine.STATUS_SKIPPED, + retried_from=MagicMock(), + meta={}, + ) + assert "resolved_at" not in delivery_model.objects.create.call_args.kwargs + + +# --------------------------------------------------------------------------- +# _deliver_chunk — the retry ladder +# --------------------------------------------------------------------------- + + +def test_deliver_chunk_success_first_attempt(): + result = ShipResult(status_code=202, duration_ms=42) + adapter = _adapter(ship=MagicMock(return_value=result)) + + outcome, attempts, info = engine._deliver_chunk( + _stream(), _chunk(), adapter, {}, {}, {} + ) + + assert (outcome, attempts, info) == (engine.DELIVERED, 1, result) + + +def test_deliver_chunk_transient_backoff_then_success(): + result = ShipResult(status_code=202) + adapter = _adapter( + ship=MagicMock( + side_effect=[AdapterTransientError("503"), AdapterTransientError("503"), result] + ) + ) + + with patch(f"{_E}.time.sleep") as mock_sleep: + outcome, attempts, _ = engine._deliver_chunk( + _stream(), _chunk(), adapter, {}, {}, {} + ) + + assert outcome == engine.DELIVERED + assert attempts == 3 + assert [call.args[0] for call in mock_sleep.call_args_list] == [5, 15] + + +def test_deliver_chunk_honors_retry_after(): + result = ShipResult(status_code=202) + adapter = _adapter( + ship=MagicMock( + side_effect=[AdapterRateLimitedError("429", retry_after=7), result] + ) + ) + + with patch(f"{_E}.time.sleep") as mock_sleep: + outcome, attempts, _ = engine._deliver_chunk( + _stream(), _chunk(), adapter, {}, {}, {} + ) + + assert outcome == engine.DELIVERED + assert attempts == 2 + mock_sleep.assert_called_once_with(7) + + +def test_deliver_chunk_auth_error_short_circuits(): + adapter = _adapter(ship=MagicMock(side_effect=AdapterAuthError("401"))) + + with patch(f"{_E}.time.sleep") as mock_sleep: + outcome, attempts, info = engine._deliver_chunk( + _stream(), _chunk(), adapter, {}, {}, {} + ) + + assert outcome == engine.AUTH_ERROR + assert attempts == 1 + assert isinstance(info, AdapterAuthError) + mock_sleep.assert_not_called() + + +def test_deliver_chunk_permanent_error_does_not_retry(): + adapter = _adapter(ship=MagicMock(side_effect=AdapterPermanentError("400"))) + + with patch(f"{_E}.time.sleep") as mock_sleep: + outcome, attempts, _ = engine._deliver_chunk( + _stream(), _chunk(), adapter, {}, {}, {} + ) + + assert outcome == engine.EXHAUSTED + assert attempts == 1 + mock_sleep.assert_not_called() + + +def test_deliver_chunk_stops_at_wall_clock_deadline(): + """Retry-After sleeps are honoured only inside the delivery deadline — + otherwise a rate-limited destination could stretch one chunk past the + ingestion-window margin and get its events accepted-then-dropped.""" + adapter = _adapter( + ship=MagicMock(side_effect=AdapterRateLimitedError("429", retry_after=300)) + ) + + with patch(f"{_E}.time.sleep") as mock_sleep, patch( + f"{_E}.time.monotonic", + side_effect=[0, engine.DELIVERY_DEADLINE_SECONDS + 1], + ): + outcome, attempts, _ = engine._deliver_chunk( + _stream(max_attempts=10), _chunk(), adapter, {}, {}, {} + ) + + assert outcome == engine.EXHAUSTED + assert attempts == 1 + mock_sleep.assert_not_called() + + +def test_deliver_chunk_aborts_when_stream_changes_mid_ladder(_shippable): + """A pause or config change landing during a backoff sleep must stop the + NEXT egress attempt — not wait for the chunk to finish its ladder.""" + _shippable.side_effect = [True, False] + adapter = _adapter(ship=MagicMock(side_effect=AdapterTransientError("503"))) + + with patch(f"{_E}.time.sleep"): + outcome, attempts, info = engine._deliver_chunk( + _stream(max_attempts=5), _chunk(), adapter, {}, {}, {} + ) + + assert outcome == engine.ABORTED + assert attempts == 1 + assert info is None + assert adapter.ship.call_count == 1 + + +def test_ship_chunk_aborted_holds_cursor_and_stops(): + """ABORTED means nothing was egressed and nothing failed — hold position + silently and let the next sweep reload fresh state.""" + stream = _stream() + chunk = _chunk() + + with patch( + f"{_E}._deliver_chunk", return_value=(engine.ABORTED, 0, None) + ), patch(f"{_E}.record_delivery") as mock_record: + outcome = engine._ship_chunk( + stream, "org_audit", chunk, MagicMock(), {}, {}, {} + ) + + assert outcome == engine.PAUSE + assert "org_audit" not in stream.cursors + mock_record.assert_not_called() + + +def test_stream_is_shippable_halts_on_config_or_entitlement_change(): + """The live check must catch pause/delete AND configuration changes (the + job still holds the old sources/credentials/options) AND loss of the + Enterprise plan — not just activity state.""" + stream = _stream( + sources=["org_audit"], + max_attempts=3, + options={"gzip": True}, + authentication_id="cred-1", + ) + + row = { + "sources": ["org_audit"], + "authentication_id": "cred-1", + "options": {"gzip": True}, + "max_attempts": 3, + } + model = MagicMock() + row_query = model.objects.filter.return_value.values.return_value + + with patch(f"{_E}.apps.get_model", return_value=model): + row_query.first.return_value = dict(row) + assert _real_stream_is_shippable(stream) is True + # The Enterprise entitlement is part of the query itself (mirrors + # quotas.can_use_log_streams). + assert model.objects.filter.call_args.kwargs["organisation__plan"] == "EN" + assert model.objects.filter.call_args.kwargs["is_active"] is True + + # Credential rotated by a concurrent update → halt. + row_query.first.return_value = {**row, "authentication_id": "cred-2"} + assert _real_stream_is_shippable(stream) is False + + # Source removed → halt. + row_query.first.return_value = {**row, "sources": []} + assert _real_stream_is_shippable(stream) is False + + # Paused / deleted / plan lost → row filtered out → halt. + row_query.first.return_value = None + assert _real_stream_is_shippable(stream) is False + + +def test_margin_bounds_worst_case_delivery_cycle(): + """Arithmetic guard: if the delivery deadline or the retry job timeout + ever exceeds the skip-ahead margin, events admitted for delivery can + cross the destination's age cutoff mid-flight and be silently discarded + while recorded as delivered.""" + assert ( + engine.DELIVERY_DEADLINE_SECONDS + 300 + <= engine.SKIP_AHEAD_MARGIN.total_seconds() + ) + assert engine.RETRY_JOB_TIMEOUT + 300 <= engine.SKIP_AHEAD_MARGIN.total_seconds() + + +def test_deliver_chunk_exhausts_at_max_attempts(): + adapter = _adapter(ship=MagicMock(side_effect=AdapterTransientError("503"))) + + with patch(f"{_E}.time.sleep") as mock_sleep: + outcome, attempts, _ = engine._deliver_chunk( + _stream(max_attempts=3), _chunk(), adapter, {}, {}, {} + ) + + assert outcome == engine.EXHAUSTED + assert attempts == 3 + # No sleep after the final attempt. + assert mock_sleep.call_count == 2 + + +# --------------------------------------------------------------------------- +# _ship_chunk — state effects +# --------------------------------------------------------------------------- + + +def test_ship_chunk_success_advances_cursor_and_marks_healthy(): + stream = _stream(health="degraded", last_failure_reason="old failure") + chunk = _chunk() + adapter = _adapter(ship=MagicMock(return_value=ShipResult(status_code=202))) + + with patch(f"{_E}.record_delivery") as mock_record: + outcome = engine._ship_chunk(stream, "org_audit", chunk, adapter, {}, {}, {}) + + assert outcome == engine.CONTINUE + assert stream.cursors["org_audit"] == chunk.last_cursor + assert stream.health == "healthy" + assert stream.last_failure_reason == "" + # Delivery-path saves must never write lifecycle fields — the job's row + # is stale relative to a pause/delete the user made while it ran. + update_fields = stream.save.call_args.kwargs["update_fields"] + assert "is_active" not in update_fields + assert "paused_reason" not in update_fields + assert mock_record.call_args.args[2] == engine.STATUS_COMPLETED + + +def test_ship_chunk_auth_error_pauses_and_keeps_cursor(): + stream = _stream() + chunk = _chunk() + adapter = _adapter(ship=MagicMock(side_effect=AdapterAuthError("401"))) + log_stream_model = MagicMock() + + with patch(f"{_E}.record_delivery") as mock_record, patch( + f"{_E}.apps.get_model", return_value=log_stream_model + ): + outcome = engine._ship_chunk(stream, "org_audit", chunk, adapter, {}, {}, {}) + + assert outcome == engine.PAUSE + assert "org_audit" not in stream.cursors + # The pause is written through a targeted queryset update, never a full + # save from the (possibly stale) in-memory row. + update_kwargs = log_stream_model.objects.filter.return_value.update.call_args.kwargs + assert update_kwargs["is_active"] is False + assert update_kwargs["paused_reason"] == "auth_error" + assert update_kwargs["health"] == "degraded" + assert stream.is_active is False + assert stream.paused_reason == "auth_error" + assert stream.health == "degraded" + assert mock_record.call_args.args[2] == engine.STATUS_FAILED + + +def test_ship_chunk_success_auto_resolves_covered_failures(): + """A successful ship covering a previously-failed range resolves the old + failure rows, so the out-of-sync badge doesn't invite a double-shipping + manual retry after an auth recovery. Containment compares (timestamp, id) + bounds — a chunk that merely shares boundary timestamps with a failed + row must NOT resolve events it didn't deliver.""" + from django.db.models import Q + + stream = _stream() + chunk = _chunk() + adapter = _adapter(ship=MagicMock(return_value=ShipResult(status_code=202))) + + delivery_model = MagicMock() + ( + delivery_model.objects.filter.return_value.filter.return_value.filter.return_value.update.return_value + ) = 2 + + with patch(f"{_E}.record_delivery") as mock_record, patch( + f"{_E}.apps.get_model", return_value=delivery_model + ): + engine._ship_chunk(stream, "org_audit", chunk, adapter, {}, {}, {}) + + filter_kwargs = delivery_model.objects.filter.call_args.kwargs + assert filter_kwargs["source"] == "org_audit" + assert filter_kwargs["resolved_at__isnull"] is True + + starts_inside = delivery_model.objects.filter.return_value.filter.call_args.args[0] + ends_inside = ( + delivery_model.objects.filter.return_value.filter.return_value.filter.call_args.args[0] + ) + assert starts_inside == Q(cursor_from__gt=chunk.cursor_from) | ( + Q(cursor_from=chunk.cursor_from) + & (Q(cursor_from_id__gte="e1") | Q(cursor_from_id="")) + ) + assert ends_inside == Q(cursor_to__lt=chunk.cursor_to) | ( + Q(cursor_to=chunk.cursor_to) + & (Q(cursor_to_id__lte="e2") | Q(cursor_to_id="")) + ) + assert mock_record.call_args.kwargs["meta"]["auto_resolved"] == 2 + assert mock_record.call_args.kwargs["cursor_from_id"] == "e1" + assert mock_record.call_args.kwargs["cursor_to_id"] == "e2" + + +def test_ship_chunk_exhaustion_skips_forward_and_degrades(): + stream = _stream() + chunk = _chunk() + adapter = _adapter(ship=MagicMock(side_effect=AdapterTransientError("503"))) + + with patch(f"{_E}.record_delivery") as mock_record, patch(f"{_E}.time.sleep"): + outcome = engine._ship_chunk(stream, "org_audit", chunk, adapter, {}, {}, {}) + + assert outcome == engine.CONTINUE + # Cursor advances PAST the failed chunk — no head-of-line blocking. The + # range is recorded and stays manually re-shippable. + assert stream.cursors["org_audit"] == chunk.last_cursor + assert stream.health == "degraded" + assert stream.is_active is True + assert mock_record.call_args.args[2] == engine.STATUS_FAILED + + +def test_ship_chunk_exhaustion_holds_cursor_when_failure_record_is_lost(): + """If the failed-range record can't be persisted, advancing the cursor + would silently lose the events with no re-shippable trace — the source + must halt at the current cursor instead.""" + stream = _stream() + chunk = _chunk() + adapter = _adapter(ship=MagicMock(side_effect=AdapterPermanentError("400"))) + + with patch(f"{_E}.record_delivery", return_value=None): + outcome = engine._ship_chunk(stream, "org_audit", chunk, adapter, {}, {}, {}) + + assert outcome == engine.HALT + assert "org_audit" not in stream.cursors + assert stream.health == "degraded" + + +# --------------------------------------------------------------------------- +# _skip_ahead — max-event-age floor +# --------------------------------------------------------------------------- + + +def test_skip_ahead_records_skipped_range_and_floors_cursor(): + stale = timezone.now() - timedelta(hours=30) + stream = _stream(cursors={"org_audit": {"ts": stale.isoformat(), "id": ""}}) + source = MagicMock() + source.count_before.return_value = 42 + + with patch(f"{_E}.record_delivery") as mock_record: + proceed = engine._skip_ahead(stream, "org_audit", source, _adapter()) + + assert proceed is True + assert mock_record.call_args.args[2] == engine.STATUS_SKIPPED + assert mock_record.call_args.kwargs["event_count"] == 42 + new_cursor_ts = datetime.fromisoformat(stream.cursors["org_audit"]["ts"]) + assert timezone.now() - new_cursor_ts < timedelta(hours=18) + + +def test_skip_ahead_halts_source_when_record_is_lost(): + """No skipped-range record → no trace of the loss. The cursor must hold + AND the source must halt (returns False) — otherwise the caller would + fetch the expired backlog from the stale cursor and ship events the + destination accepts-then-drops, advancing past them as 'completed'.""" + stale = timezone.now() - timedelta(hours=30) + cursor = {"ts": stale.isoformat(), "id": ""} + stream = _stream(cursors={"org_audit": cursor}) + source = MagicMock() + source.count_before.return_value = 42 + + with patch(f"{_E}.record_delivery", return_value=None): + proceed = engine._skip_ahead(stream, "org_audit", source, _adapter()) + + assert proceed is False + assert stream.cursors["org_audit"] == cursor + stream.save.assert_not_called() + + +def test_skip_ahead_noop_when_cursor_is_fresh(): + fresh = timezone.now() - timedelta(minutes=5) + cursor = {"ts": fresh.isoformat(), "id": ""} + stream = _stream(cursors={"org_audit": cursor}) + source = MagicMock() + + with patch(f"{_E}.record_delivery") as mock_record: + engine._skip_ahead(stream, "org_audit", source, _adapter()) + + mock_record.assert_not_called() + assert stream.cursors["org_audit"] == cursor + source.count_before.assert_not_called() + + +def test_skip_ahead_noop_without_max_event_age(): + stale = timezone.now() - timedelta(days=30) + cursor = {"ts": stale.isoformat(), "id": ""} + stream = _stream(cursors={"org_audit": cursor}) + + with patch(f"{_E}.record_delivery") as mock_record: + engine._skip_ahead(stream, "org_audit", MagicMock(), _adapter(max_event_age=None)) + + mock_record.assert_not_called() + assert stream.cursors["org_audit"] == cursor + + +# --------------------------------------------------------------------------- +# sweep — plan gate + overlap protection +# --------------------------------------------------------------------------- + + +def _sweep_setup(streams, stranded=()): + log_stream_model = MagicMock() + + active_qs = MagicMock() + active_qs.select_related.return_value.order_by.return_value = streams + + stranded_qs = MagicMock() + stranded_qs.__iter__ = MagicMock(return_value=iter(stranded)) + + def filter_side_effect(**kwargs): + if kwargs.get("authentication__isnull") is True: + return stranded_qs + return active_qs + + log_stream_model.objects.filter.side_effect = filter_side_effect + return log_stream_model + + +def test_sweep_gates_on_plan_and_running_jobs(): + stream_gated = _stream(id="s1") + stream_running = _stream(id="s2", ship_job_id="job-2") + stream_ok = _stream(id="s3", ship_job_id=None) + + running_job = MagicMock(is_queued=False, is_started=True) + + with patch(f"{_E}.apps.get_model", return_value=_sweep_setup( + [stream_gated, stream_running, stream_ok] + )), patch( + f"{_E}.can_use_log_streams", side_effect=[False, True, True] + ), patch( + f"{_E}.Job.fetch", return_value=running_job + ), patch.object( + engine.ship_log_stream, "delay", return_value=MagicMock(get_id=lambda: "job-3") + ) as mock_delay, patch( + f"{_E}._queue" + ), patch( + f"{_E}._cleanup_delivery_events" + ): + engine.sweep_log_streams() + + mock_delay.assert_called_once_with("s3") + assert stream_ok.ship_job_id == "job-3" + + +def test_sweep_warns_when_ship_jobs_starve_for_workers(): + """A ship job still *queued* a full sweep later means the worker pool is + saturated — the sweep logs the LOG_STREAM_WORKERS capacity signal.""" + stream_starved = _stream(id="s1", ship_job_id="job-1") + + queued_job = MagicMock(is_queued=True, is_started=False) + + with patch(f"{_E}.apps.get_model", return_value=_sweep_setup([stream_starved])), patch( + f"{_E}.can_use_log_streams", return_value=True + ), patch(f"{_E}.Job.fetch", return_value=queued_job), patch.object( + engine.ship_log_stream, "delay" + ) as mock_delay, patch( + f"{_E}._queue" + ), patch( + f"{_E}._cleanup_delivery_events" + ), patch( + f"{_E}.logger" + ) as mock_logger: + engine.sweep_log_streams() + + mock_delay.assert_not_called() + warning_text = mock_logger.warning.call_args.args[0] + assert "LOG_STREAM_WORKERS" in warning_text + assert mock_logger.warning.call_args.args[1] == 1 + + +def test_sweep_pauses_streams_with_deleted_credentials(): + """A hard-deleted credential row leaves authentication NULL (SET_NULL) — + the sweep must pause such streams visibly instead of leaving them + 'healthy' while they silently ship nothing.""" + stranded = _stream(id="s1", authentication_id=None) + + with patch( + f"{_E}.apps.get_model", return_value=_sweep_setup([], stranded=[stranded]) + ), patch(f"{_E}.record_delivery") as mock_record, patch( + f"{_E}._pause_stream_row" + ) as mock_pause, patch( + f"{_E}._queue" + ), patch( + f"{_E}._cleanup_delivery_events" + ): + engine.sweep_log_streams() + + assert mock_record.call_args.args[2] == engine.STATUS_FAILED + assert mock_record.call_args.kwargs["meta"]["error"] == "credentials_missing" + assert mock_pause.call_args.args[1] == "credentials_missing" + + +def test_ship_stream_stops_when_paused_mid_job(): + """rq can't stop a started job — the chunk loop itself must notice a + concurrent pause/delete via a live DB check and stop shipping.""" + stream = _stream() + source = MagicMock() + source.fetch.return_value = [SimpleNamespace(timestamp=_TS)] + source.serialize.return_value = {"event": {"id": "e"}} + source.cursor_of.return_value = {"ts": _TS.isoformat(), "id": "e"} + + with patch(f"{_E}.get_adapter", return_value=_adapter(max_event_age=None)), patch( + f"{_E}.get_credentials", return_value={} + ), patch(f"{_E}.get_source", return_value=source), patch( + f"{_E}.lag_for", return_value=0 + ), patch( + f"{_E}._stream_is_shippable", return_value=False + ), patch( + f"{_E}._ship_chunk" + ) as mock_ship_chunk: + engine._ship_stream(stream) + + mock_ship_chunk.assert_not_called() + + +def test_ship_stream_ships_one_chunk_per_tail_iteration(): + """A byte-split fetch must not ship its later chunks under the age floor + computed for the first — each iteration re-floors and refetches from the + advanced cursor instead.""" + stream = _stream() + source = MagicMock() + source.fetch.side_effect = [[SimpleNamespace(timestamp=_TS)], []] + source.serialize.return_value = {"event": {"id": "e"}} + source.cursor_of.return_value = {"ts": _TS.isoformat(), "id": "e"} + + chunk_a, chunk_b = _chunk(), _chunk() + + with patch(f"{_E}.get_adapter", return_value=_adapter(max_event_age=None)), patch( + f"{_E}.get_credentials", return_value={} + ), patch(f"{_E}.get_source", return_value=source), patch( + f"{_E}.lag_for", return_value=0 + ), patch( + f"{_E}.chunk_envelopes", return_value=[chunk_a, chunk_b] + ), patch( + f"{_E}._stream_is_shippable", return_value=True + ), patch( + f"{_E}._ship_chunk", return_value=engine.CONTINUE + ) as mock_ship_chunk: + engine._ship_stream(stream) + + # Only the FIRST chunk of the fetch ships; the loop re-fetches (empty on + # the second iteration here) instead of shipping chunk_b stale. + assert mock_ship_chunk.call_count == 1 + assert mock_ship_chunk.call_args.args[2] is chunk_a + assert source.fetch.call_count == 2 + + +def test_ship_log_stream_noops_without_redis_lock(): + stream = _stream() + log_stream_model = MagicMock() + ( + log_stream_model.objects.filter.return_value.select_related.return_value.first.return_value + ) = stream + + conn = MagicMock() + conn.set.return_value = False # lock held elsewhere + + queue = MagicMock() + queue.connection = conn + + with patch(f"{_E}.apps.get_model", return_value=log_stream_model), patch( + f"{_E}._queue", return_value=queue + ), patch(f"{_E}._ship_stream") as mock_ship: + engine.ship_log_stream("stream-1") + + mock_ship.assert_not_called() + + +# --------------------------------------------------------------------------- +# retry_delivery — manual backfill +# --------------------------------------------------------------------------- + + +def _retry_original(stream, cursor_from, cursor_to): + original = MagicMock() + original.status = "failed" + original.resolved_at = None + original.source = "org_audit" + original.stream = stream + original.cursor_from = cursor_from + original.cursor_to = cursor_to + return original + + +def _retry_setup(original, source_events=1): + delivery_model = MagicMock() + delivery_model.FAILED = "failed" + delivery_model.SKIPPED = "skipped" + delivery_model.COMPLETED = "completed" + ( + delivery_model.objects.filter.return_value.select_related.return_value.first.return_value + ) = original + + source = MagicMock() + base_ts = original.cursor_from if original.cursor_from else _TS + source.fetch_range.return_value = [ + SimpleNamespace(timestamp=base_ts + timedelta(seconds=i)) + for i in range(source_events) + ] + source.serialize.return_value = {"event": {"id": "e"}} + source.cursor_of.return_value = {"ts": base_ts.isoformat(), "id": "e"} + + return delivery_model, source + + +def test_retry_delivery_success_resolves_original(): + now = timezone.now() + stream = _stream() + original = _retry_original(stream, now - timedelta(hours=2), now - timedelta(hours=1)) + + delivery_model, source = _retry_setup(original) + + with patch(f"{_E}.apps.get_model", return_value=delivery_model), patch( + f"{_E}._redis" + ), patch( + f"{_E}.get_adapter", return_value=_adapter() + ), patch(f"{_E}.get_source", return_value=source), patch( + f"{_E}.get_credentials", return_value={} + ), patch( + f"{_E}._deliver_chunk", + return_value=(engine.DELIVERED, 1, ShipResult(status_code=202)), + ), patch( + f"{_E}.record_delivery" + ) as mock_record: + engine.retry_delivery("delivery-1") + + assert mock_record.call_args.args[2] == engine.STATUS_COMPLETED + assert mock_record.call_args.kwargs["retried_from"] is original + assert original.resolved_at is not None + original.save.assert_called_once_with(update_fields=["resolved_at"]) + # The stream row is only touched via a targeted queryset update + # (last_shipped_at) — never a full save that could rewind live cursors. + stream.save.assert_not_called() + update_kwargs = delivery_model.objects.filter.return_value.update.call_args.kwargs + assert set(update_kwargs.keys()) == {"last_shipped_at", "updated_at"} + + +def test_retry_delivery_failure_leaves_original_unresolved(): + now = timezone.now() + stream = _stream() + original = _retry_original(stream, now - timedelta(hours=2), now - timedelta(hours=1)) + + delivery_model, source = _retry_setup(original) + + with patch(f"{_E}.apps.get_model", return_value=delivery_model), patch( + f"{_E}._redis" + ), patch( + f"{_E}.get_adapter", return_value=_adapter() + ), patch(f"{_E}.get_source", return_value=source), patch( + f"{_E}.get_credentials", return_value={} + ), patch( + f"{_E}._deliver_chunk", + return_value=(engine.EXHAUSTED, 3, AdapterTransientError("503")), + ), patch( + f"{_E}.record_delivery" + ) as mock_record: + engine.retry_delivery("delivery-1") + + assert mock_record.call_args.args[2] == engine.STATUS_FAILED + assert mock_record.call_args.kwargs["retried_from"] is original + assert original.resolved_at is None + original.save.assert_not_called() + + +def test_retry_delivery_rejects_fully_expired_range(): + """Datadog 202s then silently discards events older than max_event_age — + 'successfully' re-shipping an expired range would falsely mark it + recovered.""" + now = timezone.now() + stream = _stream() + original = _retry_original( + stream, now - timedelta(hours=30), now - timedelta(hours=20) + ) + + delivery_model, source = _retry_setup(original) + + with patch(f"{_E}.apps.get_model", return_value=delivery_model), patch( + f"{_E}._redis" + ), patch( + f"{_E}.get_adapter", return_value=_adapter() + ), patch(f"{_E}.get_source", return_value=source), patch( + f"{_E}.get_credentials", return_value={} + ), patch( + f"{_E}._deliver_chunk" + ) as mock_deliver, patch( + f"{_E}.record_delivery" + ) as mock_record: + engine.retry_delivery("delivery-1") + + mock_deliver.assert_not_called() + assert mock_record.call_args.args[2] == engine.STATUS_FAILED + assert mock_record.call_args.kwargs["meta"]["error"] == "range_expired" + assert original.resolved_at is None + original.save.assert_not_called() + + +def test_retry_delivery_splits_partially_expired_range(): + """Head expired, tail still inside the window: ship the tail, record the + lost head as its own skipped row, resolve the original.""" + now = timezone.now() + stream = _stream() + original = _retry_original( + stream, now - timedelta(hours=30), now - timedelta(hours=1) + ) + + delivery_model, source = _retry_setup(original) + + with patch(f"{_E}.apps.get_model", return_value=delivery_model), patch( + f"{_E}._redis" + ), patch( + f"{_E}.get_adapter", return_value=_adapter() + ), patch(f"{_E}.get_source", return_value=source), patch( + f"{_E}.get_credentials", return_value={} + ), patch( + f"{_E}._deliver_chunk", + return_value=(engine.DELIVERED, 1, ShipResult(status_code=202)), + ), patch( + f"{_E}.record_delivery" + ) as mock_record: + engine.retry_delivery("delivery-1") + + # fetch_range starts at the ingestion-window floor, not the expired head. + fetch_from = source.fetch_range.call_args.args[1] + assert fetch_from > original.cursor_from + assert now - fetch_from < timedelta(hours=18) + + statuses = [call.args[2] for call in mock_record.call_args_list] + assert statuses == [engine.STATUS_SKIPPED, engine.STATUS_COMPLETED] + skipped_kwargs = mock_record.call_args_list[0].kwargs + assert skipped_kwargs["cursor_from"] == original.cursor_from + assert skipped_kwargs["meta"]["reason"] == "max_event_age_exceeded" + assert original.resolved_at is not None + + +def test_retry_delivery_skips_when_claim_held(): + """Duplicate enqueues (double-click, concurrent API calls) must not ship + the same range twice in parallel — the per-delivery Redis claim + serializes them.""" + conn = MagicMock() + conn.set.return_value = False # claim held by another worker + + with patch(f"{_E}._redis", return_value=conn), patch( + f"{_E}._retry_delivery_locked" + ) as mock_locked: + engine.retry_delivery("delivery-1") + + mock_locked.assert_not_called() + + +def test_retry_delivery_stops_mid_job_when_stream_paused(): + """Same live check as the ship path: a pause issued while the retry runs + must stop egress before the next chunk.""" + now = timezone.now() + stream = _stream() + original = _retry_original(stream, now - timedelta(hours=2), now - timedelta(hours=1)) + + delivery_model, source = _retry_setup(original) + + with patch(f"{_E}.apps.get_model", return_value=delivery_model), patch( + f"{_E}._redis" + ), patch( + f"{_E}.get_adapter", return_value=_adapter() + ), patch(f"{_E}.get_source", return_value=source), patch( + f"{_E}.get_credentials", return_value={} + ), patch( + f"{_E}._stream_is_shippable", return_value=False + ), patch( + f"{_E}._deliver_chunk" + ) as mock_deliver: + engine.retry_delivery("delivery-1") + + mock_deliver.assert_not_called() + assert original.resolved_at is None + + +def test_retry_delivery_noops_when_stream_paused(): + """Pause means no egress — manual retries included (the mutation raises + the user-facing error; the job guard covers queued/direct paths).""" + now = timezone.now() + stream = _stream(is_active=False) + original = _retry_original(stream, now - timedelta(hours=2), now - timedelta(hours=1)) + + delivery_model, _ = _retry_setup(original) + + with patch(f"{_E}.apps.get_model", return_value=delivery_model), patch( + f"{_E}._redis" + ), patch( + f"{_E}.get_adapter" + ) as mock_adapter: + engine.retry_delivery("delivery-1") + + mock_adapter.assert_not_called() + + +def test_retry_delivery_keeps_original_open_when_head_record_is_lost(): + """If the skipped-head insert fails during a partial-expiry retry, the + original must stay unresolved — resolving it would erase the only trace + of the expired head's loss.""" + now = timezone.now() + stream = _stream() + original = _retry_original( + stream, now - timedelta(hours=30), now - timedelta(hours=1) + ) + + delivery_model, source = _retry_setup(original) + + def record_side_effect(stream_arg, source_arg, status, **fields): + return None if status == engine.STATUS_SKIPPED else MagicMock() + + with patch(f"{_E}.apps.get_model", return_value=delivery_model), patch( + f"{_E}._redis" + ), patch( + f"{_E}.get_adapter", return_value=_adapter() + ), patch(f"{_E}.get_source", return_value=source), patch( + f"{_E}.get_credentials", return_value={} + ), patch( + f"{_E}._deliver_chunk", + return_value=(engine.DELIVERED, 1, ShipResult(status_code=202)), + ), patch( + f"{_E}.record_delivery", side_effect=record_side_effect + ): + engine.retry_delivery("delivery-1") + + assert original.resolved_at is None + original.save.assert_not_called() + + +def test_retry_delivery_rejects_oversized_range(): + """A range with more events than the cap fails honestly instead of + shipping a subset and falsely resolving the whole original.""" + now = timezone.now() + stream = _stream() + original = _retry_original(stream, now - timedelta(hours=2), now - timedelta(hours=1)) + + delivery_model, source = _retry_setup( + original, source_events=engine.RETRY_MAX_EVENTS + 1 + ) + + with patch(f"{_E}.apps.get_model", return_value=delivery_model), patch( + f"{_E}._redis" + ), patch( + f"{_E}.get_adapter", return_value=_adapter() + ), patch(f"{_E}.get_source", return_value=source), patch( + f"{_E}.get_credentials", return_value={} + ), patch( + f"{_E}._deliver_chunk" + ) as mock_deliver, patch( + f"{_E}.record_delivery" + ) as mock_record: + engine.retry_delivery("delivery-1") + + mock_deliver.assert_not_called() + assert mock_record.call_args.kwargs["meta"]["error"] == "range_too_large" + assert original.resolved_at is None + + +def test_retry_delivery_ignores_resolved_or_completed_rows(): + original = MagicMock() + original.status = "completed" + original.resolved_at = None + + delivery_model, _ = _retry_setup(original) + + with patch(f"{_E}.apps.get_model", return_value=delivery_model), patch( + f"{_E}._redis" + ), patch( + f"{_E}.get_adapter" + ) as mock_adapter: + engine.retry_delivery("delivery-1") + + mock_adapter.assert_not_called() + + +# --------------------------------------------------------------------------- +# lag + pause/resume +# --------------------------------------------------------------------------- + + +def test_lag_for_is_oldest_pending_event_age(): + """Lag = delivery delay (age of the oldest unshipped event), NOT cursor + distance — a fresh event after an idle gap must read ~0, not gap-sized.""" + now = timezone.now() + source = MagicMock() + source.oldest_pending_timestamp.return_value = now - timedelta(seconds=300) + + with patch(f"{_E}.get_source", return_value=source): + lag = engine.lag_for(_stream(), "org_audit") + + assert 295 <= lag <= 305 + + +def test_lag_for_fresh_event_after_idle_gap_reads_near_zero(): + now = timezone.now() + # Cursor is hours old (idle org), but the only pending event just arrived. + stream = _stream( + cursors={"org_audit": {"ts": (now - timedelta(hours=6)).isoformat(), "id": ""}} + ) + source = MagicMock() + source.oldest_pending_timestamp.return_value = now - timedelta(seconds=5) + + with patch(f"{_E}.get_source", return_value=source): + assert engine.lag_for(stream, "org_audit") <= 10 + + +def test_lag_for_returns_zero_when_nothing_pending(): + source = MagicMock() + source.oldest_pending_timestamp.return_value = None + + with patch(f"{_E}.get_source", return_value=source): + assert engine.lag_for(_stream(), "org_audit") == 0 + + +def test_pause_and_resume_roundtrip_preserves_cursor(): + cursor = {"ts": _TS.isoformat(), "id": "e9"} + stream = _stream(cursors={"org_audit": cursor}) + + with patch(f"{_E}.cancel_ship_job") as mock_cancel: + engine.pause(stream) + + assert stream.is_active is False + mock_cancel.assert_called_once_with(stream) + + engine.resume(stream) + + assert stream.is_active is True + assert stream.paused_reason == "" + # Resume never touches cursors — shipping continues where it left off. + assert stream.cursors["org_audit"] == cursor + + +# --------------------------------------------------------------------------- +# Expired-row resolution + delivery history retention +# --------------------------------------------------------------------------- + + +def _delivery_row(provider="datadog", cursor_to=None, meta=None, event_id="d-1"): + row = MagicMock() + row.id = event_id + row.stream = _stream(provider=provider) + row.stream_id = "stream-1" + row.cursor_to = cursor_to + row.created_at = timezone.now() - timedelta(days=3) + row.meta = meta + row.resolved_at = None + return row + + +def _expiry_update_calls(delivery_model): + """(filter_kwargs, update_kwargs) pairs for the per-row conditional + resolution writes (skipping the initial unresolved-set query).""" + updates = [] + for call, update_call in zip( + delivery_model.objects.filter.call_args_list[1:], + delivery_model.objects.filter.return_value.update.call_args_list, + ): + updates.append((call.kwargs, update_call.kwargs)) + return updates + + +def test_resolve_expired_failures_resolves_only_unretryable_rows(): + """A row whose whole range left the ingestion window can never be + re-shipped (retry rejects it as range_expired) — it must be resolved + with meta resolution=expired so the badge stays actionable and the row + ages out under retention. Rows still inside the window stay open, and + the write is a conditional update so a concurrently-written resolution + is never clobbered.""" + now = timezone.now() + # Distinct ids so an inverted window predicate (resolving the live row + # instead) can't pass with the same update count. + expired = _delivery_row(cursor_to=now - timedelta(hours=30), event_id="d-expired") + live = _delivery_row(cursor_to=now - timedelta(hours=1), event_id="d-live") + + delivery_model = MagicMock() + ( + delivery_model.objects.filter.return_value.exclude.return_value.select_related.return_value + ) = [expired, live] + + with patch(f"{_E}.apps.get_model", return_value=delivery_model): + engine._resolve_expired_failures() + + updates = _expiry_update_calls(delivery_model) + assert len(updates) == 1 + filter_kwargs, update_kwargs = updates[0] + assert filter_kwargs == {"id": "d-expired", "resolved_at__isnull": True} + assert update_kwargs["resolved_at"] is not None + assert update_kwargs["meta"]["resolution"] == "expired" + + +def test_expired_resolve_grace_covers_every_adapter_window(): + """The static grace must exceed every registered adapter's + (window - margin + retry-job lifetime) so a row is never auto-resolved + while a retry that passed the window check is still in flight. A future + wide-window adapter that breaks this trips the module-level assert.""" + from ee.integrations.logs.streams.adapters import all_adapters + + for adapter in all_adapters(): + if not adapter.max_event_age: + continue + worst_case = ( + adapter.max_event_age + - engine.SKIP_AHEAD_MARGIN + + timedelta(seconds=engine.RETRY_JOB_TIMEOUT) + ) + assert engine.EXPIRED_RESOLVE_GRACE >= worst_case, adapter.id + + +def test_resolve_expired_failures_skips_unknown_and_windowless_providers(): + """Unknown adapters (unregistered provider) and adapters without an + ingestion window have no expiry — their rows stay open until a retry + or a covering ship resolves them.""" + from types import SimpleNamespace + + now = timezone.now() + unknown = _delivery_row(provider="bogus", cursor_to=now - timedelta(days=5)) + windowless = _delivery_row(provider="webhook", cursor_to=now - timedelta(days=5)) + + def fake_get_adapter(provider): + if provider == "webhook": + return SimpleNamespace(max_event_age=None) + raise ValueError(provider) + + delivery_model = MagicMock() + ( + delivery_model.objects.filter.return_value.exclude.return_value.select_related.return_value + ) = [unknown, windowless] + + with patch(f"{_E}.apps.get_model", return_value=delivery_model), patch( + f"{_E}.get_adapter", side_effect=fake_get_adapter + ): + engine._resolve_expired_failures() + + assert _expiry_update_calls(delivery_model) == [] + + +def test_resolve_expired_failures_defers_young_rows_for_grace_period(): + """The DB filter excludes rows younger than the grace period so the + loss stays visible on the badge first — skip-ahead SKIPPED rows are + born expired and would otherwise never surface there at all.""" + delivery_model = MagicMock() + ( + delivery_model.objects.filter.return_value.exclude.return_value.select_related.return_value + ) = [] + + with patch(f"{_E}.apps.get_model", return_value=delivery_model): + engine._resolve_expired_failures() + + filter_kwargs = delivery_model.objects.filter.call_args.kwargs + assert filter_kwargs["resolved_at__isnull"] is True + assert filter_kwargs["cursor_to__isnull"] is False + grace = timezone.now() - filter_kwargs["created_at__lt"] + tolerance = timedelta(seconds=5) + assert abs(grace - engine.EXPIRED_RESOLVE_GRACE) < tolerance + + +def test_cleanup_skips_when_daily_marker_held(): + """The retention prune runs at most once a day, gated on a Redis + marker — a held marker must mean no delete query at all.""" + conn = MagicMock() + conn.set.return_value = False + delivery_model = MagicMock() + + with patch(f"{_E}._redis", return_value=conn), patch( + f"{_E}.apps.get_model", return_value=delivery_model + ): + engine._cleanup_delivery_events() + + delivery_model.objects.filter.assert_not_called() + + +def test_cleanup_prunes_aged_rows_outside_the_protected_set(): + """Retention shape: the cutoff honours DELIVERY_RETENTION_DAYS, and the + exclusion protects unresolved failed/skipped rows that carry a source + (the re-shippable out-of-sync records). The prune is batched — one + unbounded DELETE can outlive the sweep job's timeout — and the daily + marker is extended only after it completes, so an interrupted prune + retries within the claim TTL instead of skipping a day. Full predicate + semantics need a real database — the suite is DB-less by convention, so + the query structure is pinned here instead.""" + conn = MagicMock() + conn.set.return_value = True + delivery_model = MagicMock() + prunable = delivery_model.objects.filter.return_value.exclude.return_value + # Two batches, then done. + prunable.values_list.return_value.__getitem__.side_effect = [ + ["id-1", "id-2"], + ["id-3"], + [], + ] + + with patch(f"{_E}._redis", return_value=conn), patch( + f"{_E}.apps.get_model", return_value=delivery_model + ): + engine._cleanup_delivery_events() + + cutoff = delivery_model.objects.filter.call_args_list[0].kwargs["created_at__lt"] + retention = timezone.now() - cutoff + tolerance = timedelta(seconds=5) + assert abs(retention - timedelta(days=engine.DELIVERY_RETENTION_DAYS)) < tolerance + + exclusion = str(delivery_model.objects.filter.return_value.exclude.call_args.args[0]) + assert "status__in" in exclusion + assert "resolved_at__isnull" in exclusion + # Stream-level rows (source='') are NOT re-shippable, so the exclusion + # must negate them — otherwise retention would protect them forever. + assert "source" in exclusion + assert "NOT" in exclusion + # The exemption only applies to LIVE streams: a ship job in flight during + # LogStream.delete can record one more unresolved failure after delete's + # resolution pass, and nothing else ever resolves deleted streams' rows. + assert "stream__deleted_at__isnull" in exclusion + + # One DELETE per non-empty batch, each scoped by id__in. + batch_deletes = [ + call for call in delivery_model.objects.filter.call_args_list if "id__in" in call.kwargs + ] + assert [call.kwargs["id__in"] for call in batch_deletes] == [["id-1", "id-2"], ["id-3"]] + + # Claim first (short TTL, nx), extend to the daily interval on success. + claim_call, extend_call = conn.set.call_args_list + assert claim_call.kwargs.get("nx") is True + assert claim_call.kwargs.get("ex") == engine.CLEANUP_CLAIM_SECONDS + assert extend_call.kwargs.get("ex") == 86400 + + +def test_cleanup_does_not_extend_marker_when_prune_fails(): + """A failed prune must leave only the short claim so the next sweep + after the claim expires retries — extending to a day would compound the + backlog.""" + conn = MagicMock() + conn.set.return_value = True + delivery_model = MagicMock() + prunable = delivery_model.objects.filter.return_value.exclude.return_value + prunable.values_list.return_value.__getitem__.side_effect = [["id-1"]] + delivery_model.objects.filter.return_value.delete.side_effect = Exception("db error") + + with patch(f"{_E}._redis", return_value=conn), patch( + f"{_E}.apps.get_model", return_value=delivery_model + ): + engine._cleanup_delivery_events() + + conn.set.assert_called_once() # the claim — no daily extension + + +def test_sweep_runs_expiry_resolution_and_cleanup(): + with patch(f"{_E}.apps.get_model", return_value=_sweep_setup([])), patch( + f"{_E}._resolve_expired_failures" + ) as mock_expire, patch(f"{_E}._cleanup_delivery_events") as mock_cleanup: + engine.sweep_log_streams() + + mock_expire.assert_called_once() + mock_cleanup.assert_called_once() + + +# --------------------------------------------------------------------------- +# untyped/config failure fallbacks — never crash-loop, never leak internals +# --------------------------------------------------------------------------- + + +def test_ship_stream_pauses_on_unknown_provider(): + """A stream whose provider has no registered adapter can never ship — + it must pause visibly instead of raising into the generic handler and + crash-looping on every 30s sweep.""" + stream = _stream(provider="not-a-provider") + + with patch( + f"{_E}.get_adapter", side_effect=ValueError("Unknown provider") + ), patch(f"{_E}._pause_stream_row") as mock_pause, patch( + f"{_E}.record_delivery" + ) as mock_record: + engine._ship_stream(stream) + + assert mock_pause.call_args.args[1] == "unknown_provider" + assert mock_record.call_args.args[1] == "" # stream-level row + assert mock_record.call_args.kwargs["meta"] == {"error": "unknown_provider"} + + +def test_ship_stream_pauses_on_invalid_options(): + """Options are validated at create/update, so a validation failure at + ship time is a deterministic config error — pause, don't crash-loop.""" + adapter = _adapter() + adapter.validate_options.side_effect = ValueError("bad options") + stream = _stream() + + with patch(f"{_E}.get_adapter", return_value=adapter), patch( + f"{_E}.get_credentials", return_value={} + ), patch(f"{_E}._pause_stream_row") as mock_pause, patch( + f"{_E}.record_delivery" + ) as mock_record: + engine._ship_stream(stream) + + assert mock_pause.call_args.args[1] == "invalid_options" + assert mock_record.call_args.kwargs["meta"] == {"error": "invalid_options"} + + +def test_deliver_chunk_wraps_untyped_adapter_exception_as_transient(): + """Adapters must raise typed errors (base.py contract); an untyped + escape is an adapter bug and must burn the chunk through the normal + EXHAUSTED path — crashing the job would hold the cursor and head-of-line + block the stream forever on a deterministic bug.""" + stream = _stream(max_attempts=1) + adapter = _adapter(ship=MagicMock(side_effect=TypeError("boom"))) + + outcome, attempts, info = engine._deliver_chunk( + stream, _chunk(), adapter, {}, {}, {} + ) + + assert outcome == engine.EXHAUSTED + assert attempts == 1 + assert isinstance(info, AdapterTransientError) + # The raw exception text stays in the server logs, not the user_message. + assert "boom" not in info.user_message + + +def test_deliver_chunk_reraises_job_timeout(): + """JobTimeoutException must escape the untyped-exception wrap so the + ship job's own handler records the timeout.""" + from rq.timeouts import JobTimeoutException + + stream = _stream(max_attempts=3) + adapter = _adapter(ship=MagicMock(side_effect=JobTimeoutException("timeout"))) + + with pytest.raises(JobTimeoutException): + engine._deliver_chunk(stream, _chunk(), adapter, {}, {}, {}) + + +def test_ship_job_crash_reason_is_sanitized(): + """Raw exception strings can carry driver/SQL internals and both + meta.error and last_failure_reason are user-visible — the crash handler + stores the class name only.""" + stream = _stream() + log_stream_model = MagicMock() + ( + log_stream_model.objects.filter.return_value.select_related.return_value.first.return_value + ) = stream + conn = MagicMock() + conn.set.return_value = True + queue = MagicMock() + queue.connection = conn + + with patch(f"{_E}.apps.get_model", return_value=log_stream_model), patch( + f"{_E}._queue", return_value=queue + ), patch( + f"{_E}._ship_stream", side_effect=Exception("SELECT secret FROM users failed") + ), patch(f"{_E}._degrade_once") as mock_degrade: + engine.ship_log_stream("stream-1") + + error_code, reason = mock_degrade.call_args.args[1:3] + assert error_code == "ship_job_crashed: Exception" + assert "SELECT" not in reason + + +def test_retry_delivery_records_failure_when_setup_crashes(): + """The user already saw "retry queued" — a crash during retry setup + (fetch_range, serialization) must still leave a FAILED trace instead of + dying with only the Redis claim released.""" + now = timezone.now() + stream = _stream() + original = _retry_original(stream, now - timedelta(hours=2), now - timedelta(hours=1)) + + delivery_model, source = _retry_setup(original) + source.fetch_range.side_effect = Exception("db down") + + with patch(f"{_E}.apps.get_model", return_value=delivery_model), patch( + f"{_E}._redis" + ), patch( + f"{_E}.get_adapter", return_value=_adapter() + ), patch(f"{_E}.get_source", return_value=source), patch( + f"{_E}.get_credentials", return_value={} + ), patch( + f"{_E}.record_delivery" + ) as mock_record: + engine.retry_delivery("delivery-1") + + assert mock_record.call_args.args[2] == engine.STATUS_FAILED + assert mock_record.call_args.kwargs["retried_from"] is original + # Sanitized: class name only, no raw exception text. + assert mock_record.call_args.kwargs["meta"]["error"] == "retry_failed: Exception" + assert original.resolved_at is None diff --git a/backend/tests/ee/integrations/logs/streams/test_jobs.py b/backend/tests/ee/integrations/logs/streams/test_jobs.py new file mode 100644 index 000000000..7d12c1bc9 --- /dev/null +++ b/backend/tests/ee/integrations/logs/streams/test_jobs.py @@ -0,0 +1,37 @@ +"""Sweeper bootstrap: stable job id, cancel-before-schedule.""" + +from unittest.mock import MagicMock, patch + +from ee.integrations.logs.streams.jobs import SWEEP_JOB_ID, init_log_stream_sweeper + +_M = "ee.integrations.logs.streams.jobs" + + +def test_sweeper_registers_with_stable_id_and_cancels_prior(): + scheduler = MagicMock() + + with patch(f"{_M}.django_rq.get_scheduler", return_value=scheduler): + init_log_stream_sweeper() + + # Cancel any prior registration first — repeated migrates must replace the + # schedule, not accumulate duplicates (the licensing job's known wart). + scheduler.cancel.assert_called_once_with(SWEEP_JOB_ID) + + _, kwargs = scheduler.schedule.call_args + assert kwargs["id"] == SWEEP_JOB_ID + assert kwargs["interval"] == 30 + assert kwargs["repeat"] is None + # -1 = job hash never expires. Interval jobs keep their schedule metadata + # on the hash; a finite TTL lets a host freeze expire it, after which + # rq-scheduler silently drops the schedule (the 11h-stall incident). + assert kwargs["result_ttl"] == -1 + + +def test_sweeper_survives_cancel_failure(): + scheduler = MagicMock() + scheduler.cancel.side_effect = Exception("nothing to cancel") + + with patch(f"{_M}.django_rq.get_scheduler", return_value=scheduler): + init_log_stream_sweeper() + + assert scheduler.schedule.called diff --git a/backend/tests/ee/integrations/logs/streams/test_models.py b/backend/tests/ee/integrations/logs/streams/test_models.py new file mode 100644 index 000000000..3020e23ce --- /dev/null +++ b/backend/tests/ee/integrations/logs/streams/test_models.py @@ -0,0 +1,32 @@ +"""LogStream model lifecycle behaviour.""" + +from unittest.mock import MagicMock, patch + +from api.models import LogStream + + +def test_stream_delete_resolves_open_delivery_rows(): + """A deleted stream's failed/skipped ranges can never be re-shipped, and + both auto-resolve and retention skip deleted streams — delete must + resolve them or they are exempt from retention forever (unbounded table + growth across create/fail/delete cycles).""" + stream = LogStream() + events = MagicMock() + + # The reverse manager descriptor is replaced at class level, keeping the + # test DB-less. + with patch.object(LogStream, "delivery_events", events), patch.object( + LogStream, "save" + ), patch( + "ee.integrations.logs.streams.engine.cancel_ship_job" + ) as mock_cancel: + stream.delete() + + filter_kwargs = events.filter.call_args.kwargs + assert set(filter_kwargs["status__in"]) == {"failed", "skipped"} + assert filter_kwargs["resolved_at__isnull"] is True + assert events.filter.return_value.update.call_args.kwargs["resolved_at"] is not None + + assert stream.deleted_at is not None + assert stream.is_active is False + mock_cancel.assert_called_once_with(stream) diff --git a/backend/tests/ee/integrations/logs/streams/test_serializers.py b/backend/tests/ee/integrations/logs/streams/test_serializers.py new file mode 100644 index 000000000..a3c526dd8 --- /dev/null +++ b/backend/tests/ee/integrations/logs/streams/test_serializers.py @@ -0,0 +1,214 @@ +"""Envelope serialization. + +The hard rule under test: SecretEvent's E2EE ciphertext fields (key, value, +comment) and the key digest must never appear in an exported envelope. +""" + +import json +from datetime import datetime, timezone +from types import SimpleNamespace + +from ee.integrations.logs.streams.serializers import ( + audit_event_to_envelope, + secret_event_to_envelope, +) + +_TS = datetime(2026, 7, 30, 12, 0, 0, tzinfo=timezone.utc) + + +def _org(): + return SimpleNamespace(id="org-1", name="Acme Corp") + + +def _audit_event(**overrides): + defaults = dict( + id="evt-1", + event_type="R", + actor_type="user", + actor_id="member-1", + actor_metadata={ + "email": "dev@example.com", + "username": "dev", + "token": {"id": "tok-1", "name": "ci-token", "type": "user_token"}, + }, + resource_type="app", + resource_id="app-1", + resource_metadata={"name": "backend"}, + old_values=None, + new_values={"name": "backend"}, + description="Read app backend", + ip_address="203.0.113.7", + user_agent="phase-cli/1.18", + timestamp=_TS, + ) + defaults.update(overrides) + return SimpleNamespace(**defaults) + + +def _secret_event(**overrides): + environment = SimpleNamespace( + id="env-1", + name="Production", + env_type="PROD", + app=SimpleNamespace( + id="app-1", name="backend", organisation=SimpleNamespace(name="Acme Corp") + ), + ) + user = SimpleNamespace( + user=SimpleNamespace(email="dev@example.com", full_name="Dev Eloper") + ) + defaults = dict( + id="sev-1", + event_type="R", + secret_id="secret-1", + environment=environment, + path="/api/payments", + version=3, + type="secret", + # E2EE ciphertext — must never leak into the envelope + key="CIPHERTEXT_KEY_SENTINEL", + key_digest="DIGEST_SENTINEL", + value="CIPHERTEXT_VALUE_SENTINEL", + comment="CIPHERTEXT_COMMENT_SENTINEL", + user_id="member-1", + user=user, + service_account_id=None, + service_account=None, + service_account_token_id=None, + service_account_token=None, + service_token_id=None, + service_token=None, + ip_address="203.0.113.7", + user_agent="phase-cli/1.18", + timestamp=_TS, + ) + defaults.update(overrides) + return SimpleNamespace(**defaults) + + +def test_audit_envelope_shape(): + envelope = audit_event_to_envelope(_audit_event(), _org()) + + assert envelope["schema_version"] == 1 + assert envelope["event"] == {"id": "evt-1", "category": "org_audit", "type": "read"} + assert envelope["timestamp"] == _TS.isoformat() + assert envelope["client"]["address"] == "203.0.113.7" + assert envelope["user_agent"]["original"] == "phase-cli/1.18" + assert envelope["user"] == {"id": "member-1", "email": "dev@example.com", "name": "dev"} + assert envelope["actor"]["token"]["name"] == "ci-token" + assert envelope["phase"]["organisation"] == {"id": "org-1", "name": "Acme Corp"} + assert envelope["phase"]["resource"]["type"] == "app" + + +def test_audit_envelope_ships_readable_resource_type_and_fallback_description(): + """DB resource codes ("rs", "pat"…) never leak into exports, and events + whose call site omitted a description still get a readable content line.""" + event = _audit_event( + event_type="U", + resource_type="rs", + resource_metadata={"name": "stripe-key"}, + description="", + ) + envelope = audit_event_to_envelope(event, _org()) + + assert envelope["phase"]["resource"]["type"] == "rotating_secret" + assert envelope["phase"]["description"] == "Updated rotating secret 'stripe-key'" + + +def test_audit_envelope_fallback_description_without_resource_name(): + event = _audit_event( + event_type="D", resource_type="policy", resource_metadata={}, description="" + ) + envelope = audit_event_to_envelope(event, _org()) + + assert envelope["phase"]["resource"]["type"] == "network_access_policy" + assert envelope["phase"]["description"] == "Deleted network access policy" + + +def test_audit_envelope_service_account_actor_has_no_user_block(): + event = _audit_event(actor_type="sa", actor_metadata={"name": "ci-bot"}) + envelope = audit_event_to_envelope(event, _org()) + + assert envelope["actor"]["type"] == "service_account" + assert envelope["actor"]["name"] == "ci-bot" + assert "user" not in envelope + + +def test_secret_envelope_shape(): + envelope = secret_event_to_envelope(_secret_event(), _org()) + + assert envelope["event"] == {"id": "sev-1", "category": "secrets", "type": "read"} + assert envelope["phase"]["secret"] == { + "id": "secret-1", + "path": "/api/payments", + "version": 3, + "type": "secret", + } + assert envelope["phase"]["app"] == {"id": "app-1", "name": "backend"} + assert envelope["phase"]["environment"]["name"] == "Production" + assert envelope["user"] == { + "id": "member-1", + "email": "dev@example.com", + "name": "Dev Eloper", + } + # The description is the destination's human-scannable content line — + # locates the secret without its E2EE name. + assert ( + envelope["phase"]["description"] + == "Secret read in backend / Production at /api/payments by Dev Eloper" + ) + + +def test_secret_envelope_never_ships_ciphertext_or_digest(): + envelope = secret_event_to_envelope(_secret_event(), _org()) + serialized = json.dumps(envelope) + + for sentinel in ( + "CIPHERTEXT_KEY_SENTINEL", + "DIGEST_SENTINEL", + "CIPHERTEXT_VALUE_SENTINEL", + "CIPHERTEXT_COMMENT_SENTINEL", + ): + assert sentinel not in serialized + + secret_block = envelope["phase"]["secret"] + for forbidden in ("key", "key_digest", "value", "comment"): + assert forbidden not in secret_block + + +def test_secret_envelope_service_account_actor(): + event = _secret_event( + user_id=None, + user=None, + service_account_id="sa-1", + service_account=SimpleNamespace(name="deploy-bot"), + service_account_token_id="sat-1", + service_account_token=SimpleNamespace(name="gh-actions"), + ) + envelope = secret_event_to_envelope(event, _org()) + + assert envelope["actor"]["type"] == "service_account" + assert envelope["actor"]["name"] == "deploy-bot" + assert envelope["actor"]["token"]["name"] == "gh-actions" + assert "user" not in envelope + + +def test_secret_envelope_engine_driven_event_renders_phase_actor(): + event = _secret_event(user_id=None, user=None) + envelope = secret_event_to_envelope(event, _org()) + + assert envelope["actor"] == {"type": "phase", "id": "", "name": "Phase"} + assert ( + envelope["phase"]["description"] + == "Secret read in backend / Production at /api/payments by Phase" + ) + + +def test_secret_envelope_root_path_omitted_from_description(): + event = _secret_event(path="/") + envelope = secret_event_to_envelope(event, _org()) + + assert ( + envelope["phase"]["description"] + == "Secret read in backend / Production by Dev Eloper" + ) diff --git a/backend/tests/ee/integrations/logs/streams/test_sources.py b/backend/tests/ee/integrations/logs/streams/test_sources.py new file mode 100644 index 000000000..fcd350055 --- /dev/null +++ b/backend/tests/ee/integrations/logs/streams/test_sources.py @@ -0,0 +1,44 @@ +"""Source fetch semantics: the commit-safety watermark. + +Events are timestamped inside the writing transaction — a slow transaction +can commit an older-timestamped event after a newer one was fetched and +shipped, landing it permanently behind the cursor. `fetch` therefore only +returns events older than SHIP_WATERMARK_SECONDS. +""" + +from unittest.mock import MagicMock, patch + +from django.utils import timezone + +from ee.integrations.logs.streams import sources as sources_mod + + +def test_fetch_applies_commit_watermark(): + source = sources_mod.OrgAuditLogSource() + qs = MagicMock() + qs.filter.return_value.order_by.return_value.__getitem__.return_value = [] + + with patch.object(sources_mod.OrgAuditLogSource, "_queryset", return_value=qs): + result = source.fetch( + MagicMock(), {"ts": timezone.now().isoformat(), "id": ""}, 10 + ) + + assert result == [] + watermark = qs.filter.call_args.kwargs["timestamp__lt"] + age = (timezone.now() - watermark).total_seconds() + assert ( + sources_mod.SHIP_WATERMARK_SECONDS - 5 + <= age + <= sources_mod.SHIP_WATERMARK_SECONDS + 5 + ) + + +def test_cursor_filter_carries_redundant_sargable_lower_bound(): + """The timestamp__gte bound is semantically redundant with the OR arms + but load-bearing: Postgres derives no lower scan bound across an OR, so + dropping it reverts the tail query to an O(org-history) ordered scan. + Behaviorally invisible by design — pinned structurally instead.""" + q = sources_mod.LogSource()._cursor_filter( + {"ts": "2026-07-30T12:00:00+00:00", "id": "e-5"} + ) + assert "timestamp__gte" in str(q) diff --git a/backend/tests/test_org_resolution.py b/backend/tests/test_org_resolution.py index c88783844..aaa44769f 100644 --- a/backend/tests/test_org_resolution.py +++ b/backend/tests/test_org_resolution.py @@ -44,6 +44,16 @@ def test_unknown_model_returns_none(self): from api.utils.access.org_resolution import _path_to_organisation self.assertIsNone(_path_to_organisation("Nonexistent")) + def test_log_stream_models_resolve(self): + """The stream_id / delivery_event_id aliases depend on these BFS + paths existing — LogStream via its direct organisation FK, delivery + events through their stream.""" + from api.utils.access.org_resolution import _path_to_organisation + self.assertEqual(_path_to_organisation("LogStream"), "organisation__id") + path = _path_to_organisation("LogStreamDeliveryEvent") + self.assertTrue(path.endswith("organisation__id")) + self.assertIn("stream", path) + class ResolveOrgIdTest(unittest.TestCase): """End-to-end org-id resolution with all three cache layers.""" @@ -195,6 +205,35 @@ def test_provider_id_alias_resolves(self): # Must have looked up the right Django model. mock_get.assert_called_once_with("api", "OrganisationSSOProvider") + def test_log_stream_aliases_resolve(self): + """Regression: stream_id / delivery_event_id were unresolvable — + snake_to_pascal produces 'Stream'/'DeliveryEvent', which are not + models, and an unresolvable kwarg is a silent SSO-enforcement + bypass for the log stream toggle/delete/retry/deliveries ops.""" + from api.utils.access.org_resolution import resolve_org_id + + for kwarg, model_name in ( + ("stream_id", "LogStream"), + ("delivery_event_id", "LogStreamDeliveryEvent"), + ): + cache.clear() + mock_model = MagicMock() + ( + mock_model.objects.filter.return_value.values_list.return_value.first.return_value + ) = "org-resolved" + + with patch( + "api.utils.access.org_resolution.apps.get_model", + return_value=mock_model, + ) as mock_get, patch( + "api.utils.access.org_resolution._path_to_organisation", + return_value="organisation__id", + ): + result = resolve_org_id(kwarg, "obj-1", {}) + + self.assertEqual(result, "org-resolved") + mock_get.assert_called_once_with("api", model_name) + class InvalidationTest(unittest.TestCase): """post_delete signal drops the cache entry so hard-deleted resources diff --git a/backend/tests/test_provider_credential_validation.py b/backend/tests/test_provider_credential_validation.py new file mode 100644 index 000000000..6e7094f13 --- /dev/null +++ b/backend/tests/test_provider_credential_validation.py @@ -0,0 +1,52 @@ +"""Server-side validation of provider-specific credential fields. + +The Datadog site composes into intake/API URLs (an SSRF surface) and a bad +value otherwise only surfaces at ship time as a permanently-failing stream — +the allowlist must be enforced at credential save, not just in the console +picker. +""" + +from unittest.mock import patch + +import pytest +from graphql import GraphQLError + +from backend.graphene.mutations.syncing import validate_credential_values + +_KEYPAIR = (b"\x01" * 32, b"\x02" * 32) +_C = "api.utils.crypto" + + +def test_non_datadog_providers_are_untouched(): + with patch(f"{_C}.decrypt_asymmetric") as mock_decrypt: + validate_credential_values("cloudflare", {"access_token": "enc"}) + + mock_decrypt.assert_not_called() + + +def test_missing_site_is_rejected(): + with pytest.raises(GraphQLError, match="site is required"): + validate_credential_values("datadog", {"api_key": "enc"}) + + +def test_valid_site_passes_despite_scheme_and_case_noise(): + with patch(f"{_C}.get_server_keypair", return_value=_KEYPAIR), patch( + f"{_C}.decrypt_asymmetric", return_value="https://US3.datadoghq.com/" + ): + validate_credential_values("datadog", {"site": "enc"}) + + +def test_unknown_site_is_rejected(): + with patch(f"{_C}.get_server_keypair", return_value=_KEYPAIR), patch( + f"{_C}.decrypt_asymmetric", return_value="logs.evil.example" + ): + with pytest.raises(GraphQLError, match="Unknown Datadog site"): + validate_credential_values("datadog", {"site": "enc"}) + + +def test_unreadable_site_ciphertext_is_rejected(): + with patch(f"{_C}.get_server_keypair", return_value=_KEYPAIR), patch( + f"{_C}.decrypt_asymmetric", side_effect=Exception("bad ciphertext") + ): + with pytest.raises(GraphQLError, match="Could not read"): + validate_credential_values("datadog", {"site": "enc"}) diff --git a/backend/tests/test_quotas.py b/backend/tests/test_quotas.py index c3fb28842..76070d93d 100644 --- a/backend/tests/test_quotas.py +++ b/backend/tests/test_quotas.py @@ -39,6 +39,31 @@ def test_can_add_environments_valid_license_bypasses_limit(): assert can_add_environments(_org("FR"), 100) is True +@pytest.mark.parametrize( + "plan,expected", + [ + ("FR", False), + ("PR", False), + ("EN", True), + ], +) +def test_can_use_log_streams_is_enterprise_only(plan, expected): + from backend.quotas import can_use_log_streams + + assert can_use_log_streams(_org(plan)) is expected + + +def test_can_use_log_streams_ignores_license_validity(): + """Plan is the single source of truth: license activation stamps the + licensed tier onto organisation.plan, so a Pro-tier license (plan PR) + must NOT unlock this Enterprise feature via a license short-circuit.""" + from backend.quotas import can_use_log_streams + + with patch(f"{_Q}.organisation_has_valid_license", return_value=True): + assert can_use_log_streams(_org("PR")) is False + assert can_use_log_streams(_org("EN")) is True + + def test_valid_license_check_filters_on_expiry(): """organisation_has_valid_license must exclude expired licenses by filtering on expires_at, not merely check that a license row exists.""" diff --git a/backend/tests/test_url_routing.py b/backend/tests/test_url_routing.py index 62cdd990f..5287d5c67 100644 --- a/backend/tests/test_url_routing.py +++ b/backend/tests/test_url_routing.py @@ -193,6 +193,10 @@ def test_aws_iam_auth_at_root(self): def test_azure_entra_auth_at_root(self): self.assertResolves("/identities/external/v1/azure/entra/auth/") + def test_v1_audit_logs_route_disabled_at_root(self): + # Route commented out in urls.py pending an audit-API performance pass. + self.assertNotResolves("/v1/logs/audit/") + # --- public_urls also at /public/ (legacy form / nginx-stripped self-hosted) --- def test_root_endpoint_at_public(self): @@ -204,6 +208,10 @@ def test_v1_secrets_at_public(self): def test_aws_iam_auth_at_public(self): self.assertResolves("/public/identities/external/v1/aws/iam/auth/") + def test_v1_audit_logs_route_disabled_at_public(self): + # Route commented out in urls.py pending an audit-API performance pass. + self.assertNotResolves("/public/v1/logs/audit/") + # --- non-routes still 404 (sanity: we didn't accidentally match-all) --- def test_unknown_path_does_not_resolve(self): diff --git a/frontend/apollo/gql.ts b/frontend/apollo/gql.ts index 136bc7ea1..e71b7a49c 100644 --- a/frontend/apollo/gql.ts +++ b/frontend/apollo/gql.ts @@ -75,6 +75,12 @@ type Documents = { "mutation CreateExtIdentity($organisationId: ID!, $provider: String!, $name: String!, $description: String, $trustedPrincipals: String!, $signatureTtlSeconds: Int, $stsEndpoint: String, $tenantId: String, $resource: String, $tokenNamePattern: String, $defaultTtlSeconds: Int!, $maxTtlSeconds: Int!) {\n createIdentity(\n organisationId: $organisationId\n provider: $provider\n name: $name\n description: $description\n trustedPrincipals: $trustedPrincipals\n signatureTtlSeconds: $signatureTtlSeconds\n stsEndpoint: $stsEndpoint\n tenantId: $tenantId\n resource: $resource\n tokenNamePattern: $tokenNamePattern\n defaultTtlSeconds: $defaultTtlSeconds\n maxTtlSeconds: $maxTtlSeconds\n ) {\n identity {\n id\n provider\n name\n description\n config {\n ... on AwsIamConfigType {\n trustedPrincipals\n signatureTtlSeconds\n stsEndpoint\n }\n ... on AzureEntraConfigType {\n tenantId\n resource\n allowedServicePrincipalIds\n }\n }\n tokenNamePattern\n defaultTtlSeconds\n maxTtlSeconds\n }\n }\n}": typeof types.CreateExtIdentityDocument, "mutation DeleteExtIdentity($id: ID!) {\n deleteIdentity(id: $id) {\n ok\n }\n}": typeof types.DeleteExtIdentityDocument, "mutation UpdateExtIdentity($id: ID!, $name: String, $description: String, $trustedPrincipals: String, $signatureTtlSeconds: Int, $stsEndpoint: String, $tenantId: String, $resource: String, $tokenNamePattern: String, $defaultTtlSeconds: Int, $maxTtlSeconds: Int) {\n updateIdentity(\n id: $id\n name: $name\n description: $description\n trustedPrincipals: $trustedPrincipals\n signatureTtlSeconds: $signatureTtlSeconds\n stsEndpoint: $stsEndpoint\n tenantId: $tenantId\n resource: $resource\n tokenNamePattern: $tokenNamePattern\n defaultTtlSeconds: $defaultTtlSeconds\n maxTtlSeconds: $maxTtlSeconds\n ) {\n identity {\n id\n name\n description\n config {\n ... on AwsIamConfigType {\n trustedPrincipals\n signatureTtlSeconds\n stsEndpoint\n }\n ... on AzureEntraConfigType {\n tenantId\n resource\n allowedServicePrincipalIds\n }\n }\n tokenNamePattern\n defaultTtlSeconds\n maxTtlSeconds\n }\n }\n}": typeof types.UpdateExtIdentityDocument, + "mutation CreateNewLogStream($organisationId: ID!, $name: String!, $provider: String!, $credentialId: ID!, $sources: [String!]!, $service: String, $tags: String, $gzip: Boolean, $maxAttempts: Int) {\n createLogStream(\n organisationId: $organisationId\n name: $name\n provider: $provider\n credentialId: $credentialId\n sources: $sources\n service: $service\n tags: $tags\n gzip: $gzip\n maxAttempts: $maxAttempts\n ) {\n logStream {\n id\n name\n }\n }\n}": typeof types.CreateNewLogStreamDocument, + "mutation DeleteLogStreamOp($streamId: ID!) {\n deleteLogStream(streamId: $streamId) {\n ok\n }\n}": typeof types.DeleteLogStreamOpDocument, + "mutation RetryLogStreamDeliveryOp($deliveryEventId: ID!) {\n retryLogStreamDelivery(deliveryEventId: $deliveryEventId) {\n ok\n }\n}": typeof types.RetryLogStreamDeliveryOpDocument, + "mutation TestLogStreamConnectionOp($organisationId: ID!, $provider: String!, $credentialId: ID!, $service: String, $tags: String, $gzip: Boolean) {\n testLogStreamConnection(\n organisationId: $organisationId\n provider: $provider\n credentialId: $credentialId\n service: $service\n tags: $tags\n gzip: $gzip\n ) {\n ok\n message\n }\n}": typeof types.TestLogStreamConnectionOpDocument, + "mutation ToggleLogStreamOp($streamId: ID!) {\n toggleLogStream(streamId: $streamId) {\n logStream {\n id\n isActive\n pausedReason\n }\n }\n}": typeof types.ToggleLogStreamOpDocument, + "mutation UpdateLogStreamOp($streamId: ID!, $name: String!, $credentialId: ID!, $sources: [String!]!, $service: String, $tags: String, $gzip: Boolean, $maxAttempts: Int) {\n updateLogStream(\n streamId: $streamId\n name: $name\n credentialId: $credentialId\n sources: $sources\n service: $service\n tags: $tags\n gzip: $gzip\n maxAttempts: $maxAttempts\n ) {\n logStream {\n id\n name\n }\n }\n}": typeof types.UpdateLogStreamOpDocument, "mutation AcceptOrganisationInvite($orgId: ID!, $identityKey: String!, $wrappedKeyring: String!, $wrappedRecovery: String!, $inviteId: ID!) {\n createOrganisationMember(\n orgId: $orgId\n identityKey: $identityKey\n wrappedKeyring: $wrappedKeyring\n wrappedRecovery: $wrappedRecovery\n inviteId: $inviteId\n ) {\n orgMember {\n id\n email\n createdAt\n role {\n name\n }\n }\n }\n}": typeof types.AcceptOrganisationInviteDocument, "mutation BulkInviteMembers($orgId: ID!, $invites: [InviteInput!]!) {\n bulkInviteOrganisationMembers(orgId: $orgId, invites: $invites) {\n invites {\n id\n inviteeEmail\n expiresAt\n }\n }\n}": typeof types.BulkInviteMembersDocument, "mutation DeleteOrgInvite($inviteId: ID!) {\n deleteInvitation(inviteId: $inviteId) {\n ok\n }\n}": typeof types.DeleteOrgInviteDocument, @@ -154,6 +160,9 @@ type Documents = { "query GetAwsStsEndpoints {\n awsStsEndpoints\n}": typeof types.GetAwsStsEndpointsDocument, "query GetIdentityProviders {\n identityProviders {\n id\n name\n description\n iconId\n }\n}": typeof types.GetIdentityProvidersDocument, "query GetOrganisationIdentities($organisationId: ID!) {\n identities(organisationId: $organisationId) {\n id\n provider\n name\n description\n config {\n ... on AwsIamConfigType {\n trustedPrincipals\n signatureTtlSeconds\n stsEndpoint\n }\n ... on AzureEntraConfigType {\n tenantId\n resource\n allowedServicePrincipalIds\n }\n }\n tokenNamePattern\n defaultTtlSeconds\n maxTtlSeconds\n createdAt\n }\n}": typeof types.GetOrganisationIdentitiesDocument, + "query GetLogStreamDeliveries($streamId: ID!, $limit: Int, $offset: Int, $status: String) {\n logStreamDeliveries(\n streamId: $streamId\n limit: $limit\n offset: $offset\n status: $status\n ) {\n count\n events {\n id\n source\n status\n eventCount\n payloadBytes\n attempts\n cursorFrom\n cursorTo\n retriedFrom {\n id\n }\n resolvedAt\n meta\n createdAt\n completedAt\n }\n }\n}": typeof types.GetLogStreamDeliveriesDocument, + "query GetLogStreamProviders {\n logStreamProviders {\n id\n name\n maxEventAgeHours\n credentialsProvider {\n id\n name\n expectedCredentials\n optionalCredentials\n }\n }\n logStreamSources {\n id\n name\n description\n }\n}": typeof types.GetLogStreamProvidersDocument, + "query GetLogStreams($organisationId: ID!) {\n logStreams(organisationId: $organisationId) {\n id\n name\n provider\n providerInfo {\n id\n name\n maxEventAgeHours\n credentialsProvider {\n id\n name\n }\n }\n authentication {\n id\n name\n }\n sources\n options\n maxAttempts\n isActive\n health\n pausedReason\n lastShippedAt\n lastFailureAt\n lastFailureReason\n createdAt\n unresolvedFailures\n destinationUrl\n sourceLags {\n source\n name\n lagSeconds\n }\n deliverySummary {\n completed\n failed\n }\n }\n}": typeof types.GetLogStreamsDocument, "query CheckOrganisationNameAvailability($name: String!) {\n organisationNameAvailable(name: $name)\n}": typeof types.CheckOrganisationNameAvailabilityDocument, "query GetAuditLogs($organisationId: ID!, $start: BigInt, $end: BigInt, $resourceType: String, $resourceTypes: [String], $resourceId: ID, $eventTypes: [String], $actorId: ID, $offset: Int, $limit: Int) {\n auditLogs(\n organisationId: $organisationId\n start: $start\n end: $end\n resourceType: $resourceType\n resourceTypes: $resourceTypes\n resourceId: $resourceId\n eventTypes: $eventTypes\n actorId: $actorId\n offset: $offset\n limit: $limit\n ) {\n logs {\n id\n eventType\n resourceType\n resourceId\n actorType\n actorId\n actorMetadata\n resourceMetadata\n oldValues\n newValues\n description\n ipAddress\n userAgent\n timestamp\n }\n count\n }\n}": typeof types.GetAuditLogsDocument, "query GetGlobalAccessUsers($organisationId: ID!) {\n organisationGlobalAccessUsers(organisationId: $organisationId) {\n id\n role {\n name\n permissions\n }\n identityKey\n self\n }\n}": typeof types.GetGlobalAccessUsersDocument, @@ -280,6 +289,12 @@ const documents: Documents = { "mutation CreateExtIdentity($organisationId: ID!, $provider: String!, $name: String!, $description: String, $trustedPrincipals: String!, $signatureTtlSeconds: Int, $stsEndpoint: String, $tenantId: String, $resource: String, $tokenNamePattern: String, $defaultTtlSeconds: Int!, $maxTtlSeconds: Int!) {\n createIdentity(\n organisationId: $organisationId\n provider: $provider\n name: $name\n description: $description\n trustedPrincipals: $trustedPrincipals\n signatureTtlSeconds: $signatureTtlSeconds\n stsEndpoint: $stsEndpoint\n tenantId: $tenantId\n resource: $resource\n tokenNamePattern: $tokenNamePattern\n defaultTtlSeconds: $defaultTtlSeconds\n maxTtlSeconds: $maxTtlSeconds\n ) {\n identity {\n id\n provider\n name\n description\n config {\n ... on AwsIamConfigType {\n trustedPrincipals\n signatureTtlSeconds\n stsEndpoint\n }\n ... on AzureEntraConfigType {\n tenantId\n resource\n allowedServicePrincipalIds\n }\n }\n tokenNamePattern\n defaultTtlSeconds\n maxTtlSeconds\n }\n }\n}": types.CreateExtIdentityDocument, "mutation DeleteExtIdentity($id: ID!) {\n deleteIdentity(id: $id) {\n ok\n }\n}": types.DeleteExtIdentityDocument, "mutation UpdateExtIdentity($id: ID!, $name: String, $description: String, $trustedPrincipals: String, $signatureTtlSeconds: Int, $stsEndpoint: String, $tenantId: String, $resource: String, $tokenNamePattern: String, $defaultTtlSeconds: Int, $maxTtlSeconds: Int) {\n updateIdentity(\n id: $id\n name: $name\n description: $description\n trustedPrincipals: $trustedPrincipals\n signatureTtlSeconds: $signatureTtlSeconds\n stsEndpoint: $stsEndpoint\n tenantId: $tenantId\n resource: $resource\n tokenNamePattern: $tokenNamePattern\n defaultTtlSeconds: $defaultTtlSeconds\n maxTtlSeconds: $maxTtlSeconds\n ) {\n identity {\n id\n name\n description\n config {\n ... on AwsIamConfigType {\n trustedPrincipals\n signatureTtlSeconds\n stsEndpoint\n }\n ... on AzureEntraConfigType {\n tenantId\n resource\n allowedServicePrincipalIds\n }\n }\n tokenNamePattern\n defaultTtlSeconds\n maxTtlSeconds\n }\n }\n}": types.UpdateExtIdentityDocument, + "mutation CreateNewLogStream($organisationId: ID!, $name: String!, $provider: String!, $credentialId: ID!, $sources: [String!]!, $service: String, $tags: String, $gzip: Boolean, $maxAttempts: Int) {\n createLogStream(\n organisationId: $organisationId\n name: $name\n provider: $provider\n credentialId: $credentialId\n sources: $sources\n service: $service\n tags: $tags\n gzip: $gzip\n maxAttempts: $maxAttempts\n ) {\n logStream {\n id\n name\n }\n }\n}": types.CreateNewLogStreamDocument, + "mutation DeleteLogStreamOp($streamId: ID!) {\n deleteLogStream(streamId: $streamId) {\n ok\n }\n}": types.DeleteLogStreamOpDocument, + "mutation RetryLogStreamDeliveryOp($deliveryEventId: ID!) {\n retryLogStreamDelivery(deliveryEventId: $deliveryEventId) {\n ok\n }\n}": types.RetryLogStreamDeliveryOpDocument, + "mutation TestLogStreamConnectionOp($organisationId: ID!, $provider: String!, $credentialId: ID!, $service: String, $tags: String, $gzip: Boolean) {\n testLogStreamConnection(\n organisationId: $organisationId\n provider: $provider\n credentialId: $credentialId\n service: $service\n tags: $tags\n gzip: $gzip\n ) {\n ok\n message\n }\n}": types.TestLogStreamConnectionOpDocument, + "mutation ToggleLogStreamOp($streamId: ID!) {\n toggleLogStream(streamId: $streamId) {\n logStream {\n id\n isActive\n pausedReason\n }\n }\n}": types.ToggleLogStreamOpDocument, + "mutation UpdateLogStreamOp($streamId: ID!, $name: String!, $credentialId: ID!, $sources: [String!]!, $service: String, $tags: String, $gzip: Boolean, $maxAttempts: Int) {\n updateLogStream(\n streamId: $streamId\n name: $name\n credentialId: $credentialId\n sources: $sources\n service: $service\n tags: $tags\n gzip: $gzip\n maxAttempts: $maxAttempts\n ) {\n logStream {\n id\n name\n }\n }\n}": types.UpdateLogStreamOpDocument, "mutation AcceptOrganisationInvite($orgId: ID!, $identityKey: String!, $wrappedKeyring: String!, $wrappedRecovery: String!, $inviteId: ID!) {\n createOrganisationMember(\n orgId: $orgId\n identityKey: $identityKey\n wrappedKeyring: $wrappedKeyring\n wrappedRecovery: $wrappedRecovery\n inviteId: $inviteId\n ) {\n orgMember {\n id\n email\n createdAt\n role {\n name\n }\n }\n }\n}": types.AcceptOrganisationInviteDocument, "mutation BulkInviteMembers($orgId: ID!, $invites: [InviteInput!]!) {\n bulkInviteOrganisationMembers(orgId: $orgId, invites: $invites) {\n invites {\n id\n inviteeEmail\n expiresAt\n }\n }\n}": types.BulkInviteMembersDocument, "mutation DeleteOrgInvite($inviteId: ID!) {\n deleteInvitation(inviteId: $inviteId) {\n ok\n }\n}": types.DeleteOrgInviteDocument, @@ -359,6 +374,9 @@ const documents: Documents = { "query GetAwsStsEndpoints {\n awsStsEndpoints\n}": types.GetAwsStsEndpointsDocument, "query GetIdentityProviders {\n identityProviders {\n id\n name\n description\n iconId\n }\n}": types.GetIdentityProvidersDocument, "query GetOrganisationIdentities($organisationId: ID!) {\n identities(organisationId: $organisationId) {\n id\n provider\n name\n description\n config {\n ... on AwsIamConfigType {\n trustedPrincipals\n signatureTtlSeconds\n stsEndpoint\n }\n ... on AzureEntraConfigType {\n tenantId\n resource\n allowedServicePrincipalIds\n }\n }\n tokenNamePattern\n defaultTtlSeconds\n maxTtlSeconds\n createdAt\n }\n}": types.GetOrganisationIdentitiesDocument, + "query GetLogStreamDeliveries($streamId: ID!, $limit: Int, $offset: Int, $status: String) {\n logStreamDeliveries(\n streamId: $streamId\n limit: $limit\n offset: $offset\n status: $status\n ) {\n count\n events {\n id\n source\n status\n eventCount\n payloadBytes\n attempts\n cursorFrom\n cursorTo\n retriedFrom {\n id\n }\n resolvedAt\n meta\n createdAt\n completedAt\n }\n }\n}": types.GetLogStreamDeliveriesDocument, + "query GetLogStreamProviders {\n logStreamProviders {\n id\n name\n maxEventAgeHours\n credentialsProvider {\n id\n name\n expectedCredentials\n optionalCredentials\n }\n }\n logStreamSources {\n id\n name\n description\n }\n}": types.GetLogStreamProvidersDocument, + "query GetLogStreams($organisationId: ID!) {\n logStreams(organisationId: $organisationId) {\n id\n name\n provider\n providerInfo {\n id\n name\n maxEventAgeHours\n credentialsProvider {\n id\n name\n }\n }\n authentication {\n id\n name\n }\n sources\n options\n maxAttempts\n isActive\n health\n pausedReason\n lastShippedAt\n lastFailureAt\n lastFailureReason\n createdAt\n unresolvedFailures\n destinationUrl\n sourceLags {\n source\n name\n lagSeconds\n }\n deliverySummary {\n completed\n failed\n }\n }\n}": types.GetLogStreamsDocument, "query CheckOrganisationNameAvailability($name: String!) {\n organisationNameAvailable(name: $name)\n}": types.CheckOrganisationNameAvailabilityDocument, "query GetAuditLogs($organisationId: ID!, $start: BigInt, $end: BigInt, $resourceType: String, $resourceTypes: [String], $resourceId: ID, $eventTypes: [String], $actorId: ID, $offset: Int, $limit: Int) {\n auditLogs(\n organisationId: $organisationId\n start: $start\n end: $end\n resourceType: $resourceType\n resourceTypes: $resourceTypes\n resourceId: $resourceId\n eventTypes: $eventTypes\n actorId: $actorId\n offset: $offset\n limit: $limit\n ) {\n logs {\n id\n eventType\n resourceType\n resourceId\n actorType\n actorId\n actorMetadata\n resourceMetadata\n oldValues\n newValues\n description\n ipAddress\n userAgent\n timestamp\n }\n count\n }\n}": types.GetAuditLogsDocument, "query GetGlobalAccessUsers($organisationId: ID!) {\n organisationGlobalAccessUsers(organisationId: $organisationId) {\n id\n role {\n name\n permissions\n }\n identityKey\n self\n }\n}": types.GetGlobalAccessUsersDocument, @@ -682,6 +700,30 @@ export function graphql(source: "mutation DeleteExtIdentity($id: ID!) {\n delet * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ export function graphql(source: "mutation UpdateExtIdentity($id: ID!, $name: String, $description: String, $trustedPrincipals: String, $signatureTtlSeconds: Int, $stsEndpoint: String, $tenantId: String, $resource: String, $tokenNamePattern: String, $defaultTtlSeconds: Int, $maxTtlSeconds: Int) {\n updateIdentity(\n id: $id\n name: $name\n description: $description\n trustedPrincipals: $trustedPrincipals\n signatureTtlSeconds: $signatureTtlSeconds\n stsEndpoint: $stsEndpoint\n tenantId: $tenantId\n resource: $resource\n tokenNamePattern: $tokenNamePattern\n defaultTtlSeconds: $defaultTtlSeconds\n maxTtlSeconds: $maxTtlSeconds\n ) {\n identity {\n id\n name\n description\n config {\n ... on AwsIamConfigType {\n trustedPrincipals\n signatureTtlSeconds\n stsEndpoint\n }\n ... on AzureEntraConfigType {\n tenantId\n resource\n allowedServicePrincipalIds\n }\n }\n tokenNamePattern\n defaultTtlSeconds\n maxTtlSeconds\n }\n }\n}"): (typeof documents)["mutation UpdateExtIdentity($id: ID!, $name: String, $description: String, $trustedPrincipals: String, $signatureTtlSeconds: Int, $stsEndpoint: String, $tenantId: String, $resource: String, $tokenNamePattern: String, $defaultTtlSeconds: Int, $maxTtlSeconds: Int) {\n updateIdentity(\n id: $id\n name: $name\n description: $description\n trustedPrincipals: $trustedPrincipals\n signatureTtlSeconds: $signatureTtlSeconds\n stsEndpoint: $stsEndpoint\n tenantId: $tenantId\n resource: $resource\n tokenNamePattern: $tokenNamePattern\n defaultTtlSeconds: $defaultTtlSeconds\n maxTtlSeconds: $maxTtlSeconds\n ) {\n identity {\n id\n name\n description\n config {\n ... on AwsIamConfigType {\n trustedPrincipals\n signatureTtlSeconds\n stsEndpoint\n }\n ... on AzureEntraConfigType {\n tenantId\n resource\n allowedServicePrincipalIds\n }\n }\n tokenNamePattern\n defaultTtlSeconds\n maxTtlSeconds\n }\n }\n}"]; +/** + * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. + */ +export function graphql(source: "mutation CreateNewLogStream($organisationId: ID!, $name: String!, $provider: String!, $credentialId: ID!, $sources: [String!]!, $service: String, $tags: String, $gzip: Boolean, $maxAttempts: Int) {\n createLogStream(\n organisationId: $organisationId\n name: $name\n provider: $provider\n credentialId: $credentialId\n sources: $sources\n service: $service\n tags: $tags\n gzip: $gzip\n maxAttempts: $maxAttempts\n ) {\n logStream {\n id\n name\n }\n }\n}"): (typeof documents)["mutation CreateNewLogStream($organisationId: ID!, $name: String!, $provider: String!, $credentialId: ID!, $sources: [String!]!, $service: String, $tags: String, $gzip: Boolean, $maxAttempts: Int) {\n createLogStream(\n organisationId: $organisationId\n name: $name\n provider: $provider\n credentialId: $credentialId\n sources: $sources\n service: $service\n tags: $tags\n gzip: $gzip\n maxAttempts: $maxAttempts\n ) {\n logStream {\n id\n name\n }\n }\n}"]; +/** + * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. + */ +export function graphql(source: "mutation DeleteLogStreamOp($streamId: ID!) {\n deleteLogStream(streamId: $streamId) {\n ok\n }\n}"): (typeof documents)["mutation DeleteLogStreamOp($streamId: ID!) {\n deleteLogStream(streamId: $streamId) {\n ok\n }\n}"]; +/** + * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. + */ +export function graphql(source: "mutation RetryLogStreamDeliveryOp($deliveryEventId: ID!) {\n retryLogStreamDelivery(deliveryEventId: $deliveryEventId) {\n ok\n }\n}"): (typeof documents)["mutation RetryLogStreamDeliveryOp($deliveryEventId: ID!) {\n retryLogStreamDelivery(deliveryEventId: $deliveryEventId) {\n ok\n }\n}"]; +/** + * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. + */ +export function graphql(source: "mutation TestLogStreamConnectionOp($organisationId: ID!, $provider: String!, $credentialId: ID!, $service: String, $tags: String, $gzip: Boolean) {\n testLogStreamConnection(\n organisationId: $organisationId\n provider: $provider\n credentialId: $credentialId\n service: $service\n tags: $tags\n gzip: $gzip\n ) {\n ok\n message\n }\n}"): (typeof documents)["mutation TestLogStreamConnectionOp($organisationId: ID!, $provider: String!, $credentialId: ID!, $service: String, $tags: String, $gzip: Boolean) {\n testLogStreamConnection(\n organisationId: $organisationId\n provider: $provider\n credentialId: $credentialId\n service: $service\n tags: $tags\n gzip: $gzip\n ) {\n ok\n message\n }\n}"]; +/** + * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. + */ +export function graphql(source: "mutation ToggleLogStreamOp($streamId: ID!) {\n toggleLogStream(streamId: $streamId) {\n logStream {\n id\n isActive\n pausedReason\n }\n }\n}"): (typeof documents)["mutation ToggleLogStreamOp($streamId: ID!) {\n toggleLogStream(streamId: $streamId) {\n logStream {\n id\n isActive\n pausedReason\n }\n }\n}"]; +/** + * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. + */ +export function graphql(source: "mutation UpdateLogStreamOp($streamId: ID!, $name: String!, $credentialId: ID!, $sources: [String!]!, $service: String, $tags: String, $gzip: Boolean, $maxAttempts: Int) {\n updateLogStream(\n streamId: $streamId\n name: $name\n credentialId: $credentialId\n sources: $sources\n service: $service\n tags: $tags\n gzip: $gzip\n maxAttempts: $maxAttempts\n ) {\n logStream {\n id\n name\n }\n }\n}"): (typeof documents)["mutation UpdateLogStreamOp($streamId: ID!, $name: String!, $credentialId: ID!, $sources: [String!]!, $service: String, $tags: String, $gzip: Boolean, $maxAttempts: Int) {\n updateLogStream(\n streamId: $streamId\n name: $name\n credentialId: $credentialId\n sources: $sources\n service: $service\n tags: $tags\n gzip: $gzip\n maxAttempts: $maxAttempts\n ) {\n logStream {\n id\n name\n }\n }\n}"]; /** * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ @@ -998,6 +1040,18 @@ export function graphql(source: "query GetIdentityProviders {\n identityProvide * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ export function graphql(source: "query GetOrganisationIdentities($organisationId: ID!) {\n identities(organisationId: $organisationId) {\n id\n provider\n name\n description\n config {\n ... on AwsIamConfigType {\n trustedPrincipals\n signatureTtlSeconds\n stsEndpoint\n }\n ... on AzureEntraConfigType {\n tenantId\n resource\n allowedServicePrincipalIds\n }\n }\n tokenNamePattern\n defaultTtlSeconds\n maxTtlSeconds\n createdAt\n }\n}"): (typeof documents)["query GetOrganisationIdentities($organisationId: ID!) {\n identities(organisationId: $organisationId) {\n id\n provider\n name\n description\n config {\n ... on AwsIamConfigType {\n trustedPrincipals\n signatureTtlSeconds\n stsEndpoint\n }\n ... on AzureEntraConfigType {\n tenantId\n resource\n allowedServicePrincipalIds\n }\n }\n tokenNamePattern\n defaultTtlSeconds\n maxTtlSeconds\n createdAt\n }\n}"]; +/** + * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. + */ +export function graphql(source: "query GetLogStreamDeliveries($streamId: ID!, $limit: Int, $offset: Int, $status: String) {\n logStreamDeliveries(\n streamId: $streamId\n limit: $limit\n offset: $offset\n status: $status\n ) {\n count\n events {\n id\n source\n status\n eventCount\n payloadBytes\n attempts\n cursorFrom\n cursorTo\n retriedFrom {\n id\n }\n resolvedAt\n meta\n createdAt\n completedAt\n }\n }\n}"): (typeof documents)["query GetLogStreamDeliveries($streamId: ID!, $limit: Int, $offset: Int, $status: String) {\n logStreamDeliveries(\n streamId: $streamId\n limit: $limit\n offset: $offset\n status: $status\n ) {\n count\n events {\n id\n source\n status\n eventCount\n payloadBytes\n attempts\n cursorFrom\n cursorTo\n retriedFrom {\n id\n }\n resolvedAt\n meta\n createdAt\n completedAt\n }\n }\n}"]; +/** + * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. + */ +export function graphql(source: "query GetLogStreamProviders {\n logStreamProviders {\n id\n name\n maxEventAgeHours\n credentialsProvider {\n id\n name\n expectedCredentials\n optionalCredentials\n }\n }\n logStreamSources {\n id\n name\n description\n }\n}"): (typeof documents)["query GetLogStreamProviders {\n logStreamProviders {\n id\n name\n maxEventAgeHours\n credentialsProvider {\n id\n name\n expectedCredentials\n optionalCredentials\n }\n }\n logStreamSources {\n id\n name\n description\n }\n}"]; +/** + * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. + */ +export function graphql(source: "query GetLogStreams($organisationId: ID!) {\n logStreams(organisationId: $organisationId) {\n id\n name\n provider\n providerInfo {\n id\n name\n maxEventAgeHours\n credentialsProvider {\n id\n name\n }\n }\n authentication {\n id\n name\n }\n sources\n options\n maxAttempts\n isActive\n health\n pausedReason\n lastShippedAt\n lastFailureAt\n lastFailureReason\n createdAt\n unresolvedFailures\n destinationUrl\n sourceLags {\n source\n name\n lagSeconds\n }\n deliverySummary {\n completed\n failed\n }\n }\n}"): (typeof documents)["query GetLogStreams($organisationId: ID!) {\n logStreams(organisationId: $organisationId) {\n id\n name\n provider\n providerInfo {\n id\n name\n maxEventAgeHours\n credentialsProvider {\n id\n name\n }\n }\n authentication {\n id\n name\n }\n sources\n options\n maxAttempts\n isActive\n health\n pausedReason\n lastShippedAt\n lastFailureAt\n lastFailureReason\n createdAt\n unresolvedFailures\n destinationUrl\n sourceLags {\n source\n name\n lagSeconds\n }\n deliverySummary {\n completed\n failed\n }\n }\n}"]; /** * The graphql function is used to parse GraphQL queries into a document that can be used by GraphQL clients. */ diff --git a/frontend/apollo/graphql.ts b/frontend/apollo/graphql.ts index 6d6c59424..084bf797d 100644 --- a/frontend/apollo/graphql.ts +++ b/frontend/apollo/graphql.ts @@ -179,6 +179,8 @@ export enum ApiAuditEventResourceTypeChoices { Sa = 'SA', /** ServiceAccountToken */ SaToken = 'SA_TOKEN', + /** LogStream */ + Stream = 'STREAM', /** ServiceToken */ SvcToken = 'SVC_TOKEN', /** Team */ @@ -271,6 +273,24 @@ export enum ApiEnvironmentSyncStatusChoices { TimedOut = 'TIMED_OUT' } +/** An enumeration. */ +export enum ApiLogStreamDeliveryEventStatusChoices { + /** Completed */ + Completed = 'COMPLETED', + /** Failed */ + Failed = 'FAILED', + /** Skipped */ + Skipped = 'SKIPPED' +} + +/** An enumeration. */ +export enum ApiLogStreamHealthChoices { + /** Degraded */ + Degraded = 'DEGRADED', + /** Healthy */ + Healthy = 'HEALTHY' +} + /** An enumeration. */ export enum ApiOrganisationPlanChoices { /** Enterprise */ @@ -645,6 +665,11 @@ export type CreateLockboxMutation = { lockbox?: Maybe; }; +export type CreateLogStreamMutation = { + __typename?: 'CreateLogStreamMutation'; + logStream?: Maybe; +}; + export type CreateNetworkAccessPolicyMutation = { __typename?: 'CreateNetworkAccessPolicyMutation'; networkAccessPolicy?: Maybe; @@ -805,6 +830,11 @@ export type DeleteInviteMutation = { ok?: Maybe; }; +export type DeleteLogStreamMutation = { + __typename?: 'DeleteLogStreamMutation'; + ok?: Maybe; +}; + export type DeleteNetworkAccessPolicyMutation = { __typename?: 'DeleteNetworkAccessPolicyMutation'; ok?: Maybe; @@ -1258,6 +1288,84 @@ export type LockboxType = { views: Scalars['Int']['output']; }; +export type LogStreamDeliveryEventType = { + __typename?: 'LogStreamDeliveryEventType'; + attempts: Scalars['Int']['output']; + completedAt?: Maybe; + createdAt?: Maybe; + cursorFrom?: Maybe; + cursorTo?: Maybe; + eventCount: Scalars['Int']['output']; + id: Scalars['String']['output']; + meta?: Maybe; + payloadBytes: Scalars['Int']['output']; + resolvedAt?: Maybe; + retriedFrom?: Maybe; + source: Scalars['String']['output']; + status: ApiLogStreamDeliveryEventStatusChoices; +}; + +export type LogStreamDeliveryHistoryType = { + __typename?: 'LogStreamDeliveryHistoryType'; + count?: Maybe; + events?: Maybe>>; +}; + +/** Delivery counts over the last 24 hours. */ +export type LogStreamDeliverySummaryType = { + __typename?: 'LogStreamDeliverySummaryType'; + completed: Scalars['Int']['output']; + failed: Scalars['Int']['output']; +}; + +export type LogStreamProviderType = { + __typename?: 'LogStreamProviderType'; + credentialsProvider?: Maybe; + id: Scalars['String']['output']; + maxEventAgeHours?: Maybe; + name: Scalars['String']['output']; +}; + +export type LogStreamSourceLagType = { + __typename?: 'LogStreamSourceLagType'; + lagSeconds: Scalars['Int']['output']; + name: Scalars['String']['output']; + source: Scalars['String']['output']; +}; + +export type LogStreamSourceType = { + __typename?: 'LogStreamSourceType'; + description: Scalars['String']['output']; + id: Scalars['String']['output']; + name: Scalars['String']['output']; +}; + +export type LogStreamType = { + __typename?: 'LogStreamType'; + authentication?: Maybe; + createdAt?: Maybe; + deliverySummary?: Maybe; + destinationUrl?: Maybe; + health: ApiLogStreamHealthChoices; + id: Scalars['String']['output']; + isActive: Scalars['Boolean']['output']; + lastFailureAt?: Maybe; + lastFailureReason: Scalars['String']['output']; + lastShippedAt?: Maybe; + /** Delivery attempts per chunk before it is recorded as failed and skipped. */ + maxAttempts: Scalars['Int']['output']; + name: Scalars['String']['output']; + options: Scalars['JSONString']['output']; + pausedReason: Scalars['String']['output']; + /** Log stream adapter id (resolved against the log stream adapter registry). */ + provider: Scalars['String']['output']; + providerInfo?: Maybe; + sourceLags: Array; + sources: Array; + unresolvedFailures: Scalars['Int']['output']; + updatedAt: Scalars['DateTime']['output']; +}; + export type ManualRotateRotatingSecretMutation = { __typename?: 'ManualRotateRotatingSecretMutation'; rotatingSecret?: Maybe; @@ -1304,6 +1412,7 @@ export type Mutation = { createGitlabCiSync?: Maybe; createIdentity?: Maybe; createLockbox?: Maybe; + createLogStream?: Maybe; createNetworkAccessPolicy?: Maybe; createNomadSync?: Maybe; createOrganisation?: Maybe; @@ -1343,6 +1452,7 @@ export type Mutation = { deleteEnvironment?: Maybe; deleteIdentity?: Maybe; deleteInvitation?: Maybe; + deleteLogStream?: Maybe; deleteNetworkAccessPolicy?: Maybe; deleteOrganisationMember?: Maybe; deleteOrganisationSsoProvider?: Maybe; @@ -1383,12 +1493,15 @@ export type Mutation = { renewDynamicSecretLease?: Maybe; resumeRotatingSecret?: Maybe; resumeSubscription?: Maybe; + retryLogStreamDelivery?: Maybe; revokeDynamicSecretLease?: Maybe; revokeRotatingSecretCredential?: Maybe; rotateAppKeys?: Maybe; rotateRotatingSecret?: Maybe; setDefaultPaymentMethod?: Maybe; + testLogStreamConnection?: Maybe; testOrganisationSsoProvider?: Maybe; + toggleLogStream?: Maybe; /** Master switch: enable/disable SCIM for the organisation. */ toggleScim?: Maybe; /** Per-provider toggle: enable/disable a single SCIM token. */ @@ -1407,6 +1520,7 @@ export type Mutation = { updateCustomRole?: Maybe; updateEnvironmentOrder?: Maybe; updateIdentity?: Maybe; + updateLogStream?: Maybe; updateMemberEnvironmentScope?: Maybe; /** * Re-wrap this member's keyring (SSO recovery) or establish it on @@ -1641,6 +1755,19 @@ export type MutationCreateLockboxArgs = { }; +export type MutationCreateLogStreamArgs = { + credentialId: Scalars['ID']['input']; + gzip?: InputMaybe; + maxAttempts?: InputMaybe; + name: Scalars['String']['input']; + organisationId: Scalars['ID']['input']; + provider: Scalars['String']['input']; + service?: InputMaybe; + sources: Array; + tags?: InputMaybe; +}; + + export type MutationCreateNetworkAccessPolicyArgs = { allowedIps: Scalars['String']['input']; isGlobal: Scalars['Boolean']['input']; @@ -1892,6 +2019,11 @@ export type MutationDeleteInvitationArgs = { }; +export type MutationDeleteLogStreamArgs = { + streamId: Scalars['ID']['input']; +}; + + export type MutationDeleteNetworkAccessPolicyArgs = { id: Scalars['ID']['input']; }; @@ -2077,6 +2209,11 @@ export type MutationResumeSubscriptionArgs = { }; +export type MutationRetryLogStreamDeliveryArgs = { + deliveryEventId: Scalars['ID']['input']; +}; + + export type MutationRevokeDynamicSecretLeaseArgs = { leaseId: Scalars['ID']['input']; }; @@ -2105,11 +2242,26 @@ export type MutationSetDefaultPaymentMethodArgs = { }; +export type MutationTestLogStreamConnectionArgs = { + credentialId: Scalars['ID']['input']; + gzip?: InputMaybe; + organisationId: Scalars['ID']['input']; + provider: Scalars['String']['input']; + service?: InputMaybe; + tags?: InputMaybe; +}; + + export type MutationTestOrganisationSsoProviderArgs = { providerId: Scalars['ID']['input']; }; +export type MutationToggleLogStreamArgs = { + streamId: Scalars['ID']['input']; +}; + + export type MutationToggleScimArgs = { enabled: Scalars['Boolean']['input']; organisationId: Scalars['ID']['input']; @@ -2202,6 +2354,18 @@ export type MutationUpdateIdentityArgs = { }; +export type MutationUpdateLogStreamArgs = { + credentialId: Scalars['ID']['input']; + gzip?: InputMaybe; + maxAttempts?: InputMaybe; + name: Scalars['String']['input']; + service?: InputMaybe; + sources: Array; + streamId: Scalars['ID']['input']; + tags?: InputMaybe; +}; + + export type MutationUpdateMemberEnvironmentScopeArgs = { appId?: InputMaybe; envKeys?: InputMaybe>>; @@ -2531,6 +2695,10 @@ export type Query = { identityProviders?: Maybe>>; kmsLogs?: Maybe; license?: Maybe; + logStreamDeliveries?: Maybe; + logStreamProviders?: Maybe>>; + logStreamSources?: Maybe>>; + logStreams?: Maybe>>; networkAccessPolicies?: Maybe>>; openaiProjects?: Maybe>>; organisationGlobalAccessUsers?: Maybe>>; @@ -2730,6 +2898,19 @@ export type QueryKmsLogsArgs = { }; +export type QueryLogStreamDeliveriesArgs = { + limit?: InputMaybe; + offset?: InputMaybe; + status?: InputMaybe; + streamId: Scalars['ID']['input']; +}; + + +export type QueryLogStreamsArgs = { + organisationId: Scalars['ID']['input']; +}; + + export type QueryNetworkAccessPoliciesArgs = { organisationId?: InputMaybe; }; @@ -3047,6 +3228,11 @@ export type ResumeRotatingSecretMutation = { rotatingSecret?: Maybe; }; +export type RetryLogStreamDeliveryMutation = { + __typename?: 'RetryLogStreamDeliveryMutation'; + ok?: Maybe; +}; + export type RevokeLeaseMutation = { __typename?: 'RevokeLeaseMutation'; lease?: Maybe; @@ -3450,6 +3636,12 @@ export type TeamType = { updatedAt: Scalars['DateTime']['output']; }; +export type TestLogStreamConnectionMutation = { + __typename?: 'TestLogStreamConnectionMutation'; + message?: Maybe; + ok?: Maybe; +}; + export type TestOrganisationSsoProviderMutation = { __typename?: 'TestOrganisationSSOProviderMutation'; error?: Maybe; @@ -3465,6 +3657,11 @@ export enum TimeRange { Year = 'YEAR' } +export type ToggleLogStreamMutation = { + __typename?: 'ToggleLogStreamMutation'; + logStream?: Maybe; +}; + /** Master switch: enable/disable SCIM for the organisation. */ export type ToggleScimMutation = { __typename?: 'ToggleSCIMMutation'; @@ -3531,6 +3728,11 @@ export type UpdateIdentityMutation = { identity?: Maybe; }; +export type UpdateLogStreamMutation = { + __typename?: 'UpdateLogStreamMutation'; + logStream?: Maybe; +}; + export type UpdateMemberEnvScopeMutation = { __typename?: 'UpdateMemberEnvScopeMutation'; app?: Maybe; @@ -4236,6 +4438,68 @@ export type UpdateExtIdentityMutation = { __typename?: 'Mutation', updateIdentit | { __typename?: 'AzureEntraConfigType', tenantId?: string | null, resource?: string | null, allowedServicePrincipalIds?: Array | null } | null } | null } | null }; +export type CreateNewLogStreamMutationVariables = Exact<{ + organisationId: Scalars['ID']['input']; + name: Scalars['String']['input']; + provider: Scalars['String']['input']; + credentialId: Scalars['ID']['input']; + sources: Array | Scalars['String']['input']; + service?: InputMaybe; + tags?: InputMaybe; + gzip?: InputMaybe; + maxAttempts?: InputMaybe; +}>; + + +export type CreateNewLogStreamMutation = { __typename?: 'Mutation', createLogStream?: { __typename?: 'CreateLogStreamMutation', logStream?: { __typename?: 'LogStreamType', id: string, name: string } | null } | null }; + +export type DeleteLogStreamOpMutationVariables = Exact<{ + streamId: Scalars['ID']['input']; +}>; + + +export type DeleteLogStreamOpMutation = { __typename?: 'Mutation', deleteLogStream?: { __typename?: 'DeleteLogStreamMutation', ok?: boolean | null } | null }; + +export type RetryLogStreamDeliveryOpMutationVariables = Exact<{ + deliveryEventId: Scalars['ID']['input']; +}>; + + +export type RetryLogStreamDeliveryOpMutation = { __typename?: 'Mutation', retryLogStreamDelivery?: { __typename?: 'RetryLogStreamDeliveryMutation', ok?: boolean | null } | null }; + +export type TestLogStreamConnectionOpMutationVariables = Exact<{ + organisationId: Scalars['ID']['input']; + provider: Scalars['String']['input']; + credentialId: Scalars['ID']['input']; + service?: InputMaybe; + tags?: InputMaybe; + gzip?: InputMaybe; +}>; + + +export type TestLogStreamConnectionOpMutation = { __typename?: 'Mutation', testLogStreamConnection?: { __typename?: 'TestLogStreamConnectionMutation', ok?: boolean | null, message?: string | null } | null }; + +export type ToggleLogStreamOpMutationVariables = Exact<{ + streamId: Scalars['ID']['input']; +}>; + + +export type ToggleLogStreamOpMutation = { __typename?: 'Mutation', toggleLogStream?: { __typename?: 'ToggleLogStreamMutation', logStream?: { __typename?: 'LogStreamType', id: string, isActive: boolean, pausedReason: string } | null } | null }; + +export type UpdateLogStreamOpMutationVariables = Exact<{ + streamId: Scalars['ID']['input']; + name: Scalars['String']['input']; + credentialId: Scalars['ID']['input']; + sources: Array | Scalars['String']['input']; + service?: InputMaybe; + tags?: InputMaybe; + gzip?: InputMaybe; + maxAttempts?: InputMaybe; +}>; + + +export type UpdateLogStreamOpMutation = { __typename?: 'Mutation', updateLogStream?: { __typename?: 'UpdateLogStreamMutation', logStream?: { __typename?: 'LogStreamType', id: string, name: string } | null } | null }; + export type AcceptOrganisationInviteMutationVariables = Exact<{ orgId: Scalars['ID']['input']; identityKey: Scalars['String']['input']; @@ -4936,6 +5200,28 @@ export type GetOrganisationIdentitiesQuery = { __typename?: 'Query', identities? | { __typename?: 'AzureEntraConfigType', tenantId?: string | null, resource?: string | null, allowedServicePrincipalIds?: Array | null } | null } | null> | null }; +export type GetLogStreamDeliveriesQueryVariables = Exact<{ + streamId: Scalars['ID']['input']; + limit?: InputMaybe; + offset?: InputMaybe; + status?: InputMaybe; +}>; + + +export type GetLogStreamDeliveriesQuery = { __typename?: 'Query', logStreamDeliveries?: { __typename?: 'LogStreamDeliveryHistoryType', count?: number | null, events?: Array<{ __typename?: 'LogStreamDeliveryEventType', id: string, source: string, status: ApiLogStreamDeliveryEventStatusChoices, eventCount: number, payloadBytes: number, attempts: number, cursorFrom?: any | null, cursorTo?: any | null, resolvedAt?: any | null, meta?: any | null, createdAt?: any | null, completedAt?: any | null, retriedFrom?: { __typename?: 'LogStreamDeliveryEventType', id: string } | null } | null> | null } | null }; + +export type GetLogStreamProvidersQueryVariables = Exact<{ [key: string]: never; }>; + + +export type GetLogStreamProvidersQuery = { __typename?: 'Query', logStreamProviders?: Array<{ __typename?: 'LogStreamProviderType', id: string, name: string, maxEventAgeHours?: number | null, credentialsProvider?: { __typename?: 'ProviderType', id: string, name: string, expectedCredentials: Array, optionalCredentials: Array } | null } | null> | null, logStreamSources?: Array<{ __typename?: 'LogStreamSourceType', id: string, name: string, description: string } | null> | null }; + +export type GetLogStreamsQueryVariables = Exact<{ + organisationId: Scalars['ID']['input']; +}>; + + +export type GetLogStreamsQuery = { __typename?: 'Query', logStreams?: Array<{ __typename?: 'LogStreamType', id: string, name: string, provider: string, sources: Array, options: any, maxAttempts: number, isActive: boolean, health: ApiLogStreamHealthChoices, pausedReason: string, lastShippedAt?: any | null, lastFailureAt?: any | null, lastFailureReason: string, createdAt?: any | null, unresolvedFailures: number, destinationUrl?: string | null, providerInfo?: { __typename?: 'LogStreamProviderType', id: string, name: string, maxEventAgeHours?: number | null, credentialsProvider?: { __typename?: 'ProviderType', id: string, name: string } | null } | null, authentication?: { __typename?: 'ProviderCredentialsType', id: string, name: string } | null, sourceLags: Array<{ __typename?: 'LogStreamSourceLagType', source: string, name: string, lagSeconds: number }>, deliverySummary?: { __typename?: 'LogStreamDeliverySummaryType', completed: number, failed: number } | null } | null> | null }; + export type CheckOrganisationNameAvailabilityQueryVariables = Exact<{ name: Scalars['String']['input']; }>; @@ -5473,6 +5759,12 @@ export const UpdateEnvOrderDocument = {"kind":"Document","definitions":[{"kind": export const CreateExtIdentityDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"CreateExtIdentity"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"organisationId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"provider"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"name"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"description"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"trustedPrincipals"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"signatureTtlSeconds"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"Int"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"stsEndpoint"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"tenantId"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"resource"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"tokenNamePattern"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"defaultTtlSeconds"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"Int"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"maxTtlSeconds"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"Int"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"createIdentity"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"organisationId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"organisationId"}}},{"kind":"Argument","name":{"kind":"Name","value":"provider"},"value":{"kind":"Variable","name":{"kind":"Name","value":"provider"}}},{"kind":"Argument","name":{"kind":"Name","value":"name"},"value":{"kind":"Variable","name":{"kind":"Name","value":"name"}}},{"kind":"Argument","name":{"kind":"Name","value":"description"},"value":{"kind":"Variable","name":{"kind":"Name","value":"description"}}},{"kind":"Argument","name":{"kind":"Name","value":"trustedPrincipals"},"value":{"kind":"Variable","name":{"kind":"Name","value":"trustedPrincipals"}}},{"kind":"Argument","name":{"kind":"Name","value":"signatureTtlSeconds"},"value":{"kind":"Variable","name":{"kind":"Name","value":"signatureTtlSeconds"}}},{"kind":"Argument","name":{"kind":"Name","value":"stsEndpoint"},"value":{"kind":"Variable","name":{"kind":"Name","value":"stsEndpoint"}}},{"kind":"Argument","name":{"kind":"Name","value":"tenantId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"tenantId"}}},{"kind":"Argument","name":{"kind":"Name","value":"resource"},"value":{"kind":"Variable","name":{"kind":"Name","value":"resource"}}},{"kind":"Argument","name":{"kind":"Name","value":"tokenNamePattern"},"value":{"kind":"Variable","name":{"kind":"Name","value":"tokenNamePattern"}}},{"kind":"Argument","name":{"kind":"Name","value":"defaultTtlSeconds"},"value":{"kind":"Variable","name":{"kind":"Name","value":"defaultTtlSeconds"}}},{"kind":"Argument","name":{"kind":"Name","value":"maxTtlSeconds"},"value":{"kind":"Variable","name":{"kind":"Name","value":"maxTtlSeconds"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"identity"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"provider"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"config"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"AwsIamConfigType"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"trustedPrincipals"}},{"kind":"Field","name":{"kind":"Name","value":"signatureTtlSeconds"}},{"kind":"Field","name":{"kind":"Name","value":"stsEndpoint"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"AzureEntraConfigType"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"tenantId"}},{"kind":"Field","name":{"kind":"Name","value":"resource"}},{"kind":"Field","name":{"kind":"Name","value":"allowedServicePrincipalIds"}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"tokenNamePattern"}},{"kind":"Field","name":{"kind":"Name","value":"defaultTtlSeconds"}},{"kind":"Field","name":{"kind":"Name","value":"maxTtlSeconds"}}]}}]}}]}}]} as unknown as DocumentNode; export const DeleteExtIdentityDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"DeleteExtIdentity"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"deleteIdentity"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"id"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"ok"}}]}}]}}]} as unknown as DocumentNode; export const UpdateExtIdentityDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"UpdateExtIdentity"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"name"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"description"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"trustedPrincipals"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"signatureTtlSeconds"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"Int"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"stsEndpoint"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"tenantId"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"resource"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"tokenNamePattern"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"defaultTtlSeconds"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"Int"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"maxTtlSeconds"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"Int"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"updateIdentity"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"id"}}},{"kind":"Argument","name":{"kind":"Name","value":"name"},"value":{"kind":"Variable","name":{"kind":"Name","value":"name"}}},{"kind":"Argument","name":{"kind":"Name","value":"description"},"value":{"kind":"Variable","name":{"kind":"Name","value":"description"}}},{"kind":"Argument","name":{"kind":"Name","value":"trustedPrincipals"},"value":{"kind":"Variable","name":{"kind":"Name","value":"trustedPrincipals"}}},{"kind":"Argument","name":{"kind":"Name","value":"signatureTtlSeconds"},"value":{"kind":"Variable","name":{"kind":"Name","value":"signatureTtlSeconds"}}},{"kind":"Argument","name":{"kind":"Name","value":"stsEndpoint"},"value":{"kind":"Variable","name":{"kind":"Name","value":"stsEndpoint"}}},{"kind":"Argument","name":{"kind":"Name","value":"tenantId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"tenantId"}}},{"kind":"Argument","name":{"kind":"Name","value":"resource"},"value":{"kind":"Variable","name":{"kind":"Name","value":"resource"}}},{"kind":"Argument","name":{"kind":"Name","value":"tokenNamePattern"},"value":{"kind":"Variable","name":{"kind":"Name","value":"tokenNamePattern"}}},{"kind":"Argument","name":{"kind":"Name","value":"defaultTtlSeconds"},"value":{"kind":"Variable","name":{"kind":"Name","value":"defaultTtlSeconds"}}},{"kind":"Argument","name":{"kind":"Name","value":"maxTtlSeconds"},"value":{"kind":"Variable","name":{"kind":"Name","value":"maxTtlSeconds"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"identity"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"config"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"AwsIamConfigType"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"trustedPrincipals"}},{"kind":"Field","name":{"kind":"Name","value":"signatureTtlSeconds"}},{"kind":"Field","name":{"kind":"Name","value":"stsEndpoint"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"AzureEntraConfigType"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"tenantId"}},{"kind":"Field","name":{"kind":"Name","value":"resource"}},{"kind":"Field","name":{"kind":"Name","value":"allowedServicePrincipalIds"}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"tokenNamePattern"}},{"kind":"Field","name":{"kind":"Name","value":"defaultTtlSeconds"}},{"kind":"Field","name":{"kind":"Name","value":"maxTtlSeconds"}}]}}]}}]}}]} as unknown as DocumentNode; +export const CreateNewLogStreamDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"CreateNewLogStream"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"organisationId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"name"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"provider"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"credentialId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"sources"}},"type":{"kind":"NonNullType","type":{"kind":"ListType","type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"service"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"tags"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"gzip"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"Boolean"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"maxAttempts"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"Int"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"createLogStream"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"organisationId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"organisationId"}}},{"kind":"Argument","name":{"kind":"Name","value":"name"},"value":{"kind":"Variable","name":{"kind":"Name","value":"name"}}},{"kind":"Argument","name":{"kind":"Name","value":"provider"},"value":{"kind":"Variable","name":{"kind":"Name","value":"provider"}}},{"kind":"Argument","name":{"kind":"Name","value":"credentialId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"credentialId"}}},{"kind":"Argument","name":{"kind":"Name","value":"sources"},"value":{"kind":"Variable","name":{"kind":"Name","value":"sources"}}},{"kind":"Argument","name":{"kind":"Name","value":"service"},"value":{"kind":"Variable","name":{"kind":"Name","value":"service"}}},{"kind":"Argument","name":{"kind":"Name","value":"tags"},"value":{"kind":"Variable","name":{"kind":"Name","value":"tags"}}},{"kind":"Argument","name":{"kind":"Name","value":"gzip"},"value":{"kind":"Variable","name":{"kind":"Name","value":"gzip"}}},{"kind":"Argument","name":{"kind":"Name","value":"maxAttempts"},"value":{"kind":"Variable","name":{"kind":"Name","value":"maxAttempts"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"logStream"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}}]}}]}}]} as unknown as DocumentNode; +export const DeleteLogStreamOpDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"DeleteLogStreamOp"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"streamId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"deleteLogStream"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"streamId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"streamId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"ok"}}]}}]}}]} as unknown as DocumentNode; +export const RetryLogStreamDeliveryOpDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"RetryLogStreamDeliveryOp"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"deliveryEventId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"retryLogStreamDelivery"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"deliveryEventId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"deliveryEventId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"ok"}}]}}]}}]} as unknown as DocumentNode; +export const TestLogStreamConnectionOpDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"TestLogStreamConnectionOp"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"organisationId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"provider"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"credentialId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"service"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"tags"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"gzip"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"Boolean"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"testLogStreamConnection"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"organisationId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"organisationId"}}},{"kind":"Argument","name":{"kind":"Name","value":"provider"},"value":{"kind":"Variable","name":{"kind":"Name","value":"provider"}}},{"kind":"Argument","name":{"kind":"Name","value":"credentialId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"credentialId"}}},{"kind":"Argument","name":{"kind":"Name","value":"service"},"value":{"kind":"Variable","name":{"kind":"Name","value":"service"}}},{"kind":"Argument","name":{"kind":"Name","value":"tags"},"value":{"kind":"Variable","name":{"kind":"Name","value":"tags"}}},{"kind":"Argument","name":{"kind":"Name","value":"gzip"},"value":{"kind":"Variable","name":{"kind":"Name","value":"gzip"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"ok"}},{"kind":"Field","name":{"kind":"Name","value":"message"}}]}}]}}]} as unknown as DocumentNode; +export const ToggleLogStreamOpDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"ToggleLogStreamOp"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"streamId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"toggleLogStream"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"streamId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"streamId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"logStream"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"isActive"}},{"kind":"Field","name":{"kind":"Name","value":"pausedReason"}}]}}]}}]}}]} as unknown as DocumentNode; +export const UpdateLogStreamOpDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"UpdateLogStreamOp"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"streamId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"name"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"credentialId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"sources"}},"type":{"kind":"NonNullType","type":{"kind":"ListType","type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"service"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"tags"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"gzip"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"Boolean"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"maxAttempts"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"Int"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"updateLogStream"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"streamId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"streamId"}}},{"kind":"Argument","name":{"kind":"Name","value":"name"},"value":{"kind":"Variable","name":{"kind":"Name","value":"name"}}},{"kind":"Argument","name":{"kind":"Name","value":"credentialId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"credentialId"}}},{"kind":"Argument","name":{"kind":"Name","value":"sources"},"value":{"kind":"Variable","name":{"kind":"Name","value":"sources"}}},{"kind":"Argument","name":{"kind":"Name","value":"service"},"value":{"kind":"Variable","name":{"kind":"Name","value":"service"}}},{"kind":"Argument","name":{"kind":"Name","value":"tags"},"value":{"kind":"Variable","name":{"kind":"Name","value":"tags"}}},{"kind":"Argument","name":{"kind":"Name","value":"gzip"},"value":{"kind":"Variable","name":{"kind":"Name","value":"gzip"}}},{"kind":"Argument","name":{"kind":"Name","value":"maxAttempts"},"value":{"kind":"Variable","name":{"kind":"Name","value":"maxAttempts"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"logStream"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}}]}}]}}]} as unknown as DocumentNode; export const AcceptOrganisationInviteDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"AcceptOrganisationInvite"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"orgId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"identityKey"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"wrappedKeyring"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"wrappedRecovery"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"inviteId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"createOrganisationMember"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"orgId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"orgId"}}},{"kind":"Argument","name":{"kind":"Name","value":"identityKey"},"value":{"kind":"Variable","name":{"kind":"Name","value":"identityKey"}}},{"kind":"Argument","name":{"kind":"Name","value":"wrappedKeyring"},"value":{"kind":"Variable","name":{"kind":"Name","value":"wrappedKeyring"}}},{"kind":"Argument","name":{"kind":"Name","value":"wrappedRecovery"},"value":{"kind":"Variable","name":{"kind":"Name","value":"wrappedRecovery"}}},{"kind":"Argument","name":{"kind":"Name","value":"inviteId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"inviteId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"orgMember"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"email"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"role"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"name"}}]}}]}}]}}]}}]} as unknown as DocumentNode; export const BulkInviteMembersDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"BulkInviteMembers"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"orgId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"invites"}},"type":{"kind":"NonNullType","type":{"kind":"ListType","type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"InviteInput"}}}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"bulkInviteOrganisationMembers"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"orgId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"orgId"}}},{"kind":"Argument","name":{"kind":"Name","value":"invites"},"value":{"kind":"Variable","name":{"kind":"Name","value":"invites"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"invites"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"inviteeEmail"}},{"kind":"Field","name":{"kind":"Name","value":"expiresAt"}}]}}]}}]}}]} as unknown as DocumentNode; export const DeleteOrgInviteDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"DeleteOrgInvite"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"inviteId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"deleteInvitation"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"inviteId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"inviteId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"ok"}}]}}]}}]} as unknown as DocumentNode; @@ -5552,6 +5844,9 @@ export const GetOrganisationsDocument = {"kind":"Document","definitions":[{"kind export const GetAwsStsEndpointsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetAwsStsEndpoints"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"awsStsEndpoints"}}]}}]} as unknown as DocumentNode; export const GetIdentityProvidersDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetIdentityProviders"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"identityProviders"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"iconId"}}]}}]}}]} as unknown as DocumentNode; export const GetOrganisationIdentitiesDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetOrganisationIdentities"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"organisationId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"identities"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"organisationId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"organisationId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"provider"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"config"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"AwsIamConfigType"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"trustedPrincipals"}},{"kind":"Field","name":{"kind":"Name","value":"signatureTtlSeconds"}},{"kind":"Field","name":{"kind":"Name","value":"stsEndpoint"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"AzureEntraConfigType"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"tenantId"}},{"kind":"Field","name":{"kind":"Name","value":"resource"}},{"kind":"Field","name":{"kind":"Name","value":"allowedServicePrincipalIds"}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"tokenNamePattern"}},{"kind":"Field","name":{"kind":"Name","value":"defaultTtlSeconds"}},{"kind":"Field","name":{"kind":"Name","value":"maxTtlSeconds"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}}]}}]}}]} as unknown as DocumentNode; +export const GetLogStreamDeliveriesDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetLogStreamDeliveries"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"streamId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"limit"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"Int"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"offset"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"Int"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"status"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"logStreamDeliveries"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"streamId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"streamId"}}},{"kind":"Argument","name":{"kind":"Name","value":"limit"},"value":{"kind":"Variable","name":{"kind":"Name","value":"limit"}}},{"kind":"Argument","name":{"kind":"Name","value":"offset"},"value":{"kind":"Variable","name":{"kind":"Name","value":"offset"}}},{"kind":"Argument","name":{"kind":"Name","value":"status"},"value":{"kind":"Variable","name":{"kind":"Name","value":"status"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"count"}},{"kind":"Field","name":{"kind":"Name","value":"events"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"source"}},{"kind":"Field","name":{"kind":"Name","value":"status"}},{"kind":"Field","name":{"kind":"Name","value":"eventCount"}},{"kind":"Field","name":{"kind":"Name","value":"payloadBytes"}},{"kind":"Field","name":{"kind":"Name","value":"attempts"}},{"kind":"Field","name":{"kind":"Name","value":"cursorFrom"}},{"kind":"Field","name":{"kind":"Name","value":"cursorTo"}},{"kind":"Field","name":{"kind":"Name","value":"retriedFrom"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}}]}},{"kind":"Field","name":{"kind":"Name","value":"resolvedAt"}},{"kind":"Field","name":{"kind":"Name","value":"meta"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"completedAt"}}]}}]}}]}}]} as unknown as DocumentNode; +export const GetLogStreamProvidersDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetLogStreamProviders"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"logStreamProviders"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"maxEventAgeHours"}},{"kind":"Field","name":{"kind":"Name","value":"credentialsProvider"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"expectedCredentials"}},{"kind":"Field","name":{"kind":"Name","value":"optionalCredentials"}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"logStreamSources"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"description"}}]}}]}}]} as unknown as DocumentNode; +export const GetLogStreamsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetLogStreams"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"organisationId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"logStreams"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"organisationId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"organisationId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"provider"}},{"kind":"Field","name":{"kind":"Name","value":"providerInfo"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"maxEventAgeHours"}},{"kind":"Field","name":{"kind":"Name","value":"credentialsProvider"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}}]}},{"kind":"Field","name":{"kind":"Name","value":"authentication"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}},{"kind":"Field","name":{"kind":"Name","value":"sources"}},{"kind":"Field","name":{"kind":"Name","value":"options"}},{"kind":"Field","name":{"kind":"Name","value":"maxAttempts"}},{"kind":"Field","name":{"kind":"Name","value":"isActive"}},{"kind":"Field","name":{"kind":"Name","value":"health"}},{"kind":"Field","name":{"kind":"Name","value":"pausedReason"}},{"kind":"Field","name":{"kind":"Name","value":"lastShippedAt"}},{"kind":"Field","name":{"kind":"Name","value":"lastFailureAt"}},{"kind":"Field","name":{"kind":"Name","value":"lastFailureReason"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"unresolvedFailures"}},{"kind":"Field","name":{"kind":"Name","value":"destinationUrl"}},{"kind":"Field","name":{"kind":"Name","value":"sourceLags"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"source"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"lagSeconds"}}]}},{"kind":"Field","name":{"kind":"Name","value":"deliverySummary"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"completed"}},{"kind":"Field","name":{"kind":"Name","value":"failed"}}]}}]}}]}}]} as unknown as DocumentNode; export const CheckOrganisationNameAvailabilityDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"CheckOrganisationNameAvailability"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"name"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"organisationNameAvailable"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"name"},"value":{"kind":"Variable","name":{"kind":"Name","value":"name"}}}]}]}}]} as unknown as DocumentNode; export const GetAuditLogsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetAuditLogs"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"organisationId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"start"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"BigInt"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"end"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"BigInt"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"resourceType"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"resourceTypes"}},"type":{"kind":"ListType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"resourceId"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"eventTypes"}},"type":{"kind":"ListType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"actorId"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"offset"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"Int"}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"limit"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"Int"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"auditLogs"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"organisationId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"organisationId"}}},{"kind":"Argument","name":{"kind":"Name","value":"start"},"value":{"kind":"Variable","name":{"kind":"Name","value":"start"}}},{"kind":"Argument","name":{"kind":"Name","value":"end"},"value":{"kind":"Variable","name":{"kind":"Name","value":"end"}}},{"kind":"Argument","name":{"kind":"Name","value":"resourceType"},"value":{"kind":"Variable","name":{"kind":"Name","value":"resourceType"}}},{"kind":"Argument","name":{"kind":"Name","value":"resourceTypes"},"value":{"kind":"Variable","name":{"kind":"Name","value":"resourceTypes"}}},{"kind":"Argument","name":{"kind":"Name","value":"resourceId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"resourceId"}}},{"kind":"Argument","name":{"kind":"Name","value":"eventTypes"},"value":{"kind":"Variable","name":{"kind":"Name","value":"eventTypes"}}},{"kind":"Argument","name":{"kind":"Name","value":"actorId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"actorId"}}},{"kind":"Argument","name":{"kind":"Name","value":"offset"},"value":{"kind":"Variable","name":{"kind":"Name","value":"offset"}}},{"kind":"Argument","name":{"kind":"Name","value":"limit"},"value":{"kind":"Variable","name":{"kind":"Name","value":"limit"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"logs"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"eventType"}},{"kind":"Field","name":{"kind":"Name","value":"resourceType"}},{"kind":"Field","name":{"kind":"Name","value":"resourceId"}},{"kind":"Field","name":{"kind":"Name","value":"actorType"}},{"kind":"Field","name":{"kind":"Name","value":"actorId"}},{"kind":"Field","name":{"kind":"Name","value":"actorMetadata"}},{"kind":"Field","name":{"kind":"Name","value":"resourceMetadata"}},{"kind":"Field","name":{"kind":"Name","value":"oldValues"}},{"kind":"Field","name":{"kind":"Name","value":"newValues"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"ipAddress"}},{"kind":"Field","name":{"kind":"Name","value":"userAgent"}},{"kind":"Field","name":{"kind":"Name","value":"timestamp"}}]}},{"kind":"Field","name":{"kind":"Name","value":"count"}}]}}]}}]} as unknown as DocumentNode; export const GetGlobalAccessUsersDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"GetGlobalAccessUsers"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"organisationId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"organisationGlobalAccessUsers"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"organisationId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"organisationId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"role"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"permissions"}}]}},{"kind":"Field","name":{"kind":"Name","value":"identityKey"}},{"kind":"Field","name":{"kind":"Name","value":"self"}}]}}]}}]} as unknown as DocumentNode; diff --git a/frontend/apollo/schema.graphql b/frontend/apollo/schema.graphql index db7599eb7..1b11896f3 100644 --- a/frontend/apollo/schema.graphql +++ b/frontend/apollo/schema.graphql @@ -74,6 +74,10 @@ type Query { rotationProviderImportTemplate(providerId: String!, authenticationId: ID!, templateRef: String!): GenericScalar openaiProjects(authenticationId: ID!): [OpenAIProjectType] rotationCloneSpec(sourceRotatingSecretId: ID!): RotationCloneSpecType + logStreams(organisationId: ID!): [LogStreamType] + logStreamDeliveries(streamId: ID!, limit: Int, offset: Int, status: String): LogStreamDeliveryHistoryType + logStreamProviders: [LogStreamProviderType] + logStreamSources: [LogStreamSourceType] } type OrganisationType { @@ -1005,6 +1009,9 @@ enum ApiAuditEventResourceTypeChoices { """RotatingSecret""" RS + + """LogStream""" + STREAM } """An enumeration.""" @@ -1493,6 +1500,104 @@ type RotationCloneKeyMapEntry { keyName: String! } +type LogStreamType { + id: String! + name: String! + + """ + Log stream adapter id (resolved against the log stream adapter registry). + """ + provider: String! + authentication: ProviderCredentialsType + sources: [String!]! + options: JSONString! + + """ + Delivery attempts per chunk before it is recorded as failed and skipped. + """ + maxAttempts: Int! + isActive: Boolean! + health: ApiLogStreamHealthChoices! + pausedReason: String! + lastShippedAt: DateTime + lastFailureAt: DateTime + lastFailureReason: String! + createdAt: DateTime + updatedAt: DateTime! + providerInfo: LogStreamProviderType + sourceLags: [LogStreamSourceLagType!]! + unresolvedFailures: Int! + deliverySummary: LogStreamDeliverySummaryType + destinationUrl: String +} + +"""An enumeration.""" +enum ApiLogStreamHealthChoices { + """Healthy""" + HEALTHY + + """Degraded""" + DEGRADED +} + +type LogStreamProviderType { + id: String! + name: String! + credentialsProvider: ProviderType + maxEventAgeHours: Float +} + +type LogStreamSourceLagType { + source: String! + name: String! + lagSeconds: Int! +} + +"""Delivery counts over the last 24 hours.""" +type LogStreamDeliverySummaryType { + completed: Int! + failed: Int! +} + +type LogStreamDeliveryHistoryType { + events: [LogStreamDeliveryEventType] + count: Int +} + +type LogStreamDeliveryEventType { + id: String! + source: String! + status: ApiLogStreamDeliveryEventStatusChoices! + eventCount: Int! + payloadBytes: Int! + attempts: Int! + cursorFrom: DateTime + cursorTo: DateTime + retriedFrom: LogStreamDeliveryEventType + resolvedAt: DateTime + meta: JSONString + createdAt: DateTime + completedAt: DateTime +} + +"""An enumeration.""" +enum ApiLogStreamDeliveryEventStatusChoices { + """Completed""" + COMPLETED + + """Failed""" + FAILED + + """Skipped""" + SKIPPED +} + +type LogStreamSourceType { + id: String! + name: String! + description: String! +} + type Mutation { createOrganisation(id: ID!, identityKey: String!, name: String!, wrappedKeyring: String!, wrappedRecovery: String!): CreateOrganisationMutation bulkInviteOrganisationMembers(invites: [InviteInput]!, orgId: ID!): BulkInviteOrganisationMembersMutation @@ -1661,6 +1766,12 @@ type Mutation { Probe a provider with encrypted root credentials before they are persisted. """ validateRotationCredentials(credentials: JSONString!, organisationId: ID!, providerId: String!): ValidateRotationCredentialsMutation + createLogStream(credentialId: ID!, gzip: Boolean, maxAttempts: Int, name: String!, organisationId: ID!, provider: String!, service: String, sources: [String!]!, tags: String): CreateLogStreamMutation + updateLogStream(credentialId: ID!, gzip: Boolean, maxAttempts: Int, name: String!, service: String, sources: [String!]!, streamId: ID!, tags: String): UpdateLogStreamMutation + toggleLogStream(streamId: ID!): ToggleLogStreamMutation + deleteLogStream(streamId: ID!): DeleteLogStreamMutation + testLogStreamConnection(credentialId: ID!, gzip: Boolean, organisationId: ID!, provider: String!, service: String, tags: String): TestLogStreamConnectionMutation + retryLogStreamDelivery(deliveryEventId: ID!): RetryLogStreamDeliveryMutation } type CreateOrganisationMutation { @@ -2293,4 +2404,29 @@ Probe a provider with encrypted root credentials before they are persisted. type ValidateRotationCredentialsMutation { valid: Boolean error: String +} + +type CreateLogStreamMutation { + logStream: LogStreamType +} + +type UpdateLogStreamMutation { + logStream: LogStreamType +} + +type ToggleLogStreamMutation { + logStream: LogStreamType +} + +type DeleteLogStreamMutation { + ok: Boolean +} + +type TestLogStreamConnectionMutation { + ok: Boolean + message: String +} + +type RetryLogStreamDeliveryMutation { + ok: Boolean } \ No newline at end of file diff --git a/frontend/app/[team]/integrations/layout.tsx b/frontend/app/[team]/integrations/layout.tsx index 8ea86d007..36f1911ca 100644 --- a/frontend/app/[team]/integrations/layout.tsx +++ b/frontend/app/[team]/integrations/layout.tsx @@ -6,6 +6,7 @@ import clsx from 'clsx' import Link from 'next/link' import { usePathname } from 'next/navigation' import { organisationContext } from '@/contexts/organisationContext' +import { userHasGlobalAccess, userHasPermission } from '@/utils/access/permissions' export default function AccessLayout({ params, @@ -21,6 +22,13 @@ export default function AccessLayout({ const [tabIndex, setTabIndex] = useState(0) + // Log streams export org-wide activity — the backend requires a role with + // global access on top of the LogStreams permission. + const userCanReadLogStreams = activeOrganisation + ? userHasPermission(activeOrganisation.role?.permissions, 'LogStreams', 'read') && + userHasGlobalAccess(activeOrganisation.role?.permissions) + : false + const tabs = useMemo( () => [ { @@ -31,12 +39,20 @@ export default function AccessLayout({ name: 'Dynamic Secrets', link: 'dynamic-secrets', }, + ...(userCanReadLogStreams + ? [ + { + name: 'Log Streams', + link: 'log-streams', + }, + ] + : []), { name: 'Third-party credentials', link: 'credentials', }, ], - [] + [userCanReadLogStreams] ) useEffect(() => { diff --git a/frontend/app/[team]/integrations/log-streams/page.tsx b/frontend/app/[team]/integrations/log-streams/page.tsx new file mode 100644 index 000000000..cd1518688 --- /dev/null +++ b/frontend/app/[team]/integrations/log-streams/page.tsx @@ -0,0 +1,192 @@ +'use client' + +import { useContext } from 'react' +import { useQuery } from '@apollo/client' +import { FaBan, FaStream } from 'react-icons/fa' +import { ApiOrganisationPlanChoices, LogStreamType } from '@/apollo/graphql' +import { GetLogStreams } from '@/graphql/queries/logstreams/getLogStreams.gql' +import { Alert } from '@/components/common/Alert' +import { EmptyState } from '@/components/common/EmptyState' +import Spinner from '@/components/common/Spinner' +import { PlanLabel } from '@/components/settings/organisation/PlanLabel' +import { UpsellDialog } from '@/components/settings/organisation/UpsellDialog' +import { organisationContext } from '@/contexts/organisationContext' +import { userHasGlobalAccess, userHasPermission } from '@/utils/access/permissions' +import { CreateLogStreamDialog } from '@/ee/components/logstreams/CreateLogStreamDialog' +import { LogStreamCard } from '@/ee/components/logstreams/LogStreamCard' + +export default function LogStreams({ params }: { params: { team: string } }) { + const { activeOrganisation: organisation } = useContext(organisationContext) + + // permissions — log streams export org-wide activity, so every operation + // additionally requires a role with global access (enforced server-side). + const hasGlobalAccess = organisation + ? userHasGlobalAccess(organisation.role?.permissions) + : false + const userCanReadLogStreams = organisation + ? userHasPermission(organisation.role?.permissions, 'LogStreams', 'read') && + hasGlobalAccess + : false + const userCanCreateLogStreams = organisation + ? userHasPermission(organisation.role?.permissions, 'LogStreams', 'create') + : false + const userCanUpdateLogStreams = organisation + ? userHasPermission(organisation.role?.permissions, 'LogStreams', 'update') + : false + const userCanDeleteLogStreams = organisation + ? userHasPermission(organisation.role?.permissions, 'LogStreams', 'delete') + : false + + const { data, loading } = useQuery(GetLogStreams, { + variables: { organisationId: organisation?.id }, + pollInterval: 10000, + // The stream list fans out to per-stream lag/summary queries on the + // backend — don't keep polling from hidden tabs. + skipPollAttempt: () => document.hidden, + skip: !organisation || !userCanReadLogStreams, + fetchPolicy: 'cache-and-network', + nextFetchPolicy: 'cache-and-network', + }) + + const streams: LogStreamType[] = data?.logStreams ?? [] + const isEnterprise = organisation?.plan === ApiOrganisationPlanChoices.En + + // Plan gate — Log Streams are Enterprise-only. A downgraded org with + // existing streams still sees them so it can pause or delete (teardown is + // deliberately not plan-gated server-side); create/resume/update stay + // gated. The bare upsell only renders when there is nothing to manage. + if ( + organisation && + !isEnterprise && + (!userCanReadLogStreams || (!loading && streams.length === 0)) + ) { + return ( +
+
+

Log Streams

+

+ Stream audit logs and secret events to your SIEM or log management platform. +

+
+ + +
+ } + > +
+ + Upgrade + + + } + /> +
+ + + ) + } + + // Permission gate + if (organisation && !userCanReadLogStreams) { + return ( + + + + } + > + <> + + ) + } + + if (loading || !organisation) + return ( +
+ +
+ ) + + return ( +
+
+

Log Streams

+

+ Stream audit logs and secret events to your SIEM or log management platform in near + real-time. +

+
+ + {!isEnterprise && ( + +
+ + Your organisation is no longer on the Enterprise plan, so these streams have + stopped shipping. You can pause or delete them — upgrade to resume streaming. + + + Upgrade + + + } + /> +
+ + )} + + {streams.length > 0 && userCanCreateLogStreams && isEnterprise && ( +
+ +
+ )} + + {streams.length === 0 ? ( +
+ + +
+ } + > + {userCanCreateLogStreams ? ( +
+ +
+ ) : ( + <> + )} + +
+ ) : ( +
+ {streams.map((stream) => ( + + ))} +
+ )} + + ) +} diff --git a/frontend/components/common/CommandPalette.tsx b/frontend/components/common/CommandPalette.tsx index 329b574a4..8b7a75aba 100644 --- a/frontend/components/common/CommandPalette.tsx +++ b/frontend/components/common/CommandPalette.tsx @@ -338,6 +338,8 @@ const CommandPalette: React.FC = () => { keyring ) + // Matches on decrypted secret names AND id-shaped input (e.g. an id pasted + // from a SIEM event), all client-side via useSecretSearch. const secretCommands: CommandItem[] = secretResults.map((secret) => ({ id: secret.id, name: secret.key, @@ -385,11 +387,17 @@ const CommandPalette: React.FC = () => { const keywords = query.toLowerCase().split(/\s+/) + // Secret results are already query-matched (by decrypted name or by id) + // inside useSecretSearch — re-filtering them on visible text would drop + // id-based matches, whose uuid never appears in the display name. + const secretCommandIds = new Set(secretCommands.map((command) => command.id)) + return flattenedCommands.filter((command) => { + if (secretCommandIds.has(command.id)) return true const searchableText = `${command.name} ${command.description}`.toLowerCase() return keywords.every((keyword) => searchableText.includes(keyword)) }) - }, [query, flattenedCommands]) + }, [query, flattenedCommands, secretCommands]) useEffect(() => { const detectPlatform = () => { diff --git a/frontend/components/logs/AuditLogs.tsx b/frontend/components/logs/AuditLogs.tsx index 2dce7b29a..dc84eec4a 100644 --- a/frontend/components/logs/AuditLogs.tsx +++ b/frontend/components/logs/AuditLogs.tsx @@ -30,7 +30,7 @@ import { Fragment, useContext, useState, useEffect, useRef } from 'react' import { Button } from '@/components/common/Button' import { Count } from 'reaviz' import { organisationContext } from '@/contexts/organisationContext' -import { userHasPermission } from '@/utils/access/permissions' +import { userHasGlobalAccess, userHasPermission } from '@/utils/access/permissions' import { EmptyState } from '../common/EmptyState' import { Combobox, RadioGroup } from '@headlessui/react' import { Avatar } from '../common/Avatar' @@ -59,6 +59,7 @@ const RESOURCE_TABS: ResourceTab[] = [ { key: 'invite', label: 'Invites', resourceType: 'invite' }, { key: 'policy', label: 'Network Policies', resourceType: 'policy' }, { key: 'rs', label: 'Rotating Secrets', resourceType: 'rs' }, + { key: 'stream', label: 'Log Streams', resourceType: 'stream' }, { key: 'tokens', label: 'Tokens', resourceType: null }, ] @@ -101,6 +102,7 @@ const getResourceTypeLabel = (resourceType: string) => { [ApiAuditEventResourceTypeChoices.SvcToken]: 'Service Token', [ApiAuditEventResourceTypeChoices.Invite]: 'Invite', [ApiAuditEventResourceTypeChoices.Rs]: 'Rotating Secret', + [ApiAuditEventResourceTypeChoices.Stream]: 'Log Stream', } return labels[resourceType] || resourceType } @@ -146,6 +148,8 @@ const getResourceLink = ( resourceMeta?.environment_id ) return `/${team}/apps/${resourceMeta.app_id}/environments/${resourceMeta.environment_id}` + if (rt === ApiAuditEventResourceTypeChoices.Stream) + return `/${team}/integrations/log-streams` return null } @@ -1069,9 +1073,16 @@ export default function AuditLogs() { <> {userCanReadLogs ? (
- {/* Resource type tabs */} + {/* Resource type tabs. The backend scopes non-global roles to + their accessible apps/envs and excludes 'stream' events (org- + wide egress config), so that tab would be permanently empty + for them — hide it. */}
- {RESOURCE_TABS.map((tab) => ( + {RESOURCE_TABS.filter( + (tab) => + tab.key !== 'stream' || + userHasGlobalAccess(organisation?.role?.permissions) + ).map((tab) => ( + } + /> +
+ )} + + ) +}) + +CreateLogStreamDialog.displayName = 'CreateLogStreamDialog' diff --git a/frontend/ee/components/logstreams/DeleteLogStreamDialog.tsx b/frontend/ee/components/logstreams/DeleteLogStreamDialog.tsx new file mode 100644 index 000000000..febd907d2 --- /dev/null +++ b/frontend/ee/components/logstreams/DeleteLogStreamDialog.tsx @@ -0,0 +1,78 @@ +import { useContext, useRef } from 'react' +import { useMutation } from '@apollo/client' +import { toast } from 'react-toastify' +import { FaTrashCan } from 'react-icons/fa6' +import { LogStreamType } from '@/apollo/graphql' +import { DeleteLogStreamOp } from '@/graphql/mutations/logstreams/deleteLogStream.gql' +import { GetLogStreams } from '@/graphql/queries/logstreams/getLogStreams.gql' +import GenericDialog from '@/components/common/GenericDialog' +import { Button } from '@/components/common/Button' +import { organisationContext } from '@/contexts/organisationContext' + +export const DeleteLogStreamDialog = (props: { + stream: LogStreamType + onDeleted?: () => void +}) => { + const { stream, onDeleted } = props + + const { activeOrganisation: organisation } = useContext(organisationContext) + + const dialogRef = useRef<{ openModal: () => void; closeModal: () => void }>(null) + const [deleteLogStream, { loading }] = useMutation(DeleteLogStreamOp) + + const handleDelete = async () => { + try { + await deleteLogStream({ + variables: { streamId: stream.id }, + refetchQueries: [ + { query: GetLogStreams, variables: { organisationId: organisation?.id } }, + ], + }) + toast.success('Log Stream deleted') + dialogRef.current?.closeModal() + onDeleted?.() + } catch (error: any) { + toast.error(error.message || 'Could not delete the Log Stream') + } + } + + return ( + +
+
+

+ Are you sure you want to delete the{' '} + {stream.name} Log + Stream? +

+

+ Shipping will stop immediately and its delivery history will no longer be available. + Events already stored in Phase are not affected. +

+
+ +
+ + +
+
+ + ) +} diff --git a/frontend/ee/components/logstreams/LogStreamCard.tsx b/frontend/ee/components/logstreams/LogStreamCard.tsx new file mode 100644 index 000000000..4f52b819c --- /dev/null +++ b/frontend/ee/components/logstreams/LogStreamCard.tsx @@ -0,0 +1,129 @@ +import { useRef } from 'react' +import clsx from 'clsx' +import { FaCog, FaExclamationTriangle, FaExternalLinkAlt } from 'react-icons/fa' +import { LogStreamType } from '@/apollo/graphql' +import { Button } from '@/components/common/Button' +import { ProviderIcon } from '@/components/syncing/ProviderIcon' +import { relativeTimeFromDates } from '@/utils/time' +import { + ManageLogStreamDialog, + ManageLogStreamDialogHandle, +} from './ManageLogStreamDialog' +import { LogStreamStatusIndicator } from './LogStreamStatusIndicator' +import { SourceIcon } from './sourceMeta' +import { humanizeLag, lagIsCritical } from './utils' + +export const LogStreamCard = (props: { + stream: LogStreamType + userCanUpdate: boolean + userCanDelete: boolean +}) => { + const { stream, userCanUpdate, userCanDelete } = props + + const dialogRef = useRef(null) + + return ( +
+ {/* Top row: identity left, status + last shipped right */} +
+
+ +
+ + {stream.name} + + + {stream.providerInfo?.name} + {stream.authentication ? ` • ${stream.authentication.name}` : ''} + +
+
+ +
+
+ Last shipped:{' '} + {stream.lastShippedAt + ? relativeTimeFromDates(new Date(stream.lastShippedAt)) + : 'never'} +
+ +
+
+ + {/* Bottom row: source chips left, actions pinned right */} +
+
+ {stream.sourceLags?.map((sourceLag) => ( + = 60 + ? 'bg-amber-400/10 ring-amber-400/20 text-amber-500' + : 'bg-neutral-400/10 ring-neutral-400/20 text-neutral-500' + )} + title="How far this source's cursor is behind the newest events" + > + + {sourceLag!.name} + {stream.isActive ? `: ${humanizeLag(sourceLag!.lagSeconds)}` : ''} + + ))} +
+ +
+ {stream.unresolvedFailures > 0 && ( + + )} + {stream.authentication === null && ( + + Credentials missing + + )} + {stream.destinationUrl && ( + + + + )} + + +
+
+
+ ) +} diff --git a/frontend/ee/components/logstreams/LogStreamDeliveryHistory.tsx b/frontend/ee/components/logstreams/LogStreamDeliveryHistory.tsx new file mode 100644 index 000000000..f9e4594a2 --- /dev/null +++ b/frontend/ee/components/logstreams/LogStreamDeliveryHistory.tsx @@ -0,0 +1,486 @@ +import { Fragment, useRef, useState } from 'react' +import { useMutation, useQuery } from '@apollo/client' +import { Disclosure, Transition } from '@headlessui/react' +import clsx from 'clsx' +import { toast } from 'react-toastify' +import { FaArrowRotateRight, FaChevronDown } from 'react-icons/fa6' +import { FaStream } from 'react-icons/fa' +import { FiChevronsDown } from 'react-icons/fi' +import { LogStreamDeliveryEventType, LogStreamSourceType, LogStreamType } from '@/apollo/graphql' +import { GetLogStreamDeliveries } from '@/graphql/queries/logstreams/getLogStreamDeliveries.gql' +import { GetLogStreamProviders } from '@/graphql/queries/logstreams/getLogStreamProviders.gql' +import { RetryLogStreamDeliveryOp } from '@/graphql/mutations/logstreams/retryLogStreamDelivery.gql' +import { relativeTimeFromDates } from '@/utils/time' +import { Alert } from '@/components/common/Alert' +import { Button } from '@/components/common/Button' +import { EmptyState } from '@/components/common/EmptyState' +import Spinner from '@/components/common/Spinner' +import { DeliveryStatusIndicator } from './LogStreamStatusIndicator' + +const PAGE_SIZE = 25 + +const FormattedJSON = (props: { jsonData: any }) => { + const formatted = + typeof props.jsonData === 'string' + ? props.jsonData + : JSON.stringify(props.jsonData ?? {}, null, 2) + + return ( +
+ + 
{formatted}
+ +
+ ) +} + +const DeliveryRow = (props: { + event: LogStreamDeliveryEventType + stream: LogStreamType + sourceNames: Record + userCanRetry: boolean + onRetried: () => Promise +}) => { + const { event, stream, sourceNames, userCanRetry, onRetried } = props + + const [retryDelivery, { loading: retrying }] = useMutation(RetryLogStreamDeliveryOp) + + const isFailure = event.status.toLowerCase() !== 'completed' + + // Ranges older than the destination's max event age get silently dropped + // (Datadog 202s then discards). A fully-expired range is no longer + // retryable — the backend rejects it; a partially-expired one ships the + // live tail and records the expired head as skipped. The 40-minute margin + // mirrors the backend's SKIP_AHEAD_MARGIN so the Retry button disappears + // exactly when the backend starts rejecting. + const SKIP_AHEAD_MARGIN_MS = 40 * 60 * 1000 + const maxAgeHours = stream.providerInfo?.maxEventAgeHours + const windowMs = maxAgeHours ? maxAgeHours * 3600 * 1000 - SKIP_AHEAD_MARGIN_MS : null + const fullyExpired = + !!windowMs && + !!event.cursorTo && + Date.now() - new Date(event.cursorTo).getTime() > windowMs + + // Expired-resolved rows shipped nothing — never a green "Resolved". + let expiredResolution = false + try { + const meta = typeof event.meta === 'string' ? JSON.parse(event.meta) : event.meta + expiredResolution = meta?.resolution === 'expired' + } catch { + // unparseable meta — treat as a normal resolution + } + const partiallyExpired = + !fullyExpired && + !!windowMs && + !!event.cursorFrom && + Date.now() - new Date(event.cursorFrom).getTime() > windowMs + + // Stream-level failures (e.g. missing credentials) have no event range to + // re-ship, so they aren't retryable. Paused streams don't egress at all — + // manual retries included. + const retryable = + isFailure && + !event.resolvedAt && + userCanRetry && + stream.isActive && + !!event.source && + !!event.cursorFrom && + !!event.cursorTo && + !fullyExpired + + const handleRetry = async () => { + try { + await retryDelivery({ variables: { deliveryEventId: event.id } }) + toast.info('Delivery retry queued') + await onRetried() + } catch (error: any) { + toast.error(error.message || 'Could not queue the retry') + } + } + + return ( + + {({ open }) => ( + <> + + + + + + +
+ + {isFailure && event.resolvedAt && ( + + {expiredResolution ? 'Expired' : 'Resolved'} + + )} +
+ + + + {sourceNames[event.source] || event.source} + + + {event.eventCount} + + {event.attempts} + + + {relativeTimeFromDates(new Date(event.createdAt))} + + + + {retryable && ( + + )} + + + + + +
+
+ Delivery ID: + {event.id} +
+ {event.cursorFrom && event.cursorTo && ( +
+ Event range: + + {new Date(event.cursorFrom).toISOString()} —{' '} + {new Date(event.cursorTo).toISOString()} + +
+ )} +
+ + {isFailure && (expiredResolution || (!event.resolvedAt && fullyExpired)) && ( + + This range is older than the destination's {maxAgeHours}h + ingestion window and can no longer be retried — events would be + silently discarded. The events remain available in the Phase + Console logs. + + )} + + {retryable && partiallyExpired && ( + + Part of this range is older than the destination's{' '} + {maxAgeHours}h ingestion window. A retry ships the events still + inside the window and records the expired part as skipped. + + )} + + + + + + + )} + + ) +} + +export const LogStreamDeliveryHistory = (props: { + stream: LogStreamType + userCanRetry: boolean + initialStatusFilter?: string +}) => { + const { stream, userCanRetry, initialStatusFilter } = props + + const [statusFilter, setStatusFilter] = useState(initialStatusFilter || null) + + // Load-more (accumulating) pagination like the secret logs page. Polling is + // deliberately off: a poll re-runs the base query at offset 0 and would + // collapse the accumulated list back to page one. + const { data, loading, refetch, fetchMore } = useQuery(GetLogStreamDeliveries, { + variables: { + streamId: stream.id, + limit: PAGE_SIZE, + offset: 0, + status: statusFilter, + }, + fetchPolicy: 'cache-and-network', + notifyOnNetworkStatusChange: true, + }) + + // Registry names (cache-first — the settings tab's form runs the same + // query). sourceLags only covers the stream's ACTIVE sources, so a + // delivery row from a since-disabled source would fall back to its raw id + // (e.g. 'secrets' instead of 'App secret logs') without the registry map. + const { data: providersData } = useQuery(GetLogStreamProviders) + + const events: LogStreamDeliveryEventType[] = data?.logStreamDeliveries?.events ?? [] + const count: number = data?.logStreamDeliveries?.count ?? 0 + // `count` is a planner estimate above 10k rows, so also treat a short page + // as the end. + const [reachedEnd, setReachedEnd] = useState(false) + const endOfList = reachedEnd || events.length >= count + + const sourceNames: Record = Object.fromEntries([ + ...(providersData?.logStreamSources ?? []).map((source: LogStreamSourceType) => [ + source.id, + source.name, + ]), + ...(stream.sourceLags ?? []).map((lag) => [lag!.source, lag!.name]), + ]) + + // Server-offset high-water mark: rows CONSUMED from the server, including + // duplicates dropped by the dedupe below. The load-more offset must + // advance by rows consumed, not rows displayed — deriving it from the + // deduped list length would re-request the same offset forever once a + // stale-offset page is entirely already-seen rows (each new delivery row + // shifts seen rows down into later pages of the newest-first list). + const rawFetchedRef = useRef(0) + + const [refreshing, setRefreshing] = useState(false) + const handleRefresh = async () => { + setRefreshing(true) + rawFetchedRef.current = 0 + setReachedEnd(false) + try { + await refetch() + } finally { + setRefreshing(false) + } + } + + const [loadingMore, setLoadingMore] = useState(false) + const loadMore = async () => { + if (loadingMore || endOfList) return + setLoadingMore(true) + const offset = Math.max(events.length, rawFetchedRef.current) + try { + const result = await fetchMore({ + // limit is explicit: a refetch (after a retry) merges an enlarged + // limit into the query variables, and fetchMore would inherit it — + // breaking the short-page end check below. + variables: { offset, limit: PAGE_SIZE }, + updateQuery: (prev, { fetchMoreResult }) => { + const more = fetchMoreResult?.logStreamDeliveries?.events ?? [] + if (!more.length) return prev + // Rows written between pages shift the offset window, so a page + // can re-serve rows already in the list — dedupe by id (they're + // also React keys). + const seen = new Set(prev.logStreamDeliveries.events.map((e: any) => e.id)) + const fresh = more.filter((e: any) => !seen.has(e.id)) + return { + ...prev, + logStreamDeliveries: { + ...prev.logStreamDeliveries, + events: [...prev.logStreamDeliveries.events, ...fresh], + count: fetchMoreResult.logStreamDeliveries.count, + }, + } + }, + }) + const rawPageSize = result.data?.logStreamDeliveries?.events?.length ?? 0 + rawFetchedRef.current = offset + rawPageSize + if (rawPageSize < PAGE_SIZE) { + setReachedEnd(true) + } + } finally { + setLoadingMore(false) + } + } + + // Refresh the visible rows after a retry in one request. The backend caps + // limit at 100, so beyond four pages the refresh truncates the accumulated + // list to the first 100 rows — a bounded trade-off vs collapsing to page + // one. loadMore passes its own limit explicitly, so the enlarged limit + // merged into the query variables here never leaks into later pagination. + const handleRetried = async () => { + rawFetchedRef.current = 0 + setReachedEnd(false) + try { + await refetch({ offset: 0, limit: Math.min(Math.max(events.length, PAGE_SIZE), 100) }) + } catch { + // The retry itself was queued; the list just didn't refresh. + } + } + + const setFilter = (filter: string | null) => { + setStatusFilter(filter) + rawFetchedRef.current = 0 + setReachedEnd(false) + } + + const filters: { label: string; value: string | null }[] = [ + { label: 'All', value: null }, + { label: 'Out of sync', value: 'unresolved' }, + { label: 'Completed', value: 'completed' }, + ] + + return ( +
+
+
+ {filters.map((filter) => ( + + ))} +
+
+ +
+
+ + {/* Fixed height so switching filters never shifts the dialog layout; + clamped so it stays comfortably tall on short laptop screens. The + list scrolls internally and pages in via Load more. */} +
+ {loading && events.length === 0 ? ( +
+ +
+ ) : events.length === 0 ? ( +
+ + +
+ } + > + <> + +
+ ) : ( + + + + + + + + + + + + + + {events.map((event, n) => ( + + {n !== 0 && n % PAGE_SIZE === 0 && ( + + + + )} + + + ))} + + + + + +
StatusSourceEventsAttemptsCreated
+
+ Page {n / PAGE_SIZE + 1} +
+
+
+ {!endOfList ? ( + + ) : ( + `No${events.length ? ' more' : ''} deliveries to show` + )} +
+
+ )} +
+
+ ) +} diff --git a/frontend/ee/components/logstreams/LogStreamForm.tsx b/frontend/ee/components/logstreams/LogStreamForm.tsx new file mode 100644 index 000000000..0f292cc54 --- /dev/null +++ b/frontend/ee/components/logstreams/LogStreamForm.tsx @@ -0,0 +1,280 @@ +import { ReactNode, useContext, useState } from 'react' +import { useMutation, useQuery } from '@apollo/client' +import { toast } from 'react-toastify' +import { FaVial } from 'react-icons/fa' +import { FaCheck, FaCircle } from 'react-icons/fa6' +import clsx from 'clsx' +import { + LogStreamProviderType, + LogStreamSourceLagType, + LogStreamSourceType, + ProviderCredentialsType, +} from '@/apollo/graphql' +import { GetLogStreamProviders } from '@/graphql/queries/logstreams/getLogStreamProviders.gql' +import { TestLogStreamConnectionOp } from '@/graphql/mutations/logstreams/testLogStreamConnection.gql' +import { Button } from '@/components/common/Button' +import { Input } from '@/components/common/Input' +import { ToggleSwitch } from '@/components/common/ToggleSwitch' +import { ProviderCredentialPicker } from '@/components/syncing/ProviderCredentialPicker' +import { ProviderIcon } from '@/components/syncing/ProviderIcon' +import { organisationContext } from '@/contexts/organisationContext' +import { SourceIcon } from './sourceMeta' +import { humanizeLag, lagIsCritical } from './utils' + +export type LogStreamFormValues = { + name: string + credential: ProviderCredentialsType | null + sources: string[] + service: string + tags: string + gzip: boolean + maxAttempts: number +} + +export const defaultLogStreamFormValues = (): LogStreamFormValues => ({ + name: '', + credential: null, + sources: [], + service: 'phase-console', + tags: '', + gzip: true, + maxAttempts: 5, +}) + +export const LogStreamForm = (props: { + provider: LogStreamProviderType + initialValues: LogStreamFormValues + submitLabel: string + submitting: boolean + onSubmit: (values: LogStreamFormValues, provider: LogStreamProviderType) => void + footerActions?: ReactNode + sourceLags?: (LogStreamSourceLagType | null)[] | null +}) => { + const { + provider, + initialValues, + submitLabel, + submitting, + onSubmit, + footerActions, + sourceLags, + } = props + + const { activeOrganisation: organisation } = useContext(organisationContext) + + const { data: providersData } = useQuery(GetLogStreamProviders) + const [testConnection, { loading: testing }] = useMutation(TestLogStreamConnectionOp) + + const allSources: LogStreamSourceType[] = providersData?.logStreamSources ?? [] + + const [values, setValues] = useState({ + ...initialValues, + sources: initialValues.sources.length + ? initialValues.sources + : allSources.map((source) => source.id), + }) + + const setValue = ( + key: K, + value: LogStreamFormValues[K] + ) => setValues((current) => ({ ...current, [key]: value })) + + const toggleSource = (sourceId: string) => + setValues((current) => ({ + ...current, + sources: current.sources.includes(sourceId) + ? current.sources.filter((id) => id !== sourceId) + : [...current.sources, sourceId], + })) + + const formValid = + values.name.trim().length > 0 && values.credential !== null && values.sources.length > 0 + + // Pristine forms must not fire a save — a no-op update still writes an + // audit event and ships it. + const normalize = (formValues: LogStreamFormValues) => + JSON.stringify({ + name: formValues.name.trim(), + credential: formValues.credential?.id ?? null, + sources: [...formValues.sources].sort(), + service: formValues.service, + tags: formValues.tags, + maxAttempts: formValues.maxAttempts, + }) + const dirty = normalize(values) !== normalize(initialValues) + + const lagBySource = Object.fromEntries( + (sourceLags ?? []).filter(Boolean).map((lag) => [lag!.source, lag!.lagSeconds]) + ) as Record + + // Stateless by design: tests whatever credential is currently selected, + // saved or not. + const handleTestConnection = async () => { + if (!values.credential) { + toast.error('Please select credentials to test the connection') + return + } + try { + const { data } = await testConnection({ + variables: { + organisationId: organisation!.id, + provider: provider.id, + credentialId: values.credential.id, + service: values.service, + tags: values.tags, + gzip: values.gzip, + }, + }) + const result = data?.testLogStreamConnection + if (result?.ok) toast.success(result.message || 'Connection successful') + else toast.error(result?.message || 'Connection failed') + } catch (error: any) { + toast.error(error.message || 'Connection failed') + } + } + + const handleSubmit = (event: React.FormEvent) => { + event.preventDefault() + if (!formValid) return + onSubmit(values, provider) + } + + return ( +
+ setValue('name', value)} + label="Name" + placeholder="e.g. Datadog production" + required + maxLength={64} + data-autofocus + /> + +
+
+ setValue('credential', credential)} + orgId={organisation!.id} + providerFilter={provider.credentialsProvider?.id} + setDefault={!initialValues.credential} + /> +
+
+ +
+
+ +
+
Event sources
+
+ {allSources.map((source) => ( +
+
+ +
+
+
+ {source.name} +
+ {values.sources.includes(source.id) && + lagBySource[source.id] !== undefined && ( + = 60 + ? 'text-amber-500 bg-amber-400/10 ring-amber-400/20' + : 'text-emerald-500 bg-emerald-400/10 ring-emerald-400/20' + )} + title="Shipping status for this source" + > + + {humanizeLag(lagBySource[source.id])} + + )} +
+
{source.description}
+
+
+
+ toggleSource(source.id)} + /> +
+
+ ))} +
+
+ +
+ +
+
+ + + {provider.name} destination + +
+ +
+ setValue('service', value)} + label="Service name" + placeholder="phase-console" + /> + setValue('tags', value)} + label="Tags (comma-separated key:value)" + placeholder="env:prod,team:platform" + /> +
+ +
+ + setValue('maxAttempts', Math.max(1, Math.min(10, Number(value) || 1))) + } + label="Retry attempts" + type="number" + min={1} + max={10} + /> +
+
+ +
+
{footerActions}
+ +
+ + ) +} diff --git a/frontend/ee/components/logstreams/LogStreamStatusIndicator.tsx b/frontend/ee/components/logstreams/LogStreamStatusIndicator.tsx new file mode 100644 index 000000000..41c8477ec --- /dev/null +++ b/frontend/ee/components/logstreams/LogStreamStatusIndicator.tsx @@ -0,0 +1,99 @@ +import clsx from 'clsx' +import { FaCircle } from 'react-icons/fa6' +import { FaCheckCircle, FaForward, FaTimesCircle } from 'react-icons/fa' +import { + ApiLogStreamDeliveryEventStatusChoices, + ApiLogStreamHealthChoices, + LogStreamType, +} from '@/apollo/graphql' +import { streamIsDelayed } from './utils' + +type StreamStatusKey = 'healthy' | 'delayed' | 'degraded' | 'paused' + +// Tinted-pill palette shared with RotationStatusBadge: bg-*-400/10 fill + +// ring-*-400/20. Degraded (deliveries failing) is the red attention state; +// paused/delayed are amber. +const STYLES: Record = { + healthy: { color: 'text-emerald-500', bg: 'bg-emerald-400/10', ring: 'ring-emerald-400/20', label: 'Healthy' }, + delayed: { color: 'text-amber-500', bg: 'bg-amber-400/10', ring: 'ring-amber-400/20', label: 'Delayed' }, + degraded: { color: 'text-red-500', bg: 'bg-red-400/10', ring: 'ring-red-400/20', label: 'Degraded' }, + paused: { color: 'text-amber-500', bg: 'bg-amber-400/10', ring: 'ring-amber-400/20', label: 'Paused' }, +} + +const resolveState = (stream: LogStreamType): StreamStatusKey => { + if (!stream.isActive) return 'paused' + // Degraded outranks delayed: a failing stream backs up within minutes, and + // the amber "running late" badge must not mask the red failure state (and + // its lastFailureReason tooltip) for the length of an outage. + if (stream.health === ApiLogStreamHealthChoices.Degraded) return 'degraded' + if (streamIsDelayed(stream)) return 'delayed' + return 'healthy' +} + +const PAUSED_TITLES: Record = { + auth_error: 'Paused: the destination rejected the configured credentials', + credentials_missing: 'Paused: the third-party credentials for this stream were deleted', + unknown_provider: 'Paused: no adapter is available for this provider', + invalid_options: "Paused: the stream's configuration failed validation", +} + +const titleFor = (stream: LogStreamType, key: StreamStatusKey): string => { + if (key === 'paused') return PAUSED_TITLES[stream.pausedReason ?? ''] || 'Paused' + if (key === 'delayed') return 'New events are queued but deliveries are running late' + if (key === 'degraded') return stream.lastFailureReason || 'Some deliveries are failing' + return 'Shipping normally' +} + +export const LogStreamStatusIndicator = (props: { + stream: LogStreamType + size?: 'sm' | 'md' +}) => { + const { stream, size = 'md' } = props + const key = resolveState(stream) + const style = STYLES[key] + + return ( + + + {style.label} + + ) +} + +export const DeliveryStatusIndicator = (props: { + status: ApiLogStreamDeliveryEventStatusChoices | string + showLabel?: boolean +}) => { + const { status, showLabel } = props + + const statusValue = String(status).toLowerCase() + + if (statusValue === 'completed') + return ( +
+ {showLabel && 'Completed'} +
+ ) + + if (statusValue === 'failed') + return ( +
+ {showLabel && 'Failed'} +
+ ) + + return ( +
+ {showLabel && 'Skipped'} +
+ ) +} diff --git a/frontend/ee/components/logstreams/ManageLogStreamDialog.tsx b/frontend/ee/components/logstreams/ManageLogStreamDialog.tsx new file mode 100644 index 000000000..57836690f --- /dev/null +++ b/frontend/ee/components/logstreams/ManageLogStreamDialog.tsx @@ -0,0 +1,278 @@ +import { Fragment, forwardRef, useContext, useImperativeHandle, useRef, useState } from 'react' +import { Tab } from '@headlessui/react' +import { useMutation } from '@apollo/client' +import clsx from 'clsx' +import { toast } from 'react-toastify' +import { FaExternalLinkAlt } from 'react-icons/fa' +import { FaCircleExclamation, FaPause, FaPlay } from 'react-icons/fa6' +import { + ApiLogStreamHealthChoices, + LogStreamProviderType, + LogStreamType, +} from '@/apollo/graphql' +import { UpdateLogStreamOp } from '@/graphql/mutations/logstreams/updateLogStream.gql' +import { ToggleLogStreamOp } from '@/graphql/mutations/logstreams/toggleLogStream.gql' +import { GetLogStreams } from '@/graphql/queries/logstreams/getLogStreams.gql' +import GenericDialog from '@/components/common/GenericDialog' +import { Alert } from '@/components/common/Alert' +import { Button } from '@/components/common/Button' +import { ProviderIcon } from '@/components/syncing/ProviderIcon' +import { organisationContext } from '@/contexts/organisationContext' +import { relativeTimeFromDates } from '@/utils/time' +import { DeleteLogStreamDialog } from './DeleteLogStreamDialog' +import { LogStreamDeliveryHistory } from './LogStreamDeliveryHistory' +import { LogStreamForm, LogStreamFormValues } from './LogStreamForm' +import { LogStreamStatusIndicator } from './LogStreamStatusIndicator' +import { parseStreamOptions } from './utils' + +export type ManageLogStreamDialogHandle = { + openSettings: () => void + openHistory: (statusFilter?: string) => void +} + +export const ManageLogStreamDialog = forwardRef< + ManageLogStreamDialogHandle, + { + stream: LogStreamType + userCanUpdate: boolean + userCanDelete: boolean + } +>((props, ref) => { + const { stream, userCanUpdate, userCanDelete } = props + + const { activeOrganisation: organisation } = useContext(organisationContext) + + const dialogRef = useRef<{ openModal: () => void; closeModal: () => void }>(null) + const [tabIndex, setTabIndex] = useState(0) + const [historyFilter, setHistoryFilter] = useState(undefined) + + useImperativeHandle(ref, () => ({ + openSettings: () => { + // Clear any filter left over from a previous "out of sync" shortcut — + // it would silently pre-filter the Events tab on this visit. + setHistoryFilter(undefined) + setTabIndex(0) + dialogRef.current?.openModal() + }, + openHistory: (statusFilter?: string) => { + setHistoryFilter(statusFilter) + setTabIndex(1) + dialogRef.current?.openModal() + }, + })) + + const closeModal = () => dialogRef.current?.closeModal() + + const refetchStreams = [ + { query: GetLogStreams, variables: { organisationId: organisation?.id } }, + ] + + const [updateLogStream, { loading: updating }] = useMutation(UpdateLogStreamOp) + const [toggleLogStream, { loading: toggling }] = useMutation(ToggleLogStreamOp) + + const handleUpdate = async ( + values: LogStreamFormValues, + _provider: LogStreamProviderType + ) => { + try { + await updateLogStream({ + variables: { + streamId: stream.id, + name: values.name, + credentialId: values.credential!.id, + sources: values.sources, + service: values.service, + tags: values.tags, + gzip: values.gzip, + maxAttempts: values.maxAttempts, + }, + refetchQueries: refetchStreams, + }) + toast.success('Log Stream updated') + } catch (error: any) { + toast.error(error.message || 'Could not update the Log Stream') + } + } + + const handleToggle = async () => { + try { + await toggleLogStream({ + variables: { streamId: stream.id }, + refetchQueries: refetchStreams, + }) + toast.success(stream.isActive ? 'Log Stream paused' : 'Log Stream resumed') + } catch (error: any) { + toast.error(error.message || 'Could not update the Log Stream') + } + } + + const streamOptions = parseStreamOptions(stream.options) + + const initialFormValues: LogStreamFormValues = { + name: stream.name, + credential: stream.authentication ?? null, + sources: (stream.sources as string[]) ?? [], + service: streamOptions.service ?? 'phase-console', + tags: streamOptions.tags ?? '', + gzip: streamOptions.gzip ?? true, + maxAttempts: stream.maxAttempts, + } + + const dialogTitle = ( +
+

+ {stream.name} +

+ {stream.providerInfo && ( +
+ + + Stream logs to{' '} + + {stream.providerInfo.name} + + + {stream.destinationUrl && ( + + + + )} +
+ )} +
+ ) + + const statusSection = ( +
+
Status
+
+
+ + + Last shipped:{' '} + {stream.lastShippedAt + ? relativeTimeFromDates(new Date(stream.lastShippedAt)) + : 'never'} + +
+ {userCanUpdate && ( + + )} +
+
+ ) + + return ( + +
+ {!stream.isActive && stream.pausedReason === 'auth_error' && ( + + This stream was paused because the destination rejected the configured credentials. + Update the credentials and resume the stream. + + )} + + {!stream.isActive && stream.pausedReason === 'credentials_missing' && ( + + This stream was paused because its third-party credentials were deleted. Select new + credentials, save, and resume the stream. + + )} + + {stream.health === ApiLogStreamHealthChoices.Degraded && stream.lastFailureReason && ( +
+
+ Last failure +
+
{stream.lastFailureReason}
+ {stream.lastFailureAt && ( +
+ {relativeTimeFromDates(new Date(stream.lastFailureAt))} +
+ )} +
+ )} + + + + {['Configuration', 'Events'].map((tab) => ( + + {({ selected }) => ( +
+ {tab} +
+ )} + + ))} + + + + {statusSection} + + {userCanUpdate ? ( + stream.providerInfo && ( + + ) : undefined + } + /> + ) + ) : ( +
+
+ You don't have permission to edit this Log Stream. +
+ {/* Delete is a separate permission — a role can hold it without update. */} + {userCanDelete && ( + + )} +
+ )} + + + + + + +
+ + ) +}) + +ManageLogStreamDialog.displayName = 'ManageLogStreamDialog' diff --git a/frontend/ee/components/logstreams/sourceMeta.tsx b/frontend/ee/components/logstreams/sourceMeta.tsx new file mode 100644 index 000000000..599d361a1 --- /dev/null +++ b/frontend/ee/components/logstreams/sourceMeta.tsx @@ -0,0 +1,15 @@ +import { FaCubes, FaSitemap, FaStream } from 'react-icons/fa' + +/** + * Icons for the log stream event sources, keyed by registry id (org = + * sitemap, apps = cubes). Names and descriptions come from the backend source + * registry (logStreamSources / sourceLags.name) — only the visual identity + * lives client-side. + */ +export const SourceIcon = (props: { sourceId: string; className?: string }) => { + const { sourceId, className } = props + + if (sourceId === 'org_audit') return + if (sourceId === 'secrets') return + return +} diff --git a/frontend/ee/components/logstreams/utils.ts b/frontend/ee/components/logstreams/utils.ts new file mode 100644 index 000000000..b8b72efea --- /dev/null +++ b/frontend/ee/components/logstreams/utils.ts @@ -0,0 +1,67 @@ +export type LogStreamOptions = { + service?: string + tags?: string + gzip?: boolean +} + +/** + * Graphene serializes JSONField via the JSONString scalar — the client + * receives a JSON-encoded *string*, not an object. Reading properties off it + * directly silently yields undefined (and would reset stored options on the + * next save). + */ +export const parseStreamOptions = (raw: unknown): LogStreamOptions => { + if (typeof raw === 'string') { + try { + return JSON.parse(raw) + } catch { + return {} + } + } + return (raw as LogStreamOptions) ?? {} +} + +/** + * Human-readable cursor lag: "Up to date", "26 minutes behind", "3 days behind". + * The operator's backpressure signal for a log stream source. + */ +export const humanizeLag = (lagSeconds: number): string => { + if (lagSeconds < 60) return 'Up to date' + + const units: [number, string][] = [ + [7 * 24 * 3600, 'week'], + [24 * 3600, 'day'], + [3600, 'hour'], + [60, 'minute'], + ] + + for (const [seconds, label] of units) { + if (lagSeconds >= seconds) { + const value = Math.floor(lagSeconds / seconds) + return `${value} ${label}${value === 1 ? '' : 's'} behind` + } + } + + return 'Up to date' +} + +export const lagIsCritical = (lagSeconds: number, maxEventAgeHours?: number | null): boolean => { + if (!maxEventAgeHours) return false + // Within 2h of the destination's max event age, the backlog is at risk of + // being skipped. + return lagSeconds > (maxEventAgeHours - 2) * 3600 +} + +// lagSeconds is the age of the oldest event still waiting to ship (delivery +// delay), so a sustained value above this means deliveries genuinely aren't +// keeping up — surfaced as the neutral "Delayed" status. +const DELAY_THRESHOLD_SECONDS = 300 + +export const worstLag = (stream: { + sourceLags?: ({ lagSeconds: number } | null)[] | null +}): number => Math.max(0, ...(stream.sourceLags?.map((lag) => lag?.lagSeconds ?? 0) ?? [0])) + +export const streamIsDelayed = (stream: { + isActive: boolean + sourceLags?: ({ lagSeconds: number } | null)[] | null +}): boolean => stream.isActive && worstLag(stream) > DELAY_THRESHOLD_SECONDS diff --git a/frontend/graphql/mutations/logstreams/createLogStream.gql b/frontend/graphql/mutations/logstreams/createLogStream.gql new file mode 100644 index 000000000..8dcc23c67 --- /dev/null +++ b/frontend/graphql/mutations/logstreams/createLogStream.gql @@ -0,0 +1,28 @@ +mutation CreateNewLogStream( + $organisationId: ID! + $name: String! + $provider: String! + $credentialId: ID! + $sources: [String!]! + $service: String + $tags: String + $gzip: Boolean + $maxAttempts: Int +) { + createLogStream( + organisationId: $organisationId + name: $name + provider: $provider + credentialId: $credentialId + sources: $sources + service: $service + tags: $tags + gzip: $gzip + maxAttempts: $maxAttempts + ) { + logStream { + id + name + } + } +} diff --git a/frontend/graphql/mutations/logstreams/deleteLogStream.gql b/frontend/graphql/mutations/logstreams/deleteLogStream.gql new file mode 100644 index 000000000..4d5423947 --- /dev/null +++ b/frontend/graphql/mutations/logstreams/deleteLogStream.gql @@ -0,0 +1,5 @@ +mutation DeleteLogStreamOp($streamId: ID!) { + deleteLogStream(streamId: $streamId) { + ok + } +} diff --git a/frontend/graphql/mutations/logstreams/retryLogStreamDelivery.gql b/frontend/graphql/mutations/logstreams/retryLogStreamDelivery.gql new file mode 100644 index 000000000..a78fb3c24 --- /dev/null +++ b/frontend/graphql/mutations/logstreams/retryLogStreamDelivery.gql @@ -0,0 +1,5 @@ +mutation RetryLogStreamDeliveryOp($deliveryEventId: ID!) { + retryLogStreamDelivery(deliveryEventId: $deliveryEventId) { + ok + } +} diff --git a/frontend/graphql/mutations/logstreams/testLogStreamConnection.gql b/frontend/graphql/mutations/logstreams/testLogStreamConnection.gql new file mode 100644 index 000000000..e8ec5086e --- /dev/null +++ b/frontend/graphql/mutations/logstreams/testLogStreamConnection.gql @@ -0,0 +1,20 @@ +mutation TestLogStreamConnectionOp( + $organisationId: ID! + $provider: String! + $credentialId: ID! + $service: String + $tags: String + $gzip: Boolean +) { + testLogStreamConnection( + organisationId: $organisationId + provider: $provider + credentialId: $credentialId + service: $service + tags: $tags + gzip: $gzip + ) { + ok + message + } +} diff --git a/frontend/graphql/mutations/logstreams/toggleLogStream.gql b/frontend/graphql/mutations/logstreams/toggleLogStream.gql new file mode 100644 index 000000000..5980e566f --- /dev/null +++ b/frontend/graphql/mutations/logstreams/toggleLogStream.gql @@ -0,0 +1,9 @@ +mutation ToggleLogStreamOp($streamId: ID!) { + toggleLogStream(streamId: $streamId) { + logStream { + id + isActive + pausedReason + } + } +} diff --git a/frontend/graphql/mutations/logstreams/updateLogStream.gql b/frontend/graphql/mutations/logstreams/updateLogStream.gql new file mode 100644 index 000000000..ed9baa84b --- /dev/null +++ b/frontend/graphql/mutations/logstreams/updateLogStream.gql @@ -0,0 +1,26 @@ +mutation UpdateLogStreamOp( + $streamId: ID! + $name: String! + $credentialId: ID! + $sources: [String!]! + $service: String + $tags: String + $gzip: Boolean + $maxAttempts: Int +) { + updateLogStream( + streamId: $streamId + name: $name + credentialId: $credentialId + sources: $sources + service: $service + tags: $tags + gzip: $gzip + maxAttempts: $maxAttempts + ) { + logStream { + id + name + } + } +} diff --git a/frontend/graphql/queries/logstreams/getLogStreamDeliveries.gql b/frontend/graphql/queries/logstreams/getLogStreamDeliveries.gql new file mode 100644 index 000000000..54e4c395f --- /dev/null +++ b/frontend/graphql/queries/logstreams/getLogStreamDeliveries.gql @@ -0,0 +1,22 @@ +query GetLogStreamDeliveries($streamId: ID!, $limit: Int, $offset: Int, $status: String) { + logStreamDeliveries(streamId: $streamId, limit: $limit, offset: $offset, status: $status) { + count + events { + id + source + status + eventCount + payloadBytes + attempts + cursorFrom + cursorTo + retriedFrom { + id + } + resolvedAt + meta + createdAt + completedAt + } + } +} diff --git a/frontend/graphql/queries/logstreams/getLogStreamProviders.gql b/frontend/graphql/queries/logstreams/getLogStreamProviders.gql new file mode 100644 index 000000000..8bf264c0d --- /dev/null +++ b/frontend/graphql/queries/logstreams/getLogStreamProviders.gql @@ -0,0 +1,18 @@ +query GetLogStreamProviders { + logStreamProviders { + id + name + maxEventAgeHours + credentialsProvider { + id + name + expectedCredentials + optionalCredentials + } + } + logStreamSources { + id + name + description + } +} diff --git a/frontend/graphql/queries/logstreams/getLogStreams.gql b/frontend/graphql/queries/logstreams/getLogStreams.gql new file mode 100644 index 000000000..efd3b21f3 --- /dev/null +++ b/frontend/graphql/queries/logstreams/getLogStreams.gql @@ -0,0 +1,41 @@ +query GetLogStreams($organisationId: ID!) { + logStreams(organisationId: $organisationId) { + id + name + provider + providerInfo { + id + name + maxEventAgeHours + credentialsProvider { + id + name + } + } + authentication { + id + name + } + sources + options + maxAttempts + isActive + health + pausedReason + lastShippedAt + lastFailureAt + lastFailureReason + createdAt + unresolvedFailures + destinationUrl + sourceLags { + source + name + lagSeconds + } + deliverySummary { + completed + failed + } + } +} diff --git a/frontend/hooks/useSecretSearch.ts b/frontend/hooks/useSecretSearch.ts index dd38cb43e..e7221a588 100644 --- a/frontend/hooks/useSecretSearch.ts +++ b/frontend/hooks/useSecretSearch.ts @@ -91,8 +91,15 @@ export const useSecretSearch = ( setLoading(false) } - const filtered = cacheRef.current!.filter((s) => - normalize(s.key).includes(normalizedQuery) + // Id-shaped queries (e.g. pasted from a SIEM event) also match on the + // secret id — gated so short name searches don't hit every uuid. + const idQuery = query.trim().toLowerCase() + const queryLooksLikeId = /^[0-9a-f-]{8,}$/.test(idQuery) + + const filtered = cacheRef.current!.filter( + (s) => + normalize(s.key).includes(normalizedQuery) || + (queryLooksLikeId && s.id.toLowerCase().includes(idQuery)) ) setResults(filtered) } diff --git a/frontend/tests/utils/datadogSites.test.ts b/frontend/tests/utils/datadogSites.test.ts new file mode 100644 index 000000000..d73bd77ed --- /dev/null +++ b/frontend/tests/utils/datadogSites.test.ts @@ -0,0 +1,25 @@ +import { datadogSites } from '@/utils/syncing/datadog' + +describe('datadogSites', () => { + test('covers every Datadog region including UK1 and US2-FED', () => { + const sites = datadogSites.map((s) => s.site) + expect(sites).toEqual([ + 'datadoghq.com', + 'us3.datadoghq.com', + 'us5.datadoghq.com', + 'datadoghq.eu', + 'uk1.datadoghq.com', + 'ap1.datadoghq.com', + 'ap2.datadoghq.com', + 'ddog-gov.com', + 'us2.ddog-gov.com', + ]) + }) + + test('first entry is the US1 default seeded by CreateProviderCredentials', () => { + // handleProviderChange seeds credentials.site from datadogSites[0] — + // reordering the list silently changes the default region new + // credentials are saved with. + expect(datadogSites[0].site).toBe('datadoghq.com') + }) +}) diff --git a/frontend/utils/syncing/datadog.ts b/frontend/utils/syncing/datadog.ts new file mode 100644 index 000000000..203813ef8 --- /dev/null +++ b/frontend/utils/syncing/datadog.ts @@ -0,0 +1,18 @@ +export type DatadogSite = { + site: string + name: string +} + +// Mirrors the backend allowlist in backend/api/services.py (DATADOG_SITES) — +// keep the two in sync when Datadog adds a region. +export const datadogSites: DatadogSite[] = [ + { site: 'datadoghq.com', name: 'US1 (datadoghq.com)' }, + { site: 'us3.datadoghq.com', name: 'US3 (us3.datadoghq.com)' }, + { site: 'us5.datadoghq.com', name: 'US5 (us5.datadoghq.com)' }, + { site: 'datadoghq.eu', name: 'EU1 (datadoghq.eu)' }, + { site: 'uk1.datadoghq.com', name: 'UK1 (uk1.datadoghq.com)' }, + { site: 'ap1.datadoghq.com', name: 'AP1 (ap1.datadoghq.com)' }, + { site: 'ap2.datadoghq.com', name: 'AP2 (ap2.datadoghq.com)' }, + { site: 'ddog-gov.com', name: 'US1-FED (ddog-gov.com)' }, + { site: 'us2.ddog-gov.com', name: 'US2-FED (us2.ddog-gov.com)' }, +] diff --git a/frontend/utils/syncing/general.ts b/frontend/utils/syncing/general.ts index 2b959bc44..7ee0607c2 100644 --- a/frontend/utils/syncing/general.ts +++ b/frontend/utils/syncing/general.ts @@ -33,4 +33,4 @@ export const encryptProviderCredentials = async ( } export const isCredentialSecret = (credential: string) => - !/(?:addr|host|url)/i.test(credential.toLowerCase()) + !/(?:addr|host|url|site)/i.test(credential.toLowerCase())