From 2f60887284d24cd92e5ac3802055cb8f8ea1ea5a Mon Sep 17 00:00:00 2001 From: Ben Corlett Date: Tue, 10 Jun 2025 13:30:40 +0100 Subject: [PATCH] Add otel metrics alongside the statsd ones --- .pre-commit-config.yaml | 2 +- app/__init__.py | 18 ++-- app/celery/nightly_tasks.py | 7 +- app/celery/process_ses_receipts_tasks.py | 15 ++- .../process_sms_client_response_tasks.py | 30 +++++- app/celery/scheduled_tasks.py | 23 +++- app/clients/email/aws_ses.py | 27 ++++- app/clients/email/aws_ses_stub.py | 19 +++- app/clients/letter/dvla.py | 4 +- app/clients/sms/__init__.py | 21 +++- app/commands.py | 4 + app/config.py | 6 ++ app/delivery/send_to_providers.py | 41 ++++++- requirements.in | 2 +- requirements.txt | 72 ++++++++++++- requirements_for_test.txt | 100 +++++++++++++++++- requirements_for_test_common.in | 2 +- ruff.toml | 2 +- .../celery/test_process_ses_receipts_tasks.py | 16 ++- .../test_process_sms_client_response_tasks.py | 15 ++- tests/app/celery/test_scheduled_tasks.py | 63 +++++++++++ tests/app/clients/test_aws_ses.py | 9 +- tests/app/clients/test_dvla.py | 7 +- tests/app/clients/test_sms.py | 4 +- tests/app/conftest.py | 6 +- tests/app/v2/test_errors.py | 3 +- 26 files changed, 486 insertions(+), 32 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a901ba9cad..76329c7171 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -# This file was automatically copied from notifications-utils@99.8.0 +# This file was automatically copied from notifications-utils@100.1.0 repos: - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/app/__init__.py b/app/__init__.py index 9175c2b737..ed94713734 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -23,6 +23,7 @@ from gds_metrics.metrics import Gauge, Histogram from notifications_utils import request_helper from notifications_utils.celery import NotifyCelery +from notifications_utils.clients.otel.otel_client import OtelClient from notifications_utils.clients.redis.redis_client import RedisClient from notifications_utils.clients.signing.signing_client import Signing from notifications_utils.clients.statsd.statsd_client import StatsdClient @@ -49,6 +50,7 @@ notify_celery = NotifyCelery() signing = Signing() statsd_client = StatsdClient() +otel_client = OtelClient() redis_store = RedisClient() cbc_proxy_client = CBCProxyClient() metrics = GDSMetrics() @@ -70,7 +72,7 @@ _firetext_client_context_var: ContextVar[FiretextClient] = ContextVar("firetext_client") get_firetext_client: LazyLocalGetter[FiretextClient] = LazyLocalGetter( _firetext_client_context_var, - lambda: FiretextClient(current_app, statsd_client=statsd_client), + lambda: FiretextClient(current_app, statsd_client=statsd_client, otel_client=otel_client), expected_type=FiretextClient, ) memo_resetters.append(lambda: get_firetext_client.clear()) @@ -79,7 +81,7 @@ _mmg_client_context_var: ContextVar[MMGClient] = ContextVar("mmg_client") get_mmg_client: LazyLocalGetter[MMGClient] = LazyLocalGetter( _mmg_client_context_var, - lambda: MMGClient(current_app, statsd_client=statsd_client), + lambda: MMGClient(current_app, statsd_client=statsd_client, otel_client=otel_client), expected_type=MMGClient, ) memo_resetters.append(lambda: get_mmg_client.clear()) @@ -88,7 +90,7 @@ _aws_ses_client_context_var: ContextVar[AwsSesClient] = ContextVar("aws_ses_client") get_aws_ses_client: LazyLocalGetter[AwsSesClient] = LazyLocalGetter( _aws_ses_client_context_var, - lambda: AwsSesClient(current_app.config["AWS_REGION"], statsd_client=statsd_client), + lambda: AwsSesClient(current_app.config["AWS_REGION"], statsd_client=statsd_client, otel_client=otel_client), expected_type=AwsSesClient, ) memo_resetters.append(lambda: get_aws_ses_client.clear()) @@ -99,8 +101,9 @@ _aws_ses_stub_client_context_var, lambda: AwsSesStubClient( current_app.config["AWS_REGION"], - statsd_client=statsd_client, stub_url=current_app.config["SES_STUB_URL"], + statsd_client=statsd_client, + otel_client=otel_client, ), expected_type=AwsSesStubClient, ) @@ -135,7 +138,7 @@ _dvla_client_context_var: ContextVar[DVLAClient] = ContextVar("dvla_client") get_dvla_client: LazyLocalGetter[DVLAClient] = LazyLocalGetter( _dvla_client_context_var, - lambda: DVLAClient(current_app, statsd_client=statsd_client), + lambda: DVLAClient(current_app, statsd_client=statsd_client, otel_client=otel_client), ) memo_resetters.append(lambda: get_dvla_client.clear()) dvla_client = LocalProxy(get_dvla_client) @@ -160,6 +163,8 @@ def create_app(application): from app.config import Config, configs + global notify_otel_client + notify_environment = os.environ["NOTIFY_ENVIRONMENT"] if notify_environment in configs: @@ -177,7 +182,8 @@ def create_app(application): migrate.init_app(application, db=db) ma.init_app(application) statsd_client.init_app(application) - utils_logging.init_app(application, statsd_client) + otel_client.init_app(application) + utils_logging.init_app(application, statsd_client, otel_client) notify_celery.init_app(application) signing.init_app(application) diff --git a/app/celery/nightly_tasks.py b/app/celery/nightly_tasks.py index 951992216d..7548d72c30 100644 --- a/app/celery/nightly_tasks.py +++ b/app/celery/nightly_tasks.py @@ -13,7 +13,7 @@ from sqlalchemy import func from sqlalchemy.exc import SQLAlchemyError -from app import notify_celery, statsd_client, zendesk_client +from app import notify_celery, otel_client, statsd_client, zendesk_client from app.aws import s3 from app.config import QueueNames from app.constants import ( @@ -250,6 +250,11 @@ def timeout_notifications(): for notification in notifications: statsd_client.incr(f"timeout-sending.{notification.sent_by}") + otel_client.incr( + "timeout_sending", + attributes={"notification_type": notification.notification_type}, + description="Count of notifications that have timed out while sending", + ) check_and_queue_callback_task(notification) current_app.logger.info( diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 42aa439d39..170cd2f195 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -6,7 +6,7 @@ from flask import current_app, json from sqlalchemy.orm.exc import NoResultFound -from app import notify_celery, statsd_client +from app import notify_celery, otel_client, statsd_client from app.clients.email.aws_ses import get_aws_responses from app.config import QueueNames from app.constants import NOTIFICATION_PENDING, NOTIFICATION_SENDING @@ -79,11 +79,24 @@ def process_ses_results(self, response): statsd_client.incr(f"callback.ses.{notification_status}") + otel_client.incr( + "callback_success", + attributes={"provider": "ses", "status": notification_status}, + description="Count of successful SES callbacks with status", + ) + if notification.sent_at: statsd_client.timing_with_dates( f"callback.ses.{notification_status}.elapsed-time", datetime.utcnow(), notification.sent_at ) + otel_client.record( + "callback_elapsed_time", + value=(datetime.utcnow() - notification.sent_at).total_seconds(), + attributes={"provider": "ses", "status": notification_status}, + description="Elapsed time for SES callback with status", + ) + check_and_queue_callback_task(notification) return True diff --git a/app/celery/process_sms_client_response_tasks.py b/app/celery/process_sms_client_response_tasks.py index c184a43dee..e6ab21c740 100644 --- a/app/celery/process_sms_client_response_tasks.py +++ b/app/celery/process_sms_client_response_tasks.py @@ -5,7 +5,7 @@ from flask import current_app from notifications_utils.template import SMSMessageTemplate -from app import notify_celery, statsd_client +from app import notify_celery, otel_client, statsd_client from app.clients import ClientException from app.clients.sms.firetext import get_firetext_responses from app.clients.sms.mmg import get_mmg_responses @@ -74,6 +74,15 @@ def _process_for_status(notification_status, client_name, provider_reference, de statsd_client.incr(f"callback.{client_name.lower()}.{notification_status}") + otel_client.incr( + "callback", + attributes={ + "provider": client_name.lower(), + "status": notification_status, + }, + description="Count of callbacks", + ) + if notification.sent_at: statsd_client.timing_with_dates( f"callback.{client_name.lower()}.{notification_status}.elapsed-time", @@ -81,6 +90,16 @@ def _process_for_status(notification_status, client_name, provider_reference, de notification.sent_at, ) + otel_client.record( + "callback_elapsed_time", + value=(datetime.utcnow() - notification.sent_at).total_seconds(), + attributes={ + "provider": client_name.lower(), + "status": notification_status, + }, + description="Elapsed time for callbacks", + ) + if notification.billable_units == 0: service = notification.service template_model = dao_get_template_by_id(notification.template_id, notification.template_version) @@ -98,3 +117,12 @@ def _process_for_status(notification_status, client_name, provider_reference, de check_and_queue_callback_task(notification) if notification.international: statsd_client.incr(f"international-sms.{notification_status}.{notification.phone_prefix}") + + otel_client.incr( + "international_sms", + attributes={ + "status": notification_status, + "phone_prefix": notification.phone_prefix, + }, + description="Count of international SMS", + ) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 6cf1edb3ea..405688c3ed 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -18,7 +18,7 @@ from sqlalchemy import and_, between from sqlalchemy.exc import SQLAlchemyError -from app import db, dvla_client, notify_celery, redis_store, statsd_client, zendesk_client +from app import db, dvla_client, notify_celery, otel_client, redis_store, statsd_client, zendesk_client from app.aws import s3 from app.celery.letters_pdf_tasks import get_pdf_for_templated_letter from app.celery.tasks import ( @@ -216,6 +216,17 @@ def generate_sms_delivery_stats(): f"slow-delivery.{report.provider}.delivered-within-minutes.{delivery_interval}.ratio", report.slow_ratio ) + otel_client.record( + "slow_delivery_ratio", + value=report.slow_ratio, + attributes={ + "provider": report.provider, + "delivery_interval": delivery_interval, + }, + description="Ratio of slow message deliveries for in the last 15 minutes " + "for deliveries taking longer than delivery_interval minutes", + ) + total_notifications = sum(report.total_notifications for report in providers_slow_delivery_reports) slow_notifications = sum(report.slow_notifications for report in providers_slow_delivery_reports) ratio_slow_notifications = slow_notifications / total_notifications @@ -224,6 +235,16 @@ def generate_sms_delivery_stats(): f"slow-delivery.sms.delivered-within-minutes.{delivery_interval}.ratio", ratio_slow_notifications ) + otel_client.record( + "slow_sms_delivery_ratio", + value=ratio_slow_notifications, + attributes={ + "delivery_interval": delivery_interval, + }, + description="Ratio of slow message deliveries for in the last 15 minutes " + "for deliveries taking longer than delivery_interval minutes", + ) + # For the 5-minute delivery interval, let's check the percentage of all text messages sent that were slow. # TODO: delete this when we have a way to raise these alerts from eg grafana, prometheus, something else. if delivery_interval == 5 and current_app.should_check_slow_text_message_delivery: diff --git a/app/clients/email/aws_ses.py b/app/clients/email/aws_ses.py index 5ac6e6262f..7dd44e3c00 100644 --- a/app/clients/email/aws_ses.py +++ b/app/clients/email/aws_ses.py @@ -60,10 +60,11 @@ class AwsSesClient(EmailClient): name = "ses" - def __init__(self, region, statsd_client): + def __init__(self, region, statsd_client, otel_client): super().__init__() self._client = boto3.client("sesv2", region_name=region) self.statsd_client = statsd_client + self.otel_client = otel_client def send_email( self, @@ -103,6 +104,12 @@ def send_email( except botocore.exceptions.ClientError as e: self.statsd_client.incr("clients.ses.error") + self.otel_client.incr( + "clients_error", + attributes={"provider": self.name}, + description="Count of failed requests to provider", + ) + # https://docs.aws.amazon.com/ses/latest/APIReference-V2/API_SendEmail.html#API_SendEmail_Errors if e.response["Error"]["Code"] == "InvalidParameterValue": raise EmailClientNonRetryableException(e.response["Error"]["Message"]) from e @@ -110,6 +117,12 @@ def send_email( raise AwsSesClientThrottlingSendRateException(str(e)) from e else: self.statsd_client.incr("clients.ses.error") + + self.otel_client.incr( + "clients_error", + attributes={"provider": self.name}, + description="Count of failed requests to provider", + ) raise AwsSesClientException(str(e) + e.response["Error"]["Code"]) from e except Exception as e: self.statsd_client.incr("clients.ses.error") @@ -125,6 +138,18 @@ def send_email( ) self.statsd_client.timing("clients.ses.request-time", elapsed_time) self.statsd_client.incr("clients.ses.success") + + self.otel_client.record( + "clients_request_time", + value=elapsed_time, + attributes={"provider": self.name}, + description="Time taken for request to provider", + ) + self.otel_client.incr( + "clients_success", + attributes={"provider": self.name}, + description="Count of successful requests to provider", + ) return response["MessageId"] diff --git a/app/clients/email/aws_ses_stub.py b/app/clients/email/aws_ses_stub.py index 4023d734c4..4458237588 100644 --- a/app/clients/email/aws_ses_stub.py +++ b/app/clients/email/aws_ses_stub.py @@ -20,9 +20,10 @@ class AwsSesStubClient(EmailClient): name = "ses" - def __init__(self, region, statsd_client, stub_url): + def __init__(self, region, stub_url, statsd_client, otel_client): super().__init__() self.statsd_client = statsd_client + self.otel_client = otel_client self.url = stub_url self.requests_session = requests.Session() @@ -45,10 +46,26 @@ def send_email( except Exception as e: self.statsd_client.incr("clients.ses_stub.error") + self.otel_client.incr( + "clients_error", + attributes={"provider": self.name}, + description="Count of failed requests to provider", + ) raise AwsSesStubClientException(str(e)) from e else: elapsed_time = monotonic() - start_time current_app.logger.info("AWS SES stub request finished in %s", elapsed_time) self.statsd_client.timing("clients.ses_stub.request-time", elapsed_time) self.statsd_client.incr("clients.ses_stub.success") + self.otel_client.record( + "clients_request_time", + value=elapsed_time, + attributes={"provider": self.name}, + description="Time taken for requests provider", + ) + self.otel_client.incr( + "clients_success", + attributes={"provider": self.name}, + description="Count of successful requests to provider", + ) return response_json["MessageId"] diff --git a/app/clients/letter/dvla.py b/app/clients/letter/dvla.py index 50e6331f02..403e778d3d 100644 --- a/app/clients/letter/dvla.py +++ b/app/clients/letter/dvla.py @@ -112,11 +112,12 @@ class DVLAClient: name = "dvla" statsd_client = None + otel_client = None _jwt_token = None _jwt_expires_at = None - def __init__(self, application, *, statsd_client): + def __init__(self, application, *, statsd_client, otel_client): self.base_url = application.config["DVLA_API_BASE_URL"] self.ciphers = application.config["DVLA_API_TLS_CIPHERS"] ssm_client = boto3.client("ssm", region_name=application.config["AWS_REGION"]) @@ -124,6 +125,7 @@ def __init__(self, application, *, statsd_client): self.dvla_password = SSMParameter(key="/notify/api/dvla_password", ssm_client=ssm_client) self.dvla_api_key = SSMParameter(key="/notify/api/dvla_api_key", ssm_client=ssm_client) self.statsd_client = statsd_client + self.otel_client = otel_client self.session = requests.Session() self.session.mount(self.base_url, _SpecifiedCiphersAdapter(ciphers=self.ciphers)) diff --git a/app/clients/sms/__init__.py b/app/clients/sms/__init__.py index f06cf3ecad..3017a38166 100644 --- a/app/clients/sms/__init__.py +++ b/app/clients/sms/__init__.py @@ -26,10 +26,11 @@ class SmsClient(Client): Base Sms client for sending smss. """ - def __init__(self, current_app, statsd_client): + def __init__(self, current_app, statsd_client, otel_client): super().__init__() self.current_app = current_app self.statsd_client = statsd_client + self.otel_client = otel_client self.requests_session = requests.Session() if platform.system() == "Linux": # these are linux-specific socket options enabling tcp keepalive @@ -49,8 +50,19 @@ def record_outcome(self, success): if success: self.current_app.logger.info("Provider request for %s %s", self.name, "succeeded" if success else "failed") self.statsd_client.incr(f"clients.{self.name}.success") + + self.otel_client.incr( + "clients_success", + attributes={"provider": self.name}, + description="Count of successful requests to provider", + ) else: self.statsd_client.incr(f"clients.{self.name}.error") + self.otel_client.incr( + "clients_error", + attributes={"provider": self.name}, + description="Count of failed requests to provider", + ) self.current_app.logger.warning( "Provider request for %s %s", self.name, "succeeded" if success else "failed" ) @@ -66,7 +78,14 @@ def send_sms(self, to, content, reference, international, sender): raise e finally: elapsed_time = monotonic() - start_time + self.statsd_client.timing(f"clients.{self.name}.request-time", elapsed_time) + self.otel_client.record( + "clients_request_time", + value=elapsed_time, + attributes={"provider": self.name}, + description="Time taken for request to provider", + ) self.current_app.logger.info( "%s request for %s finished in %s", self.name, diff --git a/app/commands.py b/app/commands.py index d68e0eb531..67ad91c614 100644 --- a/app/commands.py +++ b/app/commands.py @@ -16,6 +16,7 @@ from click_datetime import Datetime as click_dt from dateutil import rrule from flask import current_app, json +from notifications_utils.otel_decorators import otel from notifications_utils.recipients import RecipientCSV from notifications_utils.statsd_decorators import statsd from notifications_utils.template import SMSMessageTemplate @@ -397,6 +398,7 @@ def bulk_invite_user_to_service(file_name, service_id, user_id, auth_type, permi "-s", "--start_date", default=datetime(2017, 2, 1), help="start date inclusive", type=click_dt(format="%Y-%m-%d") ) @statsd(namespace="tasks") +@otel(namespace="tasks") def populate_notification_postage(start_date): current_app.logger.info("populating historical notification postage") @@ -442,6 +444,7 @@ def populate_notification_postage(start_date): @click.option("-s", "--start_date", required=True, help="start date inclusive", type=click_dt(format="%Y-%m-%d")) @click.option("-e", "--end_date", required=True, help="end date inclusive", type=click_dt(format="%Y-%m-%d")) @statsd(namespace="tasks") +@otel(namespace="tasks") def update_jobs_archived_flag(start_date, end_date): current_app.logger.info("Archiving jobs created between %s to %s", start_date, end_date) @@ -475,6 +478,7 @@ def update_jobs_archived_flag(start_date, end_date): @notify_command(name="update-emails-to-remove-gsi") @click.option("-s", "--service_id", required=True, help="service id. Update all user.email_address to remove .gsi") @statsd(namespace="tasks") +@otel(namespace="tasks") def update_emails_to_remove_gsi(service_id): users_to_update = """SELECT u.id user_id, u.name, email_address, s.id, s.name FROM users u diff --git a/app/config.py b/app/config.py index c9a35d09ec..5f427447f8 100644 --- a/app/config.py +++ b/app/config.py @@ -100,6 +100,8 @@ class Config: CELERY_WORKER_LOG_LEVEL = os.getenv("CELERY_WORKER_LOG_LEVEL", "CRITICAL").upper() CELERY_BEAT_LOG_LEVEL = os.getenv("CELERY_BEAT_LOG_LEVEL", "INFO").upper() + OTEL_METRICS_EXPORT = os.getenv("OTEL_METRICS_EXPORT", "otlp") + # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API ADMIN_CLIENT_ID = "notify-admin" FUNCTIONAL_TESTS_CLIENT_ID = "notify-functional-tests" @@ -526,6 +528,8 @@ class Development(Config): CELERY_WORKER_LOG_LEVEL = "INFO" + OTEL_METRICS_EXPORT = os.getenv("OTEL_METRICS_EXPORT", "none") + CELERY = { **Config.CELERY, "broker_transport_options": { @@ -586,6 +590,8 @@ class Test(Development): CELERY_WORKER_LOG_LEVEL = "INFO" + OTEL_METRICS_EXPORT = os.getenv("OTEL_METRICS_EXPORT", "none") + S3_BUCKET_CSV_UPLOAD = "test-notifications-csv-upload" S3_BUCKET_CONTACT_LIST = "test-contact-list" S3_BUCKET_TEST_LETTERS = "test-test-letters" diff --git a/app/delivery/send_to_providers.py b/app/delivery/send_to_providers.py index c9e55875fc..0f3bc11bfd 100644 --- a/app/delivery/send_to_providers.py +++ b/app/delivery/send_to_providers.py @@ -11,7 +11,7 @@ SMSMessageTemplate, ) -from app import create_uuid, db, notification_provider_clients, redis_store, statsd_client +from app import create_uuid, db, notification_provider_clients, otel_client, redis_store, statsd_client from app.celery.research_mode_tasks import ( send_email_response, send_sms_response, @@ -98,14 +98,41 @@ def send_sms_to_provider(notification): update_notification_to_sending(notification, provider) if notification.international: statsd_client.incr(f"international-sms.{NOTIFICATION_SENT}.{notification.phone_prefix}") + otel_client.incr( + "international-sms.sent", + attributes={ + "phone_prefix": notification.phone_prefix, + "notification_type": SMS_TYPE, + "status": NOTIFICATION_SENT, + }, + description="Count of international SMS sent", + ) delta_seconds = (datetime.utcnow() - created_at).total_seconds() statsd_client.timing("sms.total-time", delta_seconds) + otel_client.record( + "total-time", + value=delta_seconds, + attributes={"notification_type": SMS_TYPE}, + description="Total time taken to send an SMS", + ) if key_type == KEY_TYPE_TEST: statsd_client.timing("sms.test-key.total-time", delta_seconds) + otel_client.record( + "test-key.total-time", + value=delta_seconds, + attributes={"notification_type": SMS_TYPE}, + description="Total time taken to send a test SMS", + ) else: statsd_client.timing("sms.live-key.total-time", delta_seconds) + otel_client.record( + "live-key.total-time", + value=delta_seconds, + attributes={"notification_type": SMS_TYPE}, + description="Total time taken to send a live SMS", + ) def _get_email_headers(notification: Notification, template: SerialisedTemplate) -> list[dict[str, str]]: @@ -176,8 +203,20 @@ def send_email_to_provider(notification): if key_type == KEY_TYPE_TEST: statsd_client.timing("email.test-key.total-time", delta_seconds) + otel_client.record( + "test-key.total-time", + value=delta_seconds, + attributes={"notification_type": EMAIL_TYPE}, + description="Total time taken to send a test email", + ) else: statsd_client.timing("email.live-key.total-time", delta_seconds) + otel_client.record( + "live-key.total-time", + value=delta_seconds, + attributes={"notification_type": EMAIL_TYPE}, + description="Total time taken to send a live email", + ) def update_notification_to_sending(notification, provider): diff --git a/requirements.in b/requirements.in index 933c4760b4..8a746aec03 100644 --- a/requirements.in +++ b/requirements.in @@ -25,7 +25,7 @@ psutil>=6.0.0,<7.0.0 notifications-python-client==10.0.1 # Run `make bump-utils` to update to the latest version -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@99.8.0 +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@d7b27a02ed3711a3371b8db4c0600b1337822260 # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains prometheus-client==0.14.1 diff --git a/requirements.txt b/requirements.txt index 6bcd9b2312..34e0760c75 100644 --- a/requirements.txt +++ b/requirements.txt @@ -60,6 +60,12 @@ click-plugins==1.1.1 # via celery click-repl==0.2.0 # via celery +deprecated==1.2.18 + # via + # opentelemetry-api + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-semantic-conventions dnspython==2.6.1 # via eventlet docopt==0.6.2 @@ -94,10 +100,16 @@ fqdn==1.5.1 # via jsonschema gds-metrics @ git+https://github.com/alphagov/gds_metrics_python.git@6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72 # via -r requirements.in +googleapis-common-protos==1.70.0 + # via + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http govuk-bank-holidays==0.15 # via notifications-utils greenlet==3.0.3 # via eventlet +grpcio==1.73.0 + # via opentelemetry-exporter-otlp-proto-grpc gunicorn==23.0.0 # via # -r requirements.in @@ -106,6 +118,8 @@ idna==3.7 # via # jsonschema # requests +importlib-metadata==8.6.1 + # via opentelemetry-api iso8601==2.1.0 # via -r requirements.in isoduration==20.11.0 @@ -150,8 +164,50 @@ mistune==0.8.4 # via notifications-utils notifications-python-client==10.0.1 # via -r requirements.in -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@a97b36f6a32e7bb917152c8cd716fe65fa15ac9f +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@d7b27a02ed3711a3371b8db4c0600b1337822260 # via -r requirements.in +opentelemetry-api==1.33.1 + # via + # opentelemetry-distro + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-instrumentation + # opentelemetry-instrumentation-celery + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-distro==0.54b1 + # via notifications-utils +opentelemetry-exporter-otlp==1.33.1 + # via notifications-utils +opentelemetry-exporter-otlp-proto-common==1.33.1 + # via + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-exporter-otlp-proto-grpc==1.33.1 + # via opentelemetry-exporter-otlp +opentelemetry-exporter-otlp-proto-http==1.33.1 + # via opentelemetry-exporter-otlp +opentelemetry-instrumentation==0.54b1 + # via + # opentelemetry-distro + # opentelemetry-instrumentation-celery +opentelemetry-instrumentation-celery==0.54b1 + # via notifications-utils +opentelemetry-proto==1.33.1 + # via + # opentelemetry-exporter-otlp-proto-common + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-sdk==1.33.1 + # via + # opentelemetry-distro + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-semantic-conventions==0.54b1 + # via + # opentelemetry-instrumentation + # opentelemetry-instrumentation-celery + # opentelemetry-sdk ordered-set==4.1.0 # via notifications-utils packaging==23.2 @@ -159,6 +215,7 @@ packaging==23.2 # gunicorn # marshmallow # marshmallow-sqlalchemy + # opentelemetry-instrumentation phonenumbers==8.13.52 # via notifications-utils prometheus-client==0.14.1 @@ -167,6 +224,10 @@ prometheus-client==0.14.1 # gds-metrics prompt-toolkit==3.0.31 # via click-repl +protobuf==5.29.5 + # via + # googleapis-common-protos + # opentelemetry-proto psutil==6.1.1 # via -r requirements.in psycopg2-binary==2.9.10 @@ -204,6 +265,7 @@ requests==2.32.2 # govuk-bank-holidays # notifications-python-client # notifications-utils + # opentelemetry-exporter-otlp-proto-http rfc3339-validator==0.1.4 # via jsonschema rfc3987==1.3.8 @@ -234,6 +296,8 @@ sqlalchemy==1.4.41 # sentry-sdk statsd==4.0.1 # via notifications-utils +typing-extensions==4.14.0 + # via opentelemetry-sdk tzdata==2024.1 # via celery uri-template==1.2.0 @@ -256,3 +320,9 @@ webcolors==1.12 # via jsonschema werkzeug==3.1.3 # via flask +wrapt==1.17.2 + # via + # deprecated + # opentelemetry-instrumentation +zipp==3.23.0 + # via importlib-metadata diff --git a/requirements_for_test.txt b/requirements_for_test.txt index fd5bf5877b..df9712a7cc 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -97,6 +97,13 @@ cryptography==44.0.1 # via # moto # trustme +deprecated==1.2.18 + # via + # -r requirements.txt + # opentelemetry-api + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-semantic-conventions dnspython==2.6.1 # via # -r requirements.txt @@ -143,6 +150,11 @@ freezegun==1.5.1 # via -r requirements_for_test_common.in gds-metrics @ git+https://github.com/alphagov/gds_metrics_python.git@6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72 # via -r requirements.txt +googleapis-common-protos==1.70.0 + # via + # -r requirements.txt + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http govuk-bank-holidays==0.15 # via # -r requirements.txt @@ -151,6 +163,10 @@ greenlet==3.0.3 # via # -r requirements.txt # eventlet +grpcio==1.73.0 + # via + # -r requirements.txt + # opentelemetry-exporter-otlp-proto-grpc gunicorn==23.0.0 # via # -r requirements.txt @@ -160,6 +176,10 @@ idna==3.7 # -r requirements.txt # requests # trustme +importlib-metadata==8.6.1 + # via + # -r requirements.txt + # opentelemetry-api iniconfig==2.0.0 # via pytest iso8601==2.1.0 @@ -220,8 +240,66 @@ moto==5.0.11 # via -r requirements_for_test.in notifications-python-client==10.0.1 # via -r requirements.txt -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@a97b36f6a32e7bb917152c8cd716fe65fa15ac9f +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@d7b27a02ed3711a3371b8db4c0600b1337822260 # via -r requirements.txt +opentelemetry-api==1.33.1 + # via + # -r requirements.txt + # opentelemetry-distro + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http + # opentelemetry-instrumentation + # opentelemetry-instrumentation-celery + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-distro==0.54b1 + # via + # -r requirements.txt + # notifications-utils +opentelemetry-exporter-otlp==1.33.1 + # via + # -r requirements.txt + # notifications-utils +opentelemetry-exporter-otlp-proto-common==1.33.1 + # via + # -r requirements.txt + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-exporter-otlp-proto-grpc==1.33.1 + # via + # -r requirements.txt + # opentelemetry-exporter-otlp +opentelemetry-exporter-otlp-proto-http==1.33.1 + # via + # -r requirements.txt + # opentelemetry-exporter-otlp +opentelemetry-instrumentation==0.54b1 + # via + # -r requirements.txt + # opentelemetry-distro + # opentelemetry-instrumentation-celery +opentelemetry-instrumentation-celery==0.54b1 + # via + # -r requirements.txt + # notifications-utils +opentelemetry-proto==1.33.1 + # via + # -r requirements.txt + # opentelemetry-exporter-otlp-proto-common + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-sdk==1.33.1 + # via + # -r requirements.txt + # opentelemetry-distro + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-exporter-otlp-proto-http +opentelemetry-semantic-conventions==0.54b1 + # via + # -r requirements.txt + # opentelemetry-instrumentation + # opentelemetry-instrumentation-celery + # opentelemetry-sdk ordered-set==4.1.0 # via # -r requirements.txt @@ -232,6 +310,7 @@ packaging==23.2 # gunicorn # marshmallow # marshmallow-sqlalchemy + # opentelemetry-instrumentation # pytest phonenumbers==8.13.52 # via @@ -247,6 +326,11 @@ prompt-toolkit==3.0.31 # via # -r requirements.txt # click-repl +protobuf==5.29.5 + # via + # -r requirements.txt + # googleapis-common-protos + # opentelemetry-proto psutil==6.1.1 # via -r requirements.txt psycopg2-binary==2.9.10 @@ -317,6 +401,7 @@ requests==2.32.2 # moto # notifications-python-client # notifications-utils + # opentelemetry-exporter-otlp-proto-http # requests-mock # responses requests-mock==1.12.1 @@ -370,6 +455,10 @@ statsd==4.0.1 # notifications-utils trustme==0.9.0 # via -r requirements_for_test.in +typing-extensions==4.14.0 + # via + # -r requirements.txt + # opentelemetry-sdk tzdata==2024.1 # via # -r requirements.txt @@ -401,5 +490,14 @@ werkzeug==3.1.3 # flask # moto # pytest-httpserver +wrapt==1.17.2 + # via + # -r requirements.txt + # deprecated + # opentelemetry-instrumentation xmltodict==0.14.2 # via moto +zipp==3.23.0 + # via + # -r requirements.txt + # importlib-metadata diff --git a/requirements_for_test_common.in b/requirements_for_test_common.in index 8cf8bba786..b578361f62 100644 --- a/requirements_for_test_common.in +++ b/requirements_for_test_common.in @@ -1,4 +1,4 @@ -# This file was automatically copied from notifications-utils@99.8.0 +# This file was automatically copied from notifications-utils@100.1.0 beautifulsoup4==4.12.3 pytest==8.3.4 diff --git a/ruff.toml b/ruff.toml index 2ba4008f6b..6e1fb3adf5 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,4 +1,4 @@ -# This file was automatically copied from notifications-utils@99.8.0 +# This file was automatically copied from notifications-utils@100.1.0 extend-exclude = [ "migrations/versions/", diff --git a/tests/app/celery/test_process_ses_receipts_tasks.py b/tests/app/celery/test_process_ses_receipts_tasks.py index a5e326b19a..5b86354806 100644 --- a/tests/app/celery/test_process_ses_receipts_tasks.py +++ b/tests/app/celery/test_process_ses_receipts_tasks.py @@ -3,7 +3,7 @@ from freezegun import freeze_time -from app import signing, statsd_client +from app import otel_client, signing, statsd_client from app.celery.process_ses_receipts_tasks import process_ses_results from app.celery.research_mode_tasks import ( ses_hard_bounce_callback, @@ -65,7 +65,9 @@ def test_remove_email_from_bounce(): def test_ses_callback_should_update_notification_status(client, notify_db_session, sample_email_template, mocker): with freeze_time("2001-01-01T12:00:00"): mocker.patch("app.statsd_client.incr") + mocker.patch("app.otel_client.incr") mocker.patch("app.statsd_client.timing_with_dates") + mocker.patch("app.otel_client.record") send_mock = mocker.patch("app.celery.process_ses_receipts_tasks.check_and_queue_callback_task") notification = create_notification( template=sample_email_template, @@ -79,7 +81,19 @@ def test_ses_callback_should_update_notification_status(client, notify_db_sessio statsd_client.timing_with_dates.assert_any_call( "callback.ses.delivered.elapsed-time", datetime.utcnow(), notification.sent_at ) + otel_client.record.assert_any_call( + "callback_elapsed_time", + value=(datetime.utcnow() - notification.sent_at).total_seconds(), + attributes={"provider": "ses", "status": "delivered"}, + description="Elapsed time for SES callback with status", + ) statsd_client.incr.assert_any_call("callback.ses.delivered") + otel_client.incr.assert_any_call( + "callback_success", + attributes={"provider": "ses", "status": "delivered"}, + description="Count of successful SES callbacks with status", + ) + updated_notification = Notification.query.get(notification.id) send_mock.assert_called_once_with(updated_notification) diff --git a/tests/app/celery/test_process_sms_client_response_tasks.py b/tests/app/celery/test_process_sms_client_response_tasks.py index cb7a181d56..520643580b 100644 --- a/tests/app/celery/test_process_sms_client_response_tasks.py +++ b/tests/app/celery/test_process_sms_client_response_tasks.py @@ -4,7 +4,7 @@ import pytest from freezegun import freeze_time -from app import statsd_client +from app import otel_client, statsd_client from app.celery.process_sms_client_response_tasks import ( process_sms_client_response, ) @@ -118,7 +118,9 @@ def test_sms_response_does_not_send_callback_if_notification_is_not_in_the_db(sa @freeze_time("2001-01-01T12:00:00") def test_process_sms_client_response_records_statsd_metrics(sample_notification, client, mocker): mocker.patch("app.statsd_client.incr") + mocker.patch("app.otel_client.incr") mocker.patch("app.statsd_client.timing_with_dates") + mocker.patch("app.otel_client.record") sample_notification.status = "sending" sample_notification.sent_at = datetime.utcnow() @@ -126,9 +128,20 @@ def test_process_sms_client_response_records_statsd_metrics(sample_notification, process_sms_client_response("0", str(sample_notification.id), "Firetext") statsd_client.incr.assert_any_call("callback.firetext.delivered") + otel_client.incr.assert_any_call( + "callback", + attributes={"provider": "firetext", "status": "delivered"}, + description="Count of callbacks", + ) statsd_client.timing_with_dates.assert_any_call( "callback.firetext.delivered.elapsed-time", datetime.utcnow(), sample_notification.sent_at ) + otel_client.record.assert_any_call( + "callback_elapsed_time", + value=(datetime.utcnow() - sample_notification.sent_at).total_seconds(), + attributes={"provider": "firetext", "status": "delivered"}, + description="Elapsed time for callbacks", + ) def test_process_sms_updates_billable_units_if_zero(sample_notification): diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index adab86db52..0e85615cc4 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -185,6 +185,7 @@ def test_generate_sms_delivery_stats(slow_delivery_config_option, expect_check_s return_value=slow_delivery_reports, ) mock_statsd = mocker.patch("app.celery.scheduled_tasks.statsd_client.gauge") + mock_otel = mocker.patch("app.celery.scheduled_tasks.otel_client.record") mock_check_slow_delivery = mocker.patch( "app.celery.scheduled_tasks._check_slow_text_message_delivery_reports_and_raise_error_if_needed" ) @@ -204,6 +205,68 @@ def test_generate_sms_delivery_stats(slow_delivery_config_option, expect_check_s call("slow-delivery.sms.delivered-within-minutes.10.ratio", 0.6), ] mock_statsd.assert_has_calls(calls, any_order=True) + otel_description = ( + "Ratio of slow message deliveries for in the last 15 minutes for deliveries " + "taking longer than delivery_interval minutes" + ) + + otel_calls = [ + call( + "slow_delivery_ratio", + value=0.4, + attributes={"provider": "mmg", "delivery_interval": 1}, + description=otel_description, + ), + call( + "slow_delivery_ratio", + value=0.8, + attributes={"provider": "firetext", "delivery_interval": 1}, + description=otel_description, + ), + call( + "slow_sms_delivery_ratio", + value=0.6, + attributes={"delivery_interval": 1}, + description=otel_description, + ), + call( + "slow_delivery_ratio", + value=0.4, + attributes={"provider": "mmg", "delivery_interval": 5}, + description=otel_description, + ), + call( + "slow_delivery_ratio", + value=0.8, + attributes={"provider": "firetext", "delivery_interval": 5}, + description=otel_description, + ), + call( + "slow_sms_delivery_ratio", + value=0.6, + attributes={"delivery_interval": 5}, + description=otel_description, + ), + call( + "slow_delivery_ratio", + value=0.4, + attributes={"provider": "mmg", "delivery_interval": 10}, + description=otel_description, + ), + call( + "slow_delivery_ratio", + value=0.8, + attributes={"provider": "firetext", "delivery_interval": 10}, + description=otel_description, + ), + call( + "slow_sms_delivery_ratio", + value=0.6, + attributes={"delivery_interval": 10}, + description=otel_description, + ), + ] + mock_otel.assert_has_calls(otel_calls, any_order=True) assert mock_check_slow_delivery.call_args_list == ( [mocker.call(slow_delivery_reports)] if expect_check_slow_delivery else [] diff --git a/tests/app/clients/test_aws_ses.py b/tests/app/clients/test_aws_ses.py index 1d0b8861cf..db1d83e30c 100644 --- a/tests/app/clients/test_aws_ses.py +++ b/tests/app/clients/test_aws_ses.py @@ -61,6 +61,7 @@ def test_should_be_none_if_unrecognised_status_code(): def test_send_email_handles_reply_to_address(notify_api, mocker, reply_to_address, expected_value): boto_mock = mocker.patch.object(aws_ses_client, "_client", create=True) mocker.patch.object(aws_ses_client, "statsd_client", create=True) + mocker.patch.object(aws_ses_client, "otel_client", create=True) with notify_api.app_context(): aws_ses_client.send_email( @@ -81,6 +82,7 @@ def test_send_email_handles_reply_to_address(notify_api, mocker, reply_to_addres def test_send_email_handles_punycode_to_address(notify_api, mocker): boto_mock = mocker.patch.object(aws_ses_client, "_client", create=True) mocker.patch.object(aws_ses_client, "statsd_client", create=True) + mocker.patch.object(aws_ses_client, "otel_client", create=True) with notify_api.app_context(): aws_ses_client.send_email( @@ -104,6 +106,7 @@ def test_send_email_handles_punycode_to_address(notify_api, mocker): def test_send_email_sends_content_correctly(notify_api, mocker): boto_mock = mocker.patch.object(aws_ses_client, "_client", create=True) mocker.patch.object(aws_ses_client, "statsd_client", create=True) + mocker.patch.object(aws_ses_client, "otel_client", create=True) mock_subject = Mock() mock_body = Mock() @@ -137,11 +140,11 @@ def test_send_email_sends_content_correctly(notify_api, mocker): def test_send_email_raises_invalid_parameter_value_error_as_EmailClientNonRetryableException(mocker): boto_mock = mocker.patch.object(aws_ses_client, "_client", create=True) mocker.patch.object(aws_ses_client, "statsd_client", create=True) + mocker.patch.object(aws_ses_client, "otel_client", create=True) error_response = { "Error": {"Code": "InvalidParameterValue", "Message": "some error message from amazon", "Type": "Sender"} } boto_mock.send_email.side_effect = botocore.exceptions.ClientError(error_response, "opname") - mocker.patch.object(aws_ses_client, "statsd_client", create=True) with pytest.raises(EmailClientNonRetryableException) as excinfo: aws_ses_client.send_email( @@ -160,6 +163,7 @@ def test_send_email_raises_invalid_parameter_value_error_as_EmailClientNonRetrya def test_send_email_raises_send_rate_throttling_as_AwsSesClientThrottlingSendRateException(mocker): boto_mock = mocker.patch.object(aws_ses_client, "_client", create=True) mocker.patch.object(aws_ses_client, "statsd_client", create=True) + mocker.patch.object(aws_ses_client, "otel_client", create=True) error_response = { "Error": {"Code": "TooManyRequestsException", "Message": "Maximum sending rate exceeded.", "Type": "Sender"} } @@ -180,6 +184,7 @@ def test_send_email_raises_send_rate_throttling_as_AwsSesClientThrottlingSendRat def test_send_email_does_not_raise_AwsSesClientThrottlingSendRateException_if_non_send_rate_throttling(mocker): boto_mock = mocker.patch.object(aws_ses_client, "_client", create=True) mocker.patch.object(aws_ses_client, "statsd_client", create=True) + mocker.patch.object(aws_ses_client, "otel_client", create=True) error_response = { "Error": {"Code": "TooManyRequestsException", "Message": "Daily message quota exceeded", "Type": "Sender"} } @@ -200,11 +205,11 @@ def test_send_email_does_not_raise_AwsSesClientThrottlingSendRateException_if_no def test_send_email_raises_other_errs_as_AwsSesClientException(mocker): boto_mock = mocker.patch.object(aws_ses_client, "_client", create=True) mocker.patch.object(aws_ses_client, "statsd_client", create=True) + mocker.patch.object(aws_ses_client, "otel_client", create=True) error_response = { "Error": {"Code": "ServiceUnavailable", "Message": "some error message from amazon", "Type": "Sender"} } boto_mock.send_email.side_effect = botocore.exceptions.ClientError(error_response, "opname") - mocker.patch.object(aws_ses_client, "statsd_client", create=True) with pytest.raises(AwsSesClientException) as excinfo: aws_ses_client.send_email( diff --git a/tests/app/clients/test_dvla.py b/tests/app/clients/test_dvla.py index c72a3e41cf..d8d0ccff80 100644 --- a/tests/app/clients/test_dvla.py +++ b/tests/app/clients/test_dvla.py @@ -55,7 +55,7 @@ def ssm(): @pytest.fixture def dvla_client(notify_api, client, ssm): - dvla_client = DVLAClient(notify_api, statsd_client=Mock()) + dvla_client = DVLAClient(notify_api, statsd_client=Mock(), otel_client=Mock()) yield dvla_client @@ -945,6 +945,7 @@ def test_valid_default_connection( dvla_client = DVLAClient( fake_app, statsd_client=mocker.Mock(), + otel_client=mocker.Mock(), ) with ca.cert_pem.tempfile() as ca_temp_path: @@ -958,7 +959,7 @@ def test_valid_default_connection( def test_invalid_ciphers(self, mocker, server_base_url, fake_app): with pytest.raises(ssl.SSLError) as e: fake_app.config["DVLA_API_TLS_CIPHERS"] = "not-a-valid-cipher" - DVLAClient(fake_app, statsd_client=mocker.Mock()) + DVLAClient(fake_app, statsd_client=mocker.Mock(), otel_client=mocker.Mock()) assert "No cipher can be selected." in e.value.args @@ -977,6 +978,7 @@ def test_accept_matching_cipher( dvla_client = DVLAClient( fake_app, statsd_client=mocker.Mock(), + otel_client=mocker.Mock(), ) httpserver.expect_request( @@ -1007,6 +1009,7 @@ def test_reject_cipher_not_accepted_by_server( dvla_client = DVLAClient( fake_app, statsd_client=mocker.Mock(), + otel_client=mocker.Mock(), ) httpserver.expect_request( diff --git a/tests/app/clients/test_sms.py b/tests/app/clients/test_sms.py index 21140221e8..b4bea3b984 100644 --- a/tests/app/clients/test_sms.py +++ b/tests/app/clients/test_sms.py @@ -1,6 +1,6 @@ import pytest -from app import statsd_client +from app import otel_client, statsd_client from app.clients.sms import SmsClient, SmsClientResponseException @@ -12,7 +12,7 @@ class FakeSmsClient(SmsClient): def try_send_sms(self): pass - fake_client = FakeSmsClient(notify_api, statsd_client) + fake_client = FakeSmsClient(notify_api, statsd_client, otel_client) return fake_client diff --git a/tests/app/conftest.py b/tests/app/conftest.py index ab1341f530..816a7ad2c3 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -561,7 +561,8 @@ def create_mock_firetext_config(mocker, additional_config=None): def create_mock_firetext_client(mocker, mock_config): statsd_client = mocker.Mock() - return FiretextClient(mock_config, statsd_client) + otel_client = mocker.Mock() + return FiretextClient(mock_config, statsd_client, otel_client) @pytest.fixture(scope="function") @@ -580,6 +581,7 @@ def mock_firetext_client_with_receipts(mocker): @pytest.fixture(scope="function") def mock_mmg_client_with_receipts(mocker): statsd_client = mocker.Mock() + otel_client = mocker.Mock() current_app = mocker.Mock( config={ "MMG_URL": "https://example.com/mmg", @@ -587,7 +589,7 @@ def mock_mmg_client_with_receipts(mocker): "MMG_RECEIPT_URL": "https://www.example.com/notifications/sms/mmg", } ) - return MMGClient(current_app, statsd_client) + return MMGClient(current_app, statsd_client, otel_client) @pytest.fixture(scope="function") diff --git a/tests/app/v2/test_errors.py b/tests/app/v2/test_errors.py index 6ad3f5761e..774daeabd8 100644 --- a/tests/app/v2/test_errors.py +++ b/tests/app/v2/test_errors.py @@ -18,9 +18,10 @@ def app_for_test(): app = flask.Flask(__name__) app.config["TESTING"] = True init_app(app) - from app import statsd_client + from app import otel_client, statsd_client statsd_client.init_app(app) + otel_client.init_app(app) from app.v2.errors import register_errors