From f4b0b041569881c68fc611364f871a40654dc381 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Sun, 16 Feb 2025 13:36:21 +0000 Subject: [PATCH 01/10] TMP utils bump --- gunicorn_config.py | 2 +- requirements.in | 2 +- requirements.txt | 6 ++++-- requirements_for_test.txt | 3 ++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/gunicorn_config.py b/gunicorn_config.py index d22c829dac..7c2c46da3c 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -1,6 +1,6 @@ import os -from notifications_utils.gunicorn_defaults import set_gunicorn_defaults +from notifications_utils.gunicorn.defaults import set_gunicorn_defaults set_gunicorn_defaults(globals()) diff --git a/requirements.in b/requirements.in index 16c91b8743..83ed343f99 100644 --- a/requirements.in +++ b/requirements.in @@ -25,7 +25,7 @@ psutil>=6.0.0,<7.0.0 notifications-python-client==10.0.1 # Run `make bump-utils` to update to the latest version -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@94.0.1 +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@fe153ee07478882d8d70183f26ac305822f722e3 # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains prometheus-client==0.14.1 diff --git a/requirements.txt b/requirements.txt index 1227758274..ac4874f3ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -95,7 +95,9 @@ gds-metrics @ git+https://github.com/alphagov/gds_metrics_python.git@6f1840a57b6 govuk-bank-holidays==0.15 # via notifications-utils greenlet==3.0.3 - # via eventlet + # via + # eventlet + # sqlalchemy gunicorn @ git+https://github.com/benoitc/gunicorn.git@1299ea9e967a61ae2edebe191082fd169b864c64 # via # -r requirements.in @@ -147,7 +149,7 @@ mistune==0.8.4 # via notifications-utils notifications-python-client==10.0.1 # via -r requirements.in -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@d6311c3ab83b8225265277f781157c48f9318ae9 +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@fe153ee07478882d8d70183f26ac305822f722e3 # via -r requirements.in ordered-set==4.1.0 # via notifications-utils diff --git a/requirements_for_test.txt b/requirements_for_test.txt index 6eed37af49..b097d26a28 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -151,6 +151,7 @@ greenlet==3.0.3 # via # -r requirements.txt # eventlet + # sqlalchemy gunicorn @ git+https://github.com/benoitc/gunicorn.git@1299ea9e967a61ae2edebe191082fd169b864c64 # via # -r requirements.txt @@ -219,7 +220,7 @@ moto==5.0.11 # via -r requirements_for_test.in notifications-python-client==10.0.1 # via -r requirements.txt -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@d6311c3ab83b8225265277f781157c48f9318ae9 +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@fe153ee07478882d8d70183f26ac305822f722e3 # via -r requirements.txt ordered-set==4.1.0 # via From 31ac62c5768da4b5fbc3f32ce4c4e747637973e5 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Thu, 6 Mar 2025 13:38:17 +0000 Subject: [PATCH 02/10] TMP use NotifyEventletWorker --- gunicorn_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn_config.py b/gunicorn_config.py index 7c2c46da3c..0137dd2a2e 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -16,7 +16,7 @@ def child_exit(server, worker): workers = 4 -worker_class = "eventlet" +worker_class = "notifications_utils.gunicorn.eventlet.NotifyEventletWorker" worker_connections = 8 # limit runaway greenthread creation statsd_host = "{}:8125".format(os.getenv("STATSD_HOST")) keepalive = 0 # disable temporarily for diagnosing issues From b65e0fc6d396cefe5d8cfb1ddf9c2d92b22797aa Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Thu, 6 Mar 2025 16:58:02 +0000 Subject: [PATCH 03/10] TMP bump utils again --- requirements.in | 2 +- requirements.txt | 2 +- requirements_for_test.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.in b/requirements.in index 83ed343f99..2022b4b7d2 100644 --- a/requirements.in +++ b/requirements.in @@ -25,7 +25,7 @@ psutil>=6.0.0,<7.0.0 notifications-python-client==10.0.1 # Run `make bump-utils` to update to the latest version -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@fe153ee07478882d8d70183f26ac305822f722e3 +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@05a283ed7d2a6f33ab234e9f807233a5bf5c603c # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains prometheus-client==0.14.1 diff --git a/requirements.txt b/requirements.txt index ac4874f3ff..0795b70e3e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -149,7 +149,7 @@ mistune==0.8.4 # via notifications-utils notifications-python-client==10.0.1 # via -r requirements.in -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@fe153ee07478882d8d70183f26ac305822f722e3 +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@05a283ed7d2a6f33ab234e9f807233a5bf5c603c # via -r requirements.in ordered-set==4.1.0 # via notifications-utils diff --git a/requirements_for_test.txt b/requirements_for_test.txt index b097d26a28..6d576ea9cd 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -220,7 +220,7 @@ moto==5.0.11 # via -r requirements_for_test.in notifications-python-client==10.0.1 # via -r requirements.txt -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@fe153ee07478882d8d70183f26ac305822f722e3 +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@05a283ed7d2a6f33ab234e9f807233a5bf5c603c # via -r requirements.txt ordered-set==4.1.0 # via From 2079c758e531d02ed338a5c740da27ddd65d5942 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Thu, 6 Mar 2025 17:25:02 +0000 Subject: [PATCH 04/10] TMP bump utils again again --- requirements.in | 2 +- requirements.txt | 2 +- requirements_for_test.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.in b/requirements.in index 2022b4b7d2..a4e8b2cdbb 100644 --- a/requirements.in +++ b/requirements.in @@ -25,7 +25,7 @@ psutil>=6.0.0,<7.0.0 notifications-python-client==10.0.1 # Run `make bump-utils` to update to the latest version -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@05a283ed7d2a6f33ab234e9f807233a5bf5c603c +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@4ac08167d81468af12bff1aa012c9a0a695bbda3 # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains prometheus-client==0.14.1 diff --git a/requirements.txt b/requirements.txt index 0795b70e3e..ddc429f59a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -149,7 +149,7 @@ mistune==0.8.4 # via notifications-utils notifications-python-client==10.0.1 # via -r requirements.in -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@05a283ed7d2a6f33ab234e9f807233a5bf5c603c +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@4ac08167d81468af12bff1aa012c9a0a695bbda3 # via -r requirements.in ordered-set==4.1.0 # via notifications-utils diff --git a/requirements_for_test.txt b/requirements_for_test.txt index 6d576ea9cd..f4851fa745 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -220,7 +220,7 @@ moto==5.0.11 # via -r requirements_for_test.in notifications-python-client==10.0.1 # via -r requirements.txt -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@05a283ed7d2a6f33ab234e9f807233a5bf5c603c +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@4ac08167d81468af12bff1aa012c9a0a695bbda3 # via -r requirements.txt ordered-set==4.1.0 # via From 4ecd1d19a95c752f16963ea4e75310286ff2e2c3 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Thu, 6 Mar 2025 17:44:04 +0000 Subject: [PATCH 05/10] TMP ever bumping utils --- requirements.in | 2 +- requirements.txt | 2 +- requirements_for_test.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.in b/requirements.in index a4e8b2cdbb..b15691347b 100644 --- a/requirements.in +++ b/requirements.in @@ -25,7 +25,7 @@ psutil>=6.0.0,<7.0.0 notifications-python-client==10.0.1 # Run `make bump-utils` to update to the latest version -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@4ac08167d81468af12bff1aa012c9a0a695bbda3 +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@2706b603dea5cb29913e63c098d19502f4b91987 # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains prometheus-client==0.14.1 diff --git a/requirements.txt b/requirements.txt index ddc429f59a..b7473fcad7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -149,7 +149,7 @@ mistune==0.8.4 # via notifications-utils notifications-python-client==10.0.1 # via -r requirements.in -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@4ac08167d81468af12bff1aa012c9a0a695bbda3 +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@2706b603dea5cb29913e63c098d19502f4b91987 # via -r requirements.in ordered-set==4.1.0 # via notifications-utils diff --git a/requirements_for_test.txt b/requirements_for_test.txt index f4851fa745..b973cf9119 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -220,7 +220,7 @@ moto==5.0.11 # via -r requirements_for_test.in notifications-python-client==10.0.1 # via -r requirements.txt -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@4ac08167d81468af12bff1aa012c9a0a695bbda3 +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@2706b603dea5cb29913e63c098d19502f4b91987 # via -r requirements.txt ordered-set==4.1.0 # via From ddb042ea5e87e028d826a30942fcfa6de3c97e82 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Thu, 6 Mar 2025 18:10:24 +0000 Subject: [PATCH 06/10] TMP all the bumping --- requirements.in | 2 +- requirements.txt | 2 +- requirements_for_test.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.in b/requirements.in index b15691347b..d38072349c 100644 --- a/requirements.in +++ b/requirements.in @@ -25,7 +25,7 @@ psutil>=6.0.0,<7.0.0 notifications-python-client==10.0.1 # Run `make bump-utils` to update to the latest version -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@2706b603dea5cb29913e63c098d19502f4b91987 +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@3f31ae8ecf80936244e6f419f2247c9126e5b48d # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains prometheus-client==0.14.1 diff --git a/requirements.txt b/requirements.txt index b7473fcad7..a513306273 100644 --- a/requirements.txt +++ b/requirements.txt @@ -149,7 +149,7 @@ mistune==0.8.4 # via notifications-utils notifications-python-client==10.0.1 # via -r requirements.in -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@2706b603dea5cb29913e63c098d19502f4b91987 +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@3f31ae8ecf80936244e6f419f2247c9126e5b48d # via -r requirements.in ordered-set==4.1.0 # via notifications-utils diff --git a/requirements_for_test.txt b/requirements_for_test.txt index b973cf9119..8c04f2eca1 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -220,7 +220,7 @@ moto==5.0.11 # via -r requirements_for_test.in notifications-python-client==10.0.1 # via -r requirements.txt -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@2706b603dea5cb29913e63c098d19502f4b91987 +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@3f31ae8ecf80936244e6f419f2247c9126e5b48d # via -r requirements.txt ordered-set==4.1.0 # via From 4ae38ef09dc9d1c37899de000ea22f51a87c3a5e Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Fri, 7 Mar 2025 10:35:05 +0000 Subject: [PATCH 07/10] TMP --- requirements.in | 2 +- requirements.txt | 2 +- requirements_for_test.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.in b/requirements.in index d38072349c..a763187b8c 100644 --- a/requirements.in +++ b/requirements.in @@ -25,7 +25,7 @@ psutil>=6.0.0,<7.0.0 notifications-python-client==10.0.1 # Run `make bump-utils` to update to the latest version -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@3f31ae8ecf80936244e6f419f2247c9126e5b48d +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@a59d5ddc6c0986181f15bce0817b495c06fd987b # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains prometheus-client==0.14.1 diff --git a/requirements.txt b/requirements.txt index a513306273..3fe3020ef7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -149,7 +149,7 @@ mistune==0.8.4 # via notifications-utils notifications-python-client==10.0.1 # via -r requirements.in -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@3f31ae8ecf80936244e6f419f2247c9126e5b48d +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@a59d5ddc6c0986181f15bce0817b495c06fd987b # via -r requirements.in ordered-set==4.1.0 # via notifications-utils diff --git a/requirements_for_test.txt b/requirements_for_test.txt index 8c04f2eca1..37c9e3ceac 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -220,7 +220,7 @@ moto==5.0.11 # via -r requirements_for_test.in notifications-python-client==10.0.1 # via -r requirements.txt -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@3f31ae8ecf80936244e6f419f2247c9126e5b48d +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@a59d5ddc6c0986181f15bce0817b495c06fd987b # via -r requirements.txt ordered-set==4.1.0 # via From 6c8c09ec46af963b77b482537f256778d410c862 Mon Sep 17 00:00:00 2001 From: Robert Scott Date: Fri, 7 Mar 2025 11:40:53 +0000 Subject: [PATCH 08/10] TMP --- requirements.in | 2 +- requirements.txt | 2 +- requirements_for_test.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.in b/requirements.in index a763187b8c..9d54a9be58 100644 --- a/requirements.in +++ b/requirements.in @@ -25,7 +25,7 @@ psutil>=6.0.0,<7.0.0 notifications-python-client==10.0.1 # Run `make bump-utils` to update to the latest version -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@a59d5ddc6c0986181f15bce0817b495c06fd987b +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@2d56aef11c9d34acf15176a1b2b20cc874b4deec # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains prometheus-client==0.14.1 diff --git a/requirements.txt b/requirements.txt index 3fe3020ef7..b146015d18 100644 --- a/requirements.txt +++ b/requirements.txt @@ -149,7 +149,7 @@ mistune==0.8.4 # via notifications-utils notifications-python-client==10.0.1 # via -r requirements.in -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@a59d5ddc6c0986181f15bce0817b495c06fd987b +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@2d56aef11c9d34acf15176a1b2b20cc874b4deec # via -r requirements.in ordered-set==4.1.0 # via notifications-utils diff --git a/requirements_for_test.txt b/requirements_for_test.txt index 37c9e3ceac..6c4742eb05 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -220,7 +220,7 @@ moto==5.0.11 # via -r requirements_for_test.in notifications-python-client==10.0.1 # via -r requirements.txt -notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@a59d5ddc6c0986181f15bce0817b495c06fd987b +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@2d56aef11c9d34acf15176a1b2b20cc874b4deec # via -r requirements.txt ordered-set==4.1.0 # via From a41878fef18a3c5dbbd800bfeada6147f7eda9b4 Mon Sep 17 00:00:00 2001 From: Ian Nice Date: Thu, 13 Mar 2025 08:30:30 +0000 Subject: [PATCH 09/10] re-enable keepalives for testing --- gunicorn_config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gunicorn_config.py b/gunicorn_config.py index 0137dd2a2e..4554b570bd 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -17,9 +17,11 @@ def child_exit(server, worker): workers = 4 worker_class = "notifications_utils.gunicorn.eventlet.NotifyEventletWorker" -worker_connections = 8 # limit runaway greenthread creation +#worker_connections = 8 # limit runaway greenthread creation +worker_connections = 256 # re-enabling the problem statsd_host = "{}:8125".format(os.getenv("STATSD_HOST")) -keepalive = 0 # disable temporarily for diagnosing issues +keepalive = 90 # re-enabling the problem +#keepalive = 0 # disable temporarily for diagnosing issues timeout = int(os.getenv("HTTP_SERVE_TIMEOUT_SECONDS", 30)) # though has little effect with eventlet worker_class debug_post_threshold = os.getenv("NOTIFY_GUNICORN_DEBUG_POST_REQUEST_LOG_THRESHOLD_SECONDS", None) From 35a065e29e5be9691c6e5318b55e65541d8f36fa Mon Sep 17 00:00:00 2001 From: Ian Nice Date: Fri, 14 Mar 2025 10:20:33 +0000 Subject: [PATCH 10/10] Changed keep alive to 35 seconds as per Rob's request --- gunicorn_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gunicorn_config.py b/gunicorn_config.py index 4554b570bd..2818b62c1d 100644 --- a/gunicorn_config.py +++ b/gunicorn_config.py @@ -20,7 +20,7 @@ def child_exit(server, worker): #worker_connections = 8 # limit runaway greenthread creation worker_connections = 256 # re-enabling the problem statsd_host = "{}:8125".format(os.getenv("STATSD_HOST")) -keepalive = 90 # re-enabling the problem +keepalive = 35 # re-enabling the problem #keepalive = 0 # disable temporarily for diagnosing issues timeout = int(os.getenv("HTTP_SERVE_TIMEOUT_SECONDS", 30)) # though has little effect with eventlet worker_class