From 0245e94b7a41fea56ad03505d3ed1b3eaa48efe9 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Tue, 1 Oct 2024 10:51:30 +0100 Subject: [PATCH 1/2] migration to delete broadcast data this is a data migration (so no downgrade) that deletes broadcast services and all associated data - this includes obvious things like broadcast events and templates, but also everything connected to those services like user permissions, inbound senders, ft_billing rows, etc Although some of these theoretically shouldn't exist for broadcast services (eg returned_letters), issue the deletes out of an abundance of caution. These migrations need to pass on preview and staging as well as production, and for example there was a broadcast service on staging with an inbound sms number This has been tested against a copy of the staging database. --- migrations/.current-alembic-head | 2 +- .../versions/0465_delete_broadcast_data.py | 100 ++++++++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 migrations/versions/0465_delete_broadcast_data.py diff --git a/migrations/.current-alembic-head b/migrations/.current-alembic-head index 6dc3328442..521b5c414f 100644 --- a/migrations/.current-alembic-head +++ b/migrations/.current-alembic-head @@ -1 +1 @@ -0464_create_svc_join_requests +0465_delete_broadcast_data diff --git a/migrations/versions/0465_delete_broadcast_data.py b/migrations/versions/0465_delete_broadcast_data.py new file mode 100644 index 0000000000..54e680d2ad --- /dev/null +++ b/migrations/versions/0465_delete_broadcast_data.py @@ -0,0 +1,100 @@ +from alembic import op +from sqlalchemy.sql import text +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision = "0465_delete_broadcast_data" +down_revision = "0464_create_svc_join_requests" + + +def upgrade(): + """ + Removes all existance of broadcast specific services and all their associated data: + + deletes ALL data from the following tables: + + * broadcast_event + * broadcast_message + * broadcast_provider_message + * broadcast_provider_message_number + * service_broadcast_provider_restriction + * service_broadcast_settings + + deletes all data linked to a broadcast service from any table with a `service_id` (or a `template_id` for templates + belonging to a broadcast service). + + this does not touch the following static type tables + + * broadcast_channel_types + * broadcast_provider_message_status_type + * broadcast_provider_types + * broadcast_status_type + + this does not touch the following sequence + + * broadcast_provider_message_number_seq + """ + + conn = op.get_bind() + results = conn.execute("SELECT service_id FROM service_broadcast_settings;") + res = results.fetchall() + broadcast_service_ids = tuple(x.service_id for x in res) + + # these are broadly alphabetical, but with some lines reordered due to precedence (eg needing to delete + # user_folder_permissions before we can delete the template folders they reference) + delete_statements = [ + # step 1: delete the entire tables for the broadcast specific things + "DELETE FROM broadcast_provider_message_number;", + "DELETE FROM broadcast_provider_message;", + "DELETE FROM broadcast_event;", + "DELETE FROM broadcast_message;", + "DELETE FROM service_broadcast_settings;", + "DELETE FROM service_broadcast_provider_restriction;", + # all these tables have service_id, so need to be deleted before we can delete the service, + # (note this includes templates as well, which we need to delete), + "DELETE FROM annual_billing WHERE service_id in :service_ids;", + "DELETE FROM api_keys WHERE service_id in :service_ids;", + "DELETE FROM complaints WHERE service_id in :service_ids;", + "DELETE FROM service_sms_senders WHERE service_id in :service_ids;", + "DELETE FROM inbound_numbers WHERE service_id in :service_ids;", + "DELETE FROM inbound_sms WHERE service_id in :service_ids;", + "DELETE FROM inbound_sms_history WHERE service_id in :service_ids;", + "DELETE FROM invited_users WHERE service_id in :service_ids;", + "DELETE FROM jobs WHERE service_id in :service_ids;", + "DELETE FROM notification_history WHERE service_id in :service_ids;", + "DELETE FROM notifications WHERE service_id in :service_ids;", + "DELETE FROM permissions WHERE service_id in :service_ids;", + "DELETE FROM returned_letters WHERE service_id in :service_ids;", + "DELETE FROM service_callback_api WHERE service_id in :service_ids;", + "DELETE FROM service_contact_list WHERE service_id in :service_ids;", + "DELETE FROM service_data_retention WHERE service_id in :service_ids;", + "DELETE FROM service_email_branding WHERE service_id in :service_ids;", + "DELETE FROM service_email_reply_to WHERE service_id in :service_ids;", + "DELETE FROM service_inbound_api WHERE service_id in :service_ids;", + "DELETE FROM service_join_requests WHERE service_id in :service_ids;", + "DELETE FROM service_letter_branding WHERE service_id in :service_ids;", + "DELETE FROM service_permissions WHERE service_id in :service_ids;", + "DELETE FROM service_whitelist WHERE service_id in :service_ids;", + "DELETE FROM user_folder_permissions WHERE service_id in :service_ids", + "DELETE FROM template_folder_map WHERE template_id in (SELECT id FROM templates WHERE service_id in :service_ids);", + "DELETE FROM template_redacted WHERE template_id in (SELECT id FROM templates WHERE service_id in :service_ids);", + "DELETE FROM template_folder WHERE service_id in :service_ids;", + "DELETE FROM templates WHERE service_id in :service_ids;", + "DELETE FROM templates_history WHERE service_id in :service_ids;", + "DELETE FROM service_letter_contacts WHERE service_id in :service_ids;", + "DELETE FROM unsubscribe_request WHERE service_id in :service_ids;", + "DELETE FROM unsubscribe_request_history WHERE service_id in :service_ids;", + "DELETE FROM unsubscribe_request_report WHERE service_id in :service_ids;", + "DELETE FROM user_to_service WHERE service_id in :service_ids;", + # now we can finally delete the services + "DELETE FROM services WHERE id in :service_ids;", + "DELETE FROM services_history WHERE id in :service_ids;", + ] + + for delete_statement in delete_statements: + conn.execute(text(delete_statement), service_ids=broadcast_service_ids) + + +def downgrade(): + # undowngradeable! + pass From 4db6195426b69a1be0533ec329a363dc575e2375 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Tue, 1 Oct 2024 15:50:34 +0100 Subject: [PATCH 2/2] dont delete service-specific things if theres no services --- .../versions/0465_delete_broadcast_data.py | 78 ++++++++++--------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/migrations/versions/0465_delete_broadcast_data.py b/migrations/versions/0465_delete_broadcast_data.py index 54e680d2ad..f4757e55c0 100644 --- a/migrations/versions/0465_delete_broadcast_data.py +++ b/migrations/versions/0465_delete_broadcast_data.py @@ -52,45 +52,49 @@ def upgrade(): "DELETE FROM service_broadcast_provider_restriction;", # all these tables have service_id, so need to be deleted before we can delete the service, # (note this includes templates as well, which we need to delete), - "DELETE FROM annual_billing WHERE service_id in :service_ids;", - "DELETE FROM api_keys WHERE service_id in :service_ids;", - "DELETE FROM complaints WHERE service_id in :service_ids;", - "DELETE FROM service_sms_senders WHERE service_id in :service_ids;", - "DELETE FROM inbound_numbers WHERE service_id in :service_ids;", - "DELETE FROM inbound_sms WHERE service_id in :service_ids;", - "DELETE FROM inbound_sms_history WHERE service_id in :service_ids;", - "DELETE FROM invited_users WHERE service_id in :service_ids;", - "DELETE FROM jobs WHERE service_id in :service_ids;", - "DELETE FROM notification_history WHERE service_id in :service_ids;", - "DELETE FROM notifications WHERE service_id in :service_ids;", - "DELETE FROM permissions WHERE service_id in :service_ids;", - "DELETE FROM returned_letters WHERE service_id in :service_ids;", - "DELETE FROM service_callback_api WHERE service_id in :service_ids;", - "DELETE FROM service_contact_list WHERE service_id in :service_ids;", - "DELETE FROM service_data_retention WHERE service_id in :service_ids;", - "DELETE FROM service_email_branding WHERE service_id in :service_ids;", - "DELETE FROM service_email_reply_to WHERE service_id in :service_ids;", - "DELETE FROM service_inbound_api WHERE service_id in :service_ids;", - "DELETE FROM service_join_requests WHERE service_id in :service_ids;", - "DELETE FROM service_letter_branding WHERE service_id in :service_ids;", - "DELETE FROM service_permissions WHERE service_id in :service_ids;", - "DELETE FROM service_whitelist WHERE service_id in :service_ids;", - "DELETE FROM user_folder_permissions WHERE service_id in :service_ids", - "DELETE FROM template_folder_map WHERE template_id in (SELECT id FROM templates WHERE service_id in :service_ids);", - "DELETE FROM template_redacted WHERE template_id in (SELECT id FROM templates WHERE service_id in :service_ids);", - "DELETE FROM template_folder WHERE service_id in :service_ids;", - "DELETE FROM templates WHERE service_id in :service_ids;", - "DELETE FROM templates_history WHERE service_id in :service_ids;", - "DELETE FROM service_letter_contacts WHERE service_id in :service_ids;", - "DELETE FROM unsubscribe_request WHERE service_id in :service_ids;", - "DELETE FROM unsubscribe_request_history WHERE service_id in :service_ids;", - "DELETE FROM unsubscribe_request_report WHERE service_id in :service_ids;", - "DELETE FROM user_to_service WHERE service_id in :service_ids;", - # now we can finally delete the services - "DELETE FROM services WHERE id in :service_ids;", - "DELETE FROM services_history WHERE id in :service_ids;", ] + if broadcast_service_ids: + delete_statements += [ + "DELETE FROM annual_billing WHERE service_id in :service_ids;", + "DELETE FROM api_keys WHERE service_id in :service_ids;", + "DELETE FROM complaints WHERE service_id in :service_ids;", + "DELETE FROM service_sms_senders WHERE service_id in :service_ids;", + "DELETE FROM inbound_numbers WHERE service_id in :service_ids;", + "DELETE FROM inbound_sms WHERE service_id in :service_ids;", + "DELETE FROM inbound_sms_history WHERE service_id in :service_ids;", + "DELETE FROM invited_users WHERE service_id in :service_ids;", + "DELETE FROM jobs WHERE service_id in :service_ids;", + "DELETE FROM notification_history WHERE service_id in :service_ids;", + "DELETE FROM notifications WHERE service_id in :service_ids;", + "DELETE FROM permissions WHERE service_id in :service_ids;", + "DELETE FROM returned_letters WHERE service_id in :service_ids;", + "DELETE FROM service_callback_api WHERE service_id in :service_ids;", + "DELETE FROM service_contact_list WHERE service_id in :service_ids;", + "DELETE FROM service_data_retention WHERE service_id in :service_ids;", + "DELETE FROM service_email_branding WHERE service_id in :service_ids;", + "DELETE FROM service_email_reply_to WHERE service_id in :service_ids;", + "DELETE FROM service_inbound_api WHERE service_id in :service_ids;", + "DELETE FROM service_join_requests WHERE service_id in :service_ids;", + "DELETE FROM service_letter_branding WHERE service_id in :service_ids;", + "DELETE FROM service_permissions WHERE service_id in :service_ids;", + "DELETE FROM service_whitelist WHERE service_id in :service_ids;", + "DELETE FROM user_folder_permissions WHERE service_id in :service_ids", + "DELETE FROM template_folder_map WHERE template_id in (SELECT id FROM templates WHERE service_id in :service_ids);", + "DELETE FROM template_redacted WHERE template_id in (SELECT id FROM templates WHERE service_id in :service_ids);", + "DELETE FROM template_folder WHERE service_id in :service_ids;", + "DELETE FROM templates WHERE service_id in :service_ids;", + "DELETE FROM templates_history WHERE service_id in :service_ids;", + "DELETE FROM service_letter_contacts WHERE service_id in :service_ids;", + "DELETE FROM unsubscribe_request WHERE service_id in :service_ids;", + "DELETE FROM unsubscribe_request_history WHERE service_id in :service_ids;", + "DELETE FROM unsubscribe_request_report WHERE service_id in :service_ids;", + "DELETE FROM user_to_service WHERE service_id in :service_ids;", + # now we can finally delete the services + "DELETE FROM services WHERE id in :service_ids;", + "DELETE FROM services_history WHERE id in :service_ids;", + ] + for delete_statement in delete_statements: conn.execute(text(delete_statement), service_ids=broadcast_service_ids)