From ff15ec7f8974c39d32e4c6c861d1eaad0eabda97 Mon Sep 17 00:00:00 2001 From: tamar koledetzky Date: Sun, 15 Sep 2024 13:42:56 +0300 Subject: [PATCH 01/11] add_is_exists --- DB/NEW_KT_DB/DataAccess/ObjectManager.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/DB/NEW_KT_DB/DataAccess/ObjectManager.py b/DB/NEW_KT_DB/DataAccess/ObjectManager.py index 56c6948f..1bc6fab8 100644 --- a/DB/NEW_KT_DB/DataAccess/ObjectManager.py +++ b/DB/NEW_KT_DB/DataAccess/ObjectManager.py @@ -110,3 +110,14 @@ def convert_object_attributes_to_dictionary(**kwargs): dict[key] = value return dict + +def is_exists(self, object): + table_name = convert_object_name_to_management_table_name(object.object_name) + try: + query=f'select * from {table_name} where {object.pk_column} = {object.pk_value}' + result=self.db_manager.execute_query_with_single_result(query) + if result is None: + return False + return True + except sqlite3.OperationalError as e: + return False From 48600fc417f2b2865426f65955558679e7e1407b Mon Sep 17 00:00:00 2001 From: Tem-M Date: Sun, 15 Sep 2024 21:47:40 +0300 Subject: [PATCH 02/11] implementation complete - fixes needed --- .../Controller/DBSubnetGroupController.py | 24 ++++ .../DataAccess/DBSubnetGroupManager.py | 34 +++++ DB/NEW_KT_DB/Models/DBSubnetGroupModel.py | 67 ++++++++++ .../Service/Classes/DBSubnetGroupService.py | 74 +++++++++++ DB/NEW_KT_DB/Test/DBSubnetGroupTests.py | 121 ++++++++++++++++++ 5 files changed, 320 insertions(+) create mode 100644 DB/NEW_KT_DB/Controller/DBSubnetGroupController.py create mode 100644 DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py create mode 100644 DB/NEW_KT_DB/Models/DBSubnetGroupModel.py create mode 100644 DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py create mode 100644 DB/NEW_KT_DB/Test/DBSubnetGroupTests.py diff --git a/DB/NEW_KT_DB/Controller/DBSubnetGroupController.py b/DB/NEW_KT_DB/Controller/DBSubnetGroupController.py new file mode 100644 index 00000000..027dbcc9 --- /dev/null +++ b/DB/NEW_KT_DB/Controller/DBSubnetGroupController.py @@ -0,0 +1,24 @@ +from Service.Classes import DBSubnetGroupService + +class DBSubnetGroupController: + def __init__(self, service: DBSubnetGroupService): + self.service = service + + + def create_db_subnet_group(self, **kwargs): + self.service.create_db_subnet_group(**kwargs) + + + def delete_db_subnet_group(self, name): + self.service.delete_db_subnet_group(name) + + + def modify_db_subnet_group(self, name, updates): + self.service.modify_db_subnet_group(name, updates) + + def get_db_subnet_group(self, name): + return self.service.get_db_subnet_group(name) + + def describe_db_subnet_group(self, name): + return self.service.describe_db_subnet_group(name) + \ No newline at end of file diff --git a/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py b/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py new file mode 100644 index 00000000..d6cc671d --- /dev/null +++ b/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py @@ -0,0 +1,34 @@ +from typing import Dict, Any, List + +import sys +import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +from Models.DBSubnetGroupModel import DBSubnetGroup + +class DBSubnetGroupManager: + def __init__(self, db_manager, object_manager): + self.db_manager = db_manager + self.object_manager = object_manager + self.db_manager.create_table(DBSubnetGroup.table_name, DBSubnetGroup.table_structure) + + def create(self, subnet_group: DBSubnetGroup): + self.object_manager.save_in_memory(DBSubnetGroup.table_name, subnet_group.to_sql()) + + def get(self, name: str): + cols = ['db_subnet_group_name', 'db_subnet_group_description', 'vpc_id', 'subnets', 'db_subnet_group_arn', 'status'] + data = self.object_manager.get_from_memory(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, name, cols = cols) + return data + + def delete(self, name: str): + self.object_manager.delete_from_memory(name) + + def describe(self, name: str): + cols = ['db_subnet_group_name', 'db_subnet_group_description', 'vpc_id', 'subnets', 'db_subnet_group_arn', 'status'] + return self.object_manager.get_from_memory(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, name, cols) + + def modify(self, subnet_group: DBSubnetGroup): + updates = subnet_group.to_dict() + del updates['db_subnet_group_name'] + self.object_manager.update_in_memory(DBSubnetGroup.pk_column, subnet_group.db_subnet_group_name, DBSubnetGroup.table_name, updates) \ No newline at end of file diff --git a/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py b/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py new file mode 100644 index 00000000..9b933bea --- /dev/null +++ b/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py @@ -0,0 +1,67 @@ +from typing import List, Dict, Any +import json +import sys +import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +from DataAccess.ObjectManager import ObjectManager + +class DBSubnetGroup: + + pk_column = 'db_subnet_group_name' + table_name = 'db_subnet_groups' + table_structure = f""" + db_subnet_group_name primary key not null, + db_subnet_group_description TEXT NOT NULL, + vpc_id VARCHAR(255) NOT NULL, + subnets JSONB DEFAULT '{{}}', + db_subnet_group_arn VARCHAR(255), + status VARCHAR(50) DEFAULT 'pending' + """ + + def __init__(self, **kwargs): + try: + self.db_subnet_group_name = kwargs['db_subnet_group_name'] + self.db_subnet_group_description = kwargs['db_subnet_group_description'] + self.vpc_id = kwargs['vpc_id'] + self.subnets = kwargs.get('subnets', None) + if not self.subnets: + self.subnets = dict() + self.db_subnet_group_arn = kwargs.get('db_subnet_group_arn', None) + + except KeyError as e: + raise ValueError(f"Missing required attribute for DBSubnetGroup: {str(e)}") + + # Ideally: + # self.db_subnet_group_arn should be dynamically created according to vpc-id, account-id and + # subnet-group-name, and then dynamically added to the routing table + + self.status = 'pending' + self.pk_value = self.db_subnet_group_name + + def to_dict(self) -> Dict[str, Any]: + return ObjectManager.convert_object_attributes_to_dictionary( + db_subnet_group_name=self.db_subnet_group_name, + db_subnet_group_description = self.db_subnet_group_description, + vpc_id = self.vpc_id, + subnets = self.subnets, + db_subnet_group_arn = self.db_subnet_group_arn, + status = self.status + ) + + def to_bytes(self): + bytes = json.dumps(self.to_dict()) + bytes = bytes.encode('utf-8') + return bytes + + def from_bytes_to_dict(bytes): + return json.loads(bytes.decode('utf-8')) + + + def to_sql(self): + # Convert the model instance to a dictionary + data_dict = self.to_dict() + values = '(' + ", ".join(f'\'{json.dumps(v)}\'' if isinstance(v, dict) or isinstance(v, list) else f'\'{v}\'' if isinstance(v, str) else f'\'{str(v)}\'' + for v in data_dict.values()) + ')' + return values \ No newline at end of file diff --git a/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py b/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py new file mode 100644 index 00000000..1c84896b --- /dev/null +++ b/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py @@ -0,0 +1,74 @@ +from typing import List, Dict, Any + +import sys +import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + + +from Models.DBSubnetGroupModel import DBSubnetGroup +from DataAccess.DBSubnetGroupManager import DBSubnetGroupManager +from Storage.KT_Storage.DataAccess.StorageManager import StorageManager +from Storage.KT_Storage.DataAccess.VersionManager import VersionManager +from Validation.GeneralValidations import * +class DBSubnetGroupService: + def __init__(self, db_subnet_group_manager: DBSubnetGroupManager): + self.manager = db_subnet_group_manager + self.bucket = 'db_subnet_groups' + self.storage_manager = StorageManager() + self.storage_manager.create_bucket(self.bucket) + self.version_manager = VersionManager() + + def create_db_subnet_group(self, **kwargs): + # object + if not kwargs.get('db_subnet_group_name'): + raise ValueError('Missing required argument db_subnet_group_name') + + if not is_length_in_range(kwargs['db_subnet_group_name'], 1, 255): + raise ValueError("invalid length for subnet group db_subnet_group_name: " + len(kwargs['db_subnet_group_name'])) + + if kwargs.get('description') and not is_length_in_range('description', 1, 255): + raise ValueError("invalid length for subnet group description: " + len(kwargs['description'])) + + subnet_group = DBSubnetGroup(**kwargs) + # physical object + # version = 0 assume created for the first time + self.storage_manager.create(self.bucket, subnet_group.db_subnet_group_name, subnet_group.to_bytes(), '0') + # save in memory + self.manager.create(subnet_group) + + def get_db_subnet_group(self, db_subnet_group_name: str) -> DBSubnetGroup: + data = self.manager.get(db_subnet_group_name) + return DBSubnetGroup(**data) + + def modify_db_subnet_group(self, db_subnet_group_name: str, updates: Dict[str, Any]) -> DBSubnetGroup: + if not db_subnet_group_name: + raise ValueError('Missing required argument db_subnet_group_name') + + + if updates.get('description') and not is_length_in_range(updates['description'], 1, 255): + raise ValueError("invalid length for subnet group description: " + len(updates['description'])) + + subnet_group = self.get_db_subnet_group(db_subnet_group_name) + + for key, value in updates.items(): + setattr(subnet_group, key, value) + + self.manager.modify(subnet_group) + # version = str(int(self.version_manager.get(self.bucket, subnet_group.db_subnet_group_name).version_id)+1) + # for now we override the basic version, when the latest version id can be retrieved, we will make a new version as old_version_id + 1 + self.storage_manager.create(self.bucket, db_subnet_group_name, subnet_group.to_bytes(), '0') + + def delete_db_subnet_group(self, db_subnet_group_name: str) -> None: + if not db_subnet_group_name: + raise ValueError('Missing required argument db_subnet_group_name') + + self.manager.delete(db_subnet_group_name) + self.storage_manager.delete_by_db_subnet_group_name(bucket_db_subnet_group_name=self.bucket, version_id=self.version_manager.get(self.bucket, db_subnet_group_name).version_id, key=db_subnet_group_name) + + def describe_db_subnet_group(self, db_subnet_group_name: str) -> Dict: + if not db_subnet_group_name: + raise ValueError('Missing required argument db_subnet_group_name') + + return self.manager.describe(db_subnet_group_name) \ No newline at end of file diff --git a/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py b/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py new file mode 100644 index 00000000..93fa618a --- /dev/null +++ b/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py @@ -0,0 +1,121 @@ +from sqlite3 import IntegrityError +import pytest +import json + +import sys +import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '.'))) + +from Service.Classes.DBSubnetGroupService import DBSubnetGroupService +from DataAccess.DBSubnetGroupManager import DBSubnetGroupManager +from Controller.DBSubnetGroupController import DBSubnetGroupController +from Storage.KT_Storage.DataAccess.StorageManager import StorageManager +from DataAccess.ObjectManager import ObjectManager +from DataAccess.DBManager import DBManager +from Models.DBSubnetGroupModel import DBSubnetGroup + +db_manager = DBManager('../object_management_db.db') +object_manager = ObjectManager('../object_management_db.db') +manager = DBSubnetGroupManager(db_manager=db_manager, object_manager=object_manager) +service = DBSubnetGroupService(manager) +controller = DBSubnetGroupController(service) +storage_manager = StorageManager() + +def test_create(): + controller.create_db_subnet_group( + db_subnet_group_name='subnet_group_1', + subnets=[ + {'subnet_id': 'subnet-12345678'}, + {'subnet_id': 'subnet-87654321'} + ], + db_subnet_group_description='Test subnet group', + vpc_id='vpc-12345678', + db_subnet_group_arn='arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' + ) + + # assert bucket was created + # check that file was created + assert storage_manager.get('db_subnet_groups', 'subnet_group_1', '0') != None + # check that object was saved to management table + assert controller.get_db_subnet_group('subnet_group_1') != None + # check that file content is correct + from_storage = DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', 'subnet_group_1', '0')['content']) + from_db = controller.get_db_subnet_group('subnet_group_1').to_dict() + assert from_storage['db_subnet_group_name'] == from_db['db_subnet_group_name'] + for subnet in from_storage['subnets']: + assert subnet in json.loads(from_db['subnets']) + assert from_storage['db_subnet_group_description'] == from_db['db_subnet_group_description'] + assert from_storage['vpc_id'] == from_db['vpc_id'] + assert from_storage['db_subnet_group_arn'] == from_db['db_subnet_group_arn'] + assert from_storage['status'] == from_db['status'] + +def test_unique_constraint(): + with pytest.raises(IntegrityError, match="UNIQUE constraint failed"): + controller.create_db_subnet_group( + db_subnet_group_name='subnet_group_1', + subnets=[ + {'subnet_id': 'subnet-87654321'}, + {'subnet_id': 'subnet-12345678'} + ], + db_subnet_group_description='Another subnet group with same name', + vpc_id='vpc-87654321', + db_subnet_group_arn='arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' + ) + +def test_modify(): + controller.modify_db_subnet_group( + name='subnet_group_1', + updates= {'subnets':[ + {'subnet_id': 'subnet-12345988'}, + {'subnet_id': 'subnet-876543881'}]} + ) + + from_storage = DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', 'subnet_group_1', '0')['content']) + from_db = controller.get_db_subnet_group('subnet_group_1') + assert from_storage['db_subnet_group_name'] == from_db.db_subnet_group_name + for subnet in from_storage['subnets']: + assert str(subnet) in from_db.subnets + assert from_storage['db_subnet_group_description'] == from_db.db_subnet_group_description + assert from_storage['vpc_id'] == from_db.vpc_id + assert from_storage['db_subnet_group_arn'] == from_db.db_subnet_group_arn + assert from_storage['status'] == from_db.status + +def test_get(): + subnet_group_1 = controller.get_db_subnet_group('subnet_group_1') + assert subnet_group_1 != None + assert subnet_group_1.db_subnet_group_name == 'subnet_group_1' + assert subnet_group_1.db_subnet_group_description == 'Test subnet group' + assert subnet_group_1.vpc_id == 'vpc-12345678' + assert subnet_group_1.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' + assert subnet_group_1.status == 'pending' + for subnet in json.loads(subnet_group_1.subnets): + assert subnet in [ + {"subnet_id": "subnet-12345678"}, + {"subnet_id": "subnet-87654321"} + ] + + +def test_describe(): + subnet_group_1 = controller.describe_db_subnet_group('subnet_group_1') + assert subnet_group_1 != None + assert subnet_group_1.db_subnet_group_name == 'subnet_group_1' + assert subnet_group_1.db_subnet_group_description == 'Test subnet group' + assert subnet_group_1.vpc_id == 'vpc-12345678' + assert subnet_group_1.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' + assert subnet_group_1.status == 'pending' + for subnet in json.loads(subnet_group_1.subnets): + assert subnet in json.dumps([ + {'subnet_id': 'subnet-12345678'}, + {'subnet_id': 'subnet-87654321'} + ]) + + +def test_delete(): + controller.delete_db_subnet_group('subnet_group_1') + assert storage_manager.get('db_subnet_groups', 'subnet_group_1', '0') == None + assert controller.get_db_subnet_group('subnet_group_1') == None + + + \ No newline at end of file From 7c103e8ff62a54f0266e59d065a1a8f8bdc6483f Mon Sep 17 00:00:00 2001 From: Tem-M Date: Mon, 16 Sep 2024 11:18:33 +0300 Subject: [PATCH 03/11] tests pass --- .../DataAccess/DBSubnetGroupManager.py | 26 +-- DB/NEW_KT_DB/Models/DBSubnetGroupModel.py | 5 +- .../Service/Classes/DBSubnetGroupService.py | 28 ++- DB/NEW_KT_DB/Test/DBSubnetGroupTests.py | 187 +++++++++++++----- 4 files changed, 183 insertions(+), 63 deletions(-) diff --git a/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py b/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py index d6cc671d..d601ff4f 100644 --- a/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py @@ -8,27 +8,31 @@ from Models.DBSubnetGroupModel import DBSubnetGroup class DBSubnetGroupManager: - def __init__(self, db_manager, object_manager): - self.db_manager = db_manager + def __init__(self, object_manager): self.object_manager = object_manager - self.db_manager.create_table(DBSubnetGroup.table_name, DBSubnetGroup.table_structure) + self.object_manager.create_management_table(DBSubnetGroup.table_name, DBSubnetGroup.table_structure) def create(self, subnet_group: DBSubnetGroup): - self.object_manager.save_in_memory(DBSubnetGroup.table_name, subnet_group.to_sql()) + self.object_manager.insert_object_to_management_table(DBSubnetGroup.table_name, subnet_group.to_sql()) def get(self, name: str): - cols = ['db_subnet_group_name', 'db_subnet_group_description', 'vpc_id', 'subnets', 'db_subnet_group_arn', 'status'] - data = self.object_manager.get_from_memory(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, name, cols = cols) - return data + data = self.object_manager.get_object_from_management_table(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, name) + if data: + data_mapping = {'db_subnet_group_name':name} + for key, value in data[name].items(): + data_mapping[key] = value + return DBSubnetGroup(**data_mapping) + else: + raise ValueError(f"subnet group with name '{name}' not found") + def delete(self, name: str): - self.object_manager.delete_from_memory(name) + self.object_manager.delete_object_from_management_table(DBSubnetGroup.table_name, f"{DBSubnetGroup.pk_column}='{name}'") def describe(self, name: str): - cols = ['db_subnet_group_name', 'db_subnet_group_description', 'vpc_id', 'subnets', 'db_subnet_group_arn', 'status'] - return self.object_manager.get_from_memory(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, name, cols) + return self.get(name).to_dict() def modify(self, subnet_group: DBSubnetGroup): updates = subnet_group.to_dict() del updates['db_subnet_group_name'] - self.object_manager.update_in_memory(DBSubnetGroup.pk_column, subnet_group.db_subnet_group_name, DBSubnetGroup.table_name, updates) \ No newline at end of file + self.object_manager.update_object_in_management_table_by_criteria(table_name = DBSubnetGroup.table_name, updates = updates, criteria = f"{DBSubnetGroup.pk_column}='{subnet_group.db_subnet_group_name}'") \ No newline at end of file diff --git a/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py b/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py index 9b933bea..ccb0bff7 100644 --- a/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py +++ b/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py @@ -1,4 +1,5 @@ from typing import List, Dict, Any +import ast import json import sys import os @@ -27,7 +28,9 @@ def __init__(self, **kwargs): self.vpc_id = kwargs['vpc_id'] self.subnets = kwargs.get('subnets', None) if not self.subnets: - self.subnets = dict() + self.subnets = [] + if type(self.subnets) is not list: + self.subnets = ast.literal_eval(self.subnets) self.db_subnet_group_arn = kwargs.get('db_subnet_group_arn', None) except KeyError as e: diff --git a/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py b/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py index 1c84896b..0c42b29e 100644 --- a/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py +++ b/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py @@ -1,3 +1,4 @@ +from sqlite3 import IntegrityError from typing import List, Dict, Any import sys @@ -19,6 +20,7 @@ def __init__(self, db_subnet_group_manager: DBSubnetGroupManager): self.storage_manager = StorageManager() self.storage_manager.create_bucket(self.bucket) self.version_manager = VersionManager() + self.subnet_groups = dict() def create_db_subnet_group(self, **kwargs): # object @@ -28,19 +30,29 @@ def create_db_subnet_group(self, **kwargs): if not is_length_in_range(kwargs['db_subnet_group_name'], 1, 255): raise ValueError("invalid length for subnet group db_subnet_group_name: " + len(kwargs['db_subnet_group_name'])) + if kwargs['db_subnet_group_name'] in self.subnet_groups: + raise ValueError(f"db_subnet_group_name {kwargs['db_subnet_group_name']} already exists") + if kwargs.get('description') and not is_length_in_range('description', 1, 255): raise ValueError("invalid length for subnet group description: " + len(kwargs['description'])) subnet_group = DBSubnetGroup(**kwargs) + # save in management table + # in try except block in case the server was shut down and re-run and local collection doesn't include all subnetGroups + try: + self.manager.create(subnet_group) + except IntegrityError as e: + raise ValueError(f"db_subnet_group_name {kwargs['db_subnet_group_name']} already exists") + # physical object # version = 0 assume created for the first time self.storage_manager.create(self.bucket, subnet_group.db_subnet_group_name, subnet_group.to_bytes(), '0') - # save in memory - self.manager.create(subnet_group) + # save in local collection (hash table) for quick access + self.subnet_groups[kwargs['db_subnet_group_name']] = subnet_group def get_db_subnet_group(self, db_subnet_group_name: str) -> DBSubnetGroup: data = self.manager.get(db_subnet_group_name) - return DBSubnetGroup(**data) + return data def modify_db_subnet_group(self, db_subnet_group_name: str, updates: Dict[str, Any]) -> DBSubnetGroup: if not db_subnet_group_name: @@ -63,9 +75,15 @@ def modify_db_subnet_group(self, db_subnet_group_name: str, updates: Dict[str, A def delete_db_subnet_group(self, db_subnet_group_name: str) -> None: if not db_subnet_group_name: raise ValueError('Missing required argument db_subnet_group_name') - + + # delete from management table self.manager.delete(db_subnet_group_name) - self.storage_manager.delete_by_db_subnet_group_name(bucket_db_subnet_group_name=self.bucket, version_id=self.version_manager.get(self.bucket, db_subnet_group_name).version_id, key=db_subnet_group_name) + # for now version id is 0 + # delete physical object from storage + self.storage_manager.delete_by_name(bucket_name=self.bucket, version_id='0', key=db_subnet_group_name) + # delete from local collection (hash table) + if db_subnet_group_name in self.subnet_groups: + del self.subnet_groups[db_subnet_group_name] def describe_db_subnet_group(self, db_subnet_group_name: str) -> Dict: if not db_subnet_group_name: diff --git a/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py b/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py index 93fa618a..8dc69fcb 100644 --- a/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py +++ b/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py @@ -13,17 +13,23 @@ from Controller.DBSubnetGroupController import DBSubnetGroupController from Storage.KT_Storage.DataAccess.StorageManager import StorageManager from DataAccess.ObjectManager import ObjectManager -from DataAccess.DBManager import DBManager from Models.DBSubnetGroupModel import DBSubnetGroup +import sqlite3 -db_manager = DBManager('../object_management_db.db') object_manager = ObjectManager('../object_management_db.db') -manager = DBSubnetGroupManager(db_manager=db_manager, object_manager=object_manager) +manager = DBSubnetGroupManager(object_manager=object_manager) service = DBSubnetGroupService(manager) controller = DBSubnetGroupController(service) storage_manager = StorageManager() def test_create(): + + # remove existing subnet group from previous tests + conn = sqlite3.connect('../object_management_db.db') + conn.execute("delete from db_subnet_groups where db_subnet_group_name = 'subnet_group_1'") + conn.commit() + conn.close() + controller.create_db_subnet_group( db_subnet_group_name='subnet_group_1', subnets=[ @@ -35,24 +41,47 @@ def test_create(): db_subnet_group_arn='arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' ) - # assert bucket was created - # check that file was created - assert storage_manager.get('db_subnet_groups', 'subnet_group_1', '0') != None - # check that object was saved to management table - assert controller.get_db_subnet_group('subnet_group_1') != None + # check that file was created (no error raised on get) + storage_manager.get('db_subnet_groups', 'subnet_group_1', '0') + # check that object was saved to management table (no error raised on get) + controller.get_db_subnet_group('subnet_group_1') + # check that file content is correct - from_storage = DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', 'subnet_group_1', '0')['content']) - from_db = controller.get_db_subnet_group('subnet_group_1').to_dict() - assert from_storage['db_subnet_group_name'] == from_db['db_subnet_group_name'] - for subnet in from_storage['subnets']: - assert subnet in json.loads(from_db['subnets']) - assert from_storage['db_subnet_group_description'] == from_db['db_subnet_group_description'] - assert from_storage['vpc_id'] == from_db['vpc_id'] - assert from_storage['db_subnet_group_arn'] == from_db['db_subnet_group_arn'] - assert from_storage['status'] == from_db['status'] + from_storage = DBSubnetGroup(**DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', 'subnet_group_1', '0')['content'])) + from_db = controller.get_db_subnet_group('subnet_group_1') + # check values were stored correctly in management table as well as storage + + # group name + assert from_storage.db_subnet_group_name == 'subnet_group_1' + assert from_db.db_subnet_group_name == 'subnet_group_1' + + # subnets + for subnet in from_storage.subnets: + assert subnet in [ + {'subnet_id': 'subnet-12345678'}, + {'subnet_id': 'subnet-87654321'} + ] + for subnet in from_db.subnets: + assert subnet in [ + {'subnet_id': 'subnet-12345678'}, + {'subnet_id': 'subnet-87654321'} + ] + + # description + assert from_storage.db_subnet_group_description == 'Test subnet group' + assert from_db.db_subnet_group_description == 'Test subnet group' + + # vpc_id + assert from_storage.vpc_id == 'vpc-12345678' + assert from_db.vpc_id == 'vpc-12345678' + + # arn + assert from_storage.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' + assert from_db.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' + def test_unique_constraint(): - with pytest.raises(IntegrityError, match="UNIQUE constraint failed"): + with pytest.raises(ValueError): controller.create_db_subnet_group( db_subnet_group_name='subnet_group_1', subnets=[ @@ -63,24 +92,7 @@ def test_unique_constraint(): vpc_id='vpc-87654321', db_subnet_group_arn='arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' ) - -def test_modify(): - controller.modify_db_subnet_group( - name='subnet_group_1', - updates= {'subnets':[ - {'subnet_id': 'subnet-12345988'}, - {'subnet_id': 'subnet-876543881'}]} - ) - - from_storage = DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', 'subnet_group_1', '0')['content']) - from_db = controller.get_db_subnet_group('subnet_group_1') - assert from_storage['db_subnet_group_name'] == from_db.db_subnet_group_name - for subnet in from_storage['subnets']: - assert str(subnet) in from_db.subnets - assert from_storage['db_subnet_group_description'] == from_db.db_subnet_group_description - assert from_storage['vpc_id'] == from_db.vpc_id - assert from_storage['db_subnet_group_arn'] == from_db.db_subnet_group_arn - assert from_storage['status'] == from_db.status + def test_get(): subnet_group_1 = controller.get_db_subnet_group('subnet_group_1') @@ -90,32 +102,115 @@ def test_get(): assert subnet_group_1.vpc_id == 'vpc-12345678' assert subnet_group_1.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' assert subnet_group_1.status == 'pending' - for subnet in json.loads(subnet_group_1.subnets): + for subnet in subnet_group_1.subnets: assert subnet in [ {"subnet_id": "subnet-12345678"}, {"subnet_id": "subnet-87654321"} ] - +def test_modify(): + controller.modify_db_subnet_group( + name='subnet_group_1', + updates= {'subnets':[ + {'subnet_id': 'subnet-12345988'}, + {'subnet_id': 'subnet-876543881'}]} + ) + + from_storage = DBSubnetGroup(**DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', 'subnet_group_1', '0')['content'])) + from_db = controller.get_db_subnet_group('subnet_group_1') + assert from_storage.db_subnet_group_name == from_db.db_subnet_group_name + for subnet in from_storage.subnets: + assert subnet in from_db.subnets + assert from_storage.db_subnet_group_description == from_db.db_subnet_group_description + assert from_storage.vpc_id == from_db.vpc_id + assert from_storage.db_subnet_group_arn == from_db.db_subnet_group_arn + assert from_storage.status == from_db.status + def test_describe(): subnet_group_1 = controller.describe_db_subnet_group('subnet_group_1') + assert type(subnet_group_1) == dict + assert type(subnet_group_1['subnets']) == list + assert type(subnet_group_1['subnets'][0]) == dict + subnet_group_1 = DBSubnetGroup(**subnet_group_1) assert subnet_group_1 != None assert subnet_group_1.db_subnet_group_name == 'subnet_group_1' assert subnet_group_1.db_subnet_group_description == 'Test subnet group' assert subnet_group_1.vpc_id == 'vpc-12345678' assert subnet_group_1.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' assert subnet_group_1.status == 'pending' - for subnet in json.loads(subnet_group_1.subnets): - assert subnet in json.dumps([ - {'subnet_id': 'subnet-12345678'}, - {'subnet_id': 'subnet-87654321'} - ]) + for subnet in subnet_group_1.subnets: + assert subnet in [ + {'subnet_id': 'subnet-12345988'}, + {'subnet_id': 'subnet-876543881'} + ] def test_delete(): controller.delete_db_subnet_group('subnet_group_1') - assert storage_manager.get('db_subnet_groups', 'subnet_group_1', '0') == None - assert controller.get_db_subnet_group('subnet_group_1') == None + with pytest.raises(FileNotFoundError): + storage_manager.get('db_subnet_groups', 'subnet_group_1', '0') + with pytest.raises(Exception): + controller.get_db_subnet_group('subnet_group_1') + + +@pytest.mark.parametrize("index", range(20)) +def test_insert_many(index): + controller.create_db_subnet_group( + db_subnet_group_name=f'subnet_group_{index}', + subnets=[ + {'subnet_id': f'subnet-1234567{index}'}, + {'subnet_id': f'subnet-8765432{index}'} + ], + db_subnet_group_description=f'Test subnet group {index}', + vpc_id=f'vpc-1234567{index}', + db_subnet_group_arn=f'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_{index}' + ) + + # check that file was created (no error raised on get) + storage_manager.get('db_subnet_groups', f'subnet_group_{index}', '0') + # check that object was saved to management table (no error raised on get) + controller.get_db_subnet_group(f'subnet_group_{index}') + + # check that file content is correct + from_storage = DBSubnetGroup(**DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', f'subnet_group_{index}', '0')['content'])) + from_db = controller.get_db_subnet_group(f'subnet_group_{index}') + # check values were stored correctly in management table as well as storage + + # group name + assert from_storage.db_subnet_group_name == f'subnet_group_{index}' + assert from_db.db_subnet_group_name == f'subnet_group_{index}' + + # subnets + for subnet in from_storage.subnets: + assert subnet in [ + {'subnet_id': f'subnet-1234567{index}'}, + {'subnet_id': f'subnet-8765432{index}'} + ] + for subnet in from_db.subnets: + assert subnet in [ + {'subnet_id': f'subnet-1234567{index}'}, + {'subnet_id': f'subnet-8765432{index}'} + ] + + # description + assert from_storage.db_subnet_group_description == f'Test subnet group {index}' + assert from_db.db_subnet_group_description == f'Test subnet group {index}' + + # vpc_id + assert from_storage.vpc_id == f'vpc-1234567{index}' + assert from_db.vpc_id == f'vpc-1234567{index}' + + # arn + assert from_storage.db_subnet_group_arn == f'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_{index}' + assert from_db.db_subnet_group_arn == f'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_{index}' + - \ No newline at end of file +@pytest.mark.parametrize("index", range(20)) +def test_delete_many_from_prev_test(index): + db_subnet_group_name=f'subnet_group_{index}' + controller.delete_db_subnet_group(db_subnet_group_name) + with pytest.raises(FileNotFoundError): + storage_manager.get('db_subnet_groups', db_subnet_group_name, '0') + with pytest.raises(Exception): + controller.get_db_subnet_group(db_subnet_group_name) From 2f5fdf8aac752d95625a51244ed370f0ac3549ff Mon Sep 17 00:00:00 2001 From: Tem-M Date: Mon, 16 Sep 2024 12:09:43 +0300 Subject: [PATCH 04/11] add cleanup fixture to tests --- DB/NEW_KT_DB/Test/DBSubnetGroupTests.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py b/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py index 8dc69fcb..b2fe7674 100644 --- a/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py +++ b/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py @@ -21,15 +21,27 @@ service = DBSubnetGroupService(manager) controller = DBSubnetGroupController(service) storage_manager = StorageManager() - -def test_create(): - - # remove existing subnet group from previous tests - conn = sqlite3.connect('../object_management_db.db') - conn.execute("delete from db_subnet_groups where db_subnet_group_name = 'subnet_group_1'") + +@pytest.fixture +def clear_table(): + # Connect to the SQLite database + conn = conn = sqlite3.connect('../object_management_db.db') + cursor = conn.cursor() + + # Clear the table if it exists + table_name = "db_subnet_groups" + cursor.execute(f"DELETE FROM {table_name};") + + # Commit changes and close the connection conn.commit() conn.close() + + # Yield to allow tests to run + yield + +def test_create(clear_table): + # remove existing subnet group from previous tests controller.create_db_subnet_group( db_subnet_group_name='subnet_group_1', subnets=[ From b7b97a3e576873342b201a05ca78e1cc4af76326 Mon Sep 17 00:00:00 2001 From: Tem-M Date: Mon, 16 Sep 2024 12:22:19 +0300 Subject: [PATCH 05/11] technical commit --- DB/NEW_KT_DB/DataAccess/DBManager.py | 80 ++++++++++--------- DB/NEW_KT_DB/DataAccess/ObjectManager.py | 16 ++-- .../KT_Storage/DataAccess/StorageManager.py | 10 +-- .../KT_Storage/DataAccess/VersionManager.py | 10 ++- Storage/KT_Storage/Models/VesionModel.py | 4 +- 5 files changed, 66 insertions(+), 54 deletions(-) diff --git a/DB/NEW_KT_DB/DataAccess/DBManager.py b/DB/NEW_KT_DB/DataAccess/DBManager.py index dd0dde58..79a4ec0d 100644 --- a/DB/NEW_KT_DB/DataAccess/DBManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBManager.py @@ -11,21 +11,21 @@ def __init__(self, db_file: str): # rachel-8511, ShaniStrassProg def close(self): - '''Close the database connection.''' - self.connection.close() + '''Close the database connection.''' + self.connection.close() # saraNoigershel def execute_query_with_multiple_results(self, query: str) -> Optional[List[Tuple]]: - '''Execute a given query and return the results.''' - try: - c = self.connection.cursor() - c.execute(query) - results = c.fetchall() - # self.connection.commit() ??? - return results if results else None - except OperationalError as e: - raise Exception(f'Error executing query {query}: {e}') + '''Execute a given query and return the results.''' + try: + c = self.connection.cursor() + c.execute(query) + results = c.fetchall() + # self.connection.commit() ??? + return results if results else None + except OperationalError as e: + raise Exception(f'Error executing query {query}: {e}') # ShaniStrassProg @@ -56,42 +56,41 @@ def execute_query_without_results(self, query: str): def create_table(self, table_name, table_structure): '''create a table in a given db by given table_structure''' create_statement = f'''CREATE TABLE IF NOT EXISTS {table_name} ({table_structure})''' - execute_query_without_results(create_statement) + self.execute_query_without_results(create_statement) # Riki7649255 based on rachel-8511, ShaniStrassProg def insert_data_into_table(self, table_name, data): insert_statement = f'''INSERT INTO {table_name} VALUES {data}''' - execute_query_without_results(insert_statement) + self.execute_query_without_results(insert_statement) # Riki7649255 based on rachel-8511, Shani def update_records_in_table(self, table_name: str, updates: Dict[str, Any], criteria: str) -> None: - '''Update records in the specified table based on criteria.''' - - # add documentation here - set_clause = ', '.join([f'{k} = ?' for k in updates.keys()]) - values = list(updates.values()) - - update_statement = f''' - UPDATE {table_name} - SET {set_clause} - WHERE {criteria} - ''' - - execute_query_without_results(update_statement) + '''Update records in the specified table based on criteria.''' + + # add documentation here + set_clause = '(' + ', '.join([f'{k}' for k in updates.keys()]) + ') = (' + ', '.join([f'"{v}"' for v in updates.values()]) + ')' + + update_statement = f''' + UPDATE {table_name} + SET {set_clause} + WHERE {criteria} + ''' + + self.execute_query_without_results(update_statement) # Riki7649255 based on rachel-8511 def delete_data_from_table(self, table_name: str, criteria: str) -> None: - '''Delete a record from the specified table based on criteria.''' - - delete_statement = f''' - DELETE FROM {table_name} - WHERE {criteria} - ''') - - execute_query_without_results(delete_statement) + '''Delete a record from the specified table based on criteria.''' + + delete_statement = f''' + DELETE FROM {table_name} + WHERE {criteria} + ''' + + self.execute_query_without_results(delete_statement) # rachel-8511, Riki7649255 @@ -104,14 +103,21 @@ def select_and_return_records_from_table(self, table_name: str, columns: List[st Returns: Dict[int, Dict[str, Any]]: A dictionary where keys are object_ids and values are metadata. ''' + + if columns == ['*']: + columns = [res[1] for res in self.connection.execute(f'PRAGMA table_info({table_name});').fetchall()] + columns_clause = ', '.join(columns) query = f'SELECT {columns_clause} FROM {table_name}' if criteria: - query += f' WHERE {criteria}' + query += f' WHERE {criteria};' + try: - results = execute_query_with_multiple_results(query) - return {result[0]: dict(zip(columns, result[1:])) for result in results} + results = self.execute_query_with_multiple_results(query) + return {result[0]: dict(zip(columns[1:], result[1:])) for result in results} except OperationalError as e: + raise Exception(f'Error selecting from {table_name}: {e}') + except TypeError as e: raise Exception(f'Error selecting from {table_name}: {e}') diff --git a/DB/NEW_KT_DB/DataAccess/ObjectManager.py b/DB/NEW_KT_DB/DataAccess/ObjectManager.py index 56c6948f..21baa349 100644 --- a/DB/NEW_KT_DB/DataAccess/ObjectManager.py +++ b/DB/NEW_KT_DB/DataAccess/ObjectManager.py @@ -1,7 +1,7 @@ from typing import Dict, Any import json import sqlite3 -from DBManager import DBManager +from .DBManager import DBManager class ObjectManager: def __init__(self, db_file: str): @@ -12,7 +12,7 @@ def __init__(self, db_file: str): # for internal use only: # Riki7649255 based on rachel-8511 - def create_management_table(self, table_name, table_structure='object_id INTEGER PRIMARY KEY AUTOINCREMENT,type_object TEXT NOT NULL,metadata TEXT NOT NULL') + def create_management_table(self, table_name, table_structure='object_id INTEGER PRIMARY KEY AUTOINCREMENT,type_object TEXT NOT NULL,metadata TEXT NOT NULL'): self.db_manager.create_table(table_name, table_structure) @@ -27,11 +27,11 @@ def update_object_in_management_table_by_criteria(self, table_name, updates, cri # rachel-8511, Riki7649255 - def get_object_from_management_table(self, object_id: int) -> Dict[str, Any]: + def get_object_from_management_table(self, pk_col, table_name, object_id: int) -> Dict[str, Any]: '''Retrieve an object from the database.''' - result = self.db_manager.select_and_return_records_from_table(self.table_name, ['type_object', 'metadata'], f'object_id = {object_id}') + result = self.db_manager.select_and_return_records_from_table(table_name=table_name, criteria=f'{pk_col} = \'{object_id}\'') if result: - return result[object_id] + return result else: raise FileNotFoundError(f'Object with ID {object_id} not found.') @@ -68,7 +68,7 @@ def save_in_memory(self, object): # insert object info into management table mng_{object_name}s # for exmple: object db_instance will be saved in table mng_db_instances - table_name = convert_object_name_to_management_table_name(self.object_name) + table_name = ObjectManager.convert_object_name_to_management_table_name(self.object_name) if not is_management_table_exist(table_name): create_management_table(table_name) @@ -82,7 +82,7 @@ def delete_from_memory(self,criteria='default'): if criteria == 'default': criteria = f'{self.pk_column} = {self.pk_value}' - table_name = convert_object_name_to_management_table_name(self.object_name) + table_name = ObjectManager.convert_object_name_to_management_table_name(self.object_name) delete_data_from_table(table_name, criteria) @@ -93,7 +93,7 @@ def update_in_memory(self, updates, criteria='default'): if criteria == 'default': criteria = f'{self.pk_column} = {self.pk_value}' - table_name = convert_object_name_to_management_table_name(self.object_name) + table_name = ObjectManager.convert_object_name_to_management_table_name(self.object_name) update_object_in_management_table_by_criteria(table_name, updates, criteria) diff --git a/Storage/KT_Storage/DataAccess/StorageManager.py b/Storage/KT_Storage/DataAccess/StorageManager.py index eee45534..94f7e5d2 100644 --- a/Storage/KT_Storage/DataAccess/StorageManager.py +++ b/Storage/KT_Storage/DataAccess/StorageManager.py @@ -3,9 +3,9 @@ import os import aiofiles import shutil -from Crypto.Cipher import AES -from Crypto.Util.Padding import pad -import base6import +from Cryptodome.Cipher import AES +from Cryptodome.Util.Padding import pad +# import base6import URL_SERVER = 's3/KT_cloud/Storage/server' @@ -48,7 +48,7 @@ def get(self, bucket, key , version_id) -> Dict[str, Any]: file_path = os.path.join(self.server_path, bucket, versioned_file_name) if not os.path.exists(file_path): - return {'error': 'File not found'} + raise FileNotFoundError(f"File '{key}' with version '{version_id}' not found in bucket '{bucket}'.") if os.path.isdir(file_path): # If the object is a directory, return its metadata and list of contents @@ -76,7 +76,7 @@ def get(self, bucket, key , version_id) -> Dict[str, Any]: def delete_by_name(self, bucket_name, version_id, key) -> None: """Delete a specified file or directory by name in a bucket and version.""" file_name, file_extension = os.path.splitext(key) - file_name_path = f"{file_name}{file_extension}" + file_name_path = f"{file_name}.v{version_id}{file_extension}" file_path = os.path.join(self.server_path, bucket_name, file_name_path) if os.path.exists(file_path): diff --git a/Storage/KT_Storage/DataAccess/VersionManager.py b/Storage/KT_Storage/DataAccess/VersionManager.py index dfccf931..d7aca55c 100644 --- a/Storage/KT_Storage/DataAccess/VersionManager.py +++ b/Storage/KT_Storage/DataAccess/VersionManager.py @@ -1,10 +1,16 @@ -from DataAccess import StorageManager +# from DataAccess import StorageManager from typing import Dict, Any import json import aiofiles import os -from Models.VesionModel import Version +import sys +import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '.'))) + +from ..Models.VesionModel import Version from .StorageManager import StorageManager class VersionManager: def __init__(self, metadata_file="s3 project/KT_Cloud/Storage/server/metadata.json"): diff --git a/Storage/KT_Storage/Models/VesionModel.py b/Storage/KT_Storage/Models/VesionModel.py index 6bbf2ad5..5cdb8b00 100644 --- a/Storage/KT_Storage/Models/VesionModel.py +++ b/Storage/KT_Storage/Models/VesionModel.py @@ -1,7 +1,7 @@ from datetime import datetime import hashlib -from AclModel import Acl -from Tag import Tag +from .AclModel import Acl +from .Tag import Tag # domain/versioning.py From a7c69c7234756cb11a66aa8804828cff40d62183 Mon Sep 17 00:00:00 2001 From: Tem-M Date: Mon, 16 Sep 2024 14:38:01 +0300 Subject: [PATCH 06/11] fix db_manager and object_manager --- DB/NEW_KT_DB/DataAccess/DBManager.py | 32 +++++-- .../DataAccess/DBSubnetGroupManager.py | 10 +-- DB/NEW_KT_DB/DataAccess/ObjectManager.py | 83 ++++++++++--------- DB/NEW_KT_DB/Models/DBSubnetGroupModel.py | 1 + .../Service/Classes/DBSubnetGroupService.py | 3 +- 5 files changed, 76 insertions(+), 53 deletions(-) diff --git a/DB/NEW_KT_DB/DataAccess/DBManager.py b/DB/NEW_KT_DB/DataAccess/DBManager.py index c1d7580f..08daddf0 100644 --- a/DB/NEW_KT_DB/DataAccess/DBManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBManager.py @@ -67,7 +67,7 @@ def update_records_in_table(self, table_name: str, updates: Dict[str, Any], crit # add documentation here set_clause = ', '.join([f'{k} = ?' for k in updates.keys()]) - values = list(updates.values()) + values = tuple(updates.values()) update_statement = f''' UPDATE {table_name} @@ -90,7 +90,25 @@ def delete_data_from_table(self, table_name: str, criteria: str) -> None: self.execute_query_without_results(delete_statement) - + # Tem-M + def get_columns_from_table(self, table_name): + '''Get the columns from the specified table.''' + try: + get_columns_query = f"""PRAGMA table_info({table_name});""" + cols = self.execute_query_with_multiple_results(get_columns_query) + return [col[1] for col in cols] + except Exception as e: + print(f"Error occurred while fetching columns from table {table_name}: {e}") + return [] + + def get_all_data_from_table(self, table_name): + try: + get_all_data_query = f"""SELECT * FROM {table_name}""" + return self.execute_query_with_multiple_results(get_all_data_query) + except Exception as e: + print(f"Error occurred while fetching data from table {table_name}: {e}") + return [] + # rachel-8511, Riki7649255 def select_and_return_records_from_table(self, table_name: str, columns: List[str] = ['*'], criteria: Optional[str] = None) -> Dict[int, Dict[str, Any]]: '''Select records from the specified table based on criteria. @@ -101,18 +119,18 @@ def select_and_return_records_from_table(self, table_name: str, columns: List[st Returns: Dict[int, Dict[str, Any]]: A dictionary where keys are object_ids and values are metadata. ''' - - if columns == ['*']: - columns = [res[1] for res in self.connection.execute(f'PRAGMA table_info({table_name});').fetchall()] + cols = columns + if cols == ['*']: + cols = self.get_columns_from_table(table_name) - columns_clause = ', '.join(columns) + columns_clause = ', '.join(cols) query = f'SELECT {columns_clause} FROM {table_name}' if criteria: query += f' WHERE {criteria};' try: results = self.execute_query_with_multiple_results(query) - return {result[0]: dict(zip(columns, result[1:])) for result in results} + return {result[0]: dict(zip(cols if columns != ['*'] else cols[1:], result[1:])) for result in results} except OperationalError as e: raise Exception(f'Error selecting from {table_name}: {e}') except TypeError as e: diff --git a/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py b/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py index d601ff4f..68c8af52 100644 --- a/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py @@ -10,13 +10,13 @@ class DBSubnetGroupManager: def __init__(self, object_manager): self.object_manager = object_manager - self.object_manager.create_management_table(DBSubnetGroup.table_name, DBSubnetGroup.table_structure) + self.object_manager._create_management_table(DBSubnetGroup.table_name, DBSubnetGroup.table_structure) def create(self, subnet_group: DBSubnetGroup): - self.object_manager.insert_object_to_management_table(DBSubnetGroup.table_name, subnet_group.to_sql()) + self.object_manager.save_in_memory(DBSubnetGroup.table_name, subnet_group) def get(self, name: str): - data = self.object_manager.get_object_from_management_table(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, name) + data = self.object_manager.get_from_memory_by_id(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, name) if data: data_mapping = {'db_subnet_group_name':name} for key, value in data[name].items(): @@ -27,7 +27,7 @@ def get(self, name: str): def delete(self, name: str): - self.object_manager.delete_object_from_management_table(DBSubnetGroup.table_name, f"{DBSubnetGroup.pk_column}='{name}'") + self.object_manager.delete_from_memory_by_id(DBSubnetGroup.pk_column, name, DBSubnetGroup.table_name) def describe(self, name: str): return self.get(name).to_dict() @@ -35,4 +35,4 @@ def describe(self, name: str): def modify(self, subnet_group: DBSubnetGroup): updates = subnet_group.to_dict() del updates['db_subnet_group_name'] - self.object_manager.update_object_in_management_table_by_criteria(table_name = DBSubnetGroup.table_name, updates = updates, criteria = f"{DBSubnetGroup.pk_column}='{subnet_group.db_subnet_group_name}'") \ No newline at end of file + self.object_manager.update_in_memory_by_id(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, updates, subnet_group.db_subnet_group_name) diff --git a/DB/NEW_KT_DB/DataAccess/ObjectManager.py b/DB/NEW_KT_DB/DataAccess/ObjectManager.py index 46400dfe..ec3b2ece 100644 --- a/DB/NEW_KT_DB/DataAccess/ObjectManager.py +++ b/DB/NEW_KT_DB/DataAccess/ObjectManager.py @@ -12,28 +12,39 @@ def __init__(self, db_file: str): # for internal use only: # Riki7649255 based on rachel-8511 - def create_management_table(self, table_name, table_structure='object_id INTEGER PRIMARY KEY AUTOINCREMENT,type_object TEXT NOT NULL,metadata TEXT NOT NULL'): + def _create_management_table(self, table_name, table_structure='object_id INTEGER PRIMARY KEY AUTOINCREMENT,type_object TEXT NOT NULL,metadata TEXT NOT NULL'): + """ + creates a management table with the name and the structure you specify + make sure to keep track of the table name you send here - you will use it whenever you want to access the table + you created - this function should only be called from within the specific manager you created (i.e. DBInstanceManager) + """ self.db_manager.create_table(table_name, table_structure) - # Riki7649255 based on saraNoigershel - def insert_object_to_management_table(self, table_name, object): - self.db_manager.insert_data_into_table(table_name, object) - - - + # Riki7649255 based on saraNoigershel, Tem-M + def _insert_object_to_management_table(self, table_name, object): + """ + inserts an object to the management table you specified, the object should be sent as is! not converted to a tuple or dictionary! + if the table does not exist, the function will abort and raise an error + the table should be created within the __init__ function of the manager you created (i.e. DBInstanceManager) + """ + columns = self.db_manager.get_columns_from_table(table_name) + values = tuple([str(getattr(object, column)) for column in columns]) + self.db_manager.insert_data_into_table(table_name, columns, values) + + # Malki1844 + def _get_all_data_from_table(self, table_name): + self.db_manager.get_all_data_from_table(table_name) # Riki7649255 based on rachel-8511 - def update_object_in_management_table_by_criteria(self, table_name, updates, criteria): + def _update_object_in_management_table_by_criteria(self, table_name, updates, criteria): + updates = {k: str(v) for k, v in updates.items()} self.db_manager.update_records_in_table(table_name, updates, criteria) - def update_object_in_management_table_by_id(self, pk_col, table_name, object_id, updates): - self.db_manager.update_records_in_table(table_name, updates, f'{pk_col} = \'{object_id}\'') - # rachel-8511, Riki7649255 - def get_object_from_management_table(self, pk_col, table_name, object_id: int) -> Dict[str, Any]: + def _get_object_from_management_table(self, pk_col, table_name, object_id: int) -> Dict[str, Any]: '''Retrieve an object from the database.''' result = self.db_manager.select_and_return_records_from_table(table_name=table_name, criteria=f'{pk_col} = \'{object_id}\'') if result: @@ -41,9 +52,9 @@ def get_object_from_management_table(self, pk_col, table_name, object_id: int) - else: raise FileNotFoundError(f'Object with ID {object_id} not found.') - def get_objects_from_management_table_by_criteria(self, object_id: int, columns = ["*"], criteria:Optional[str] = None) -> Dict: + def _get_objects_from_management_table_by_criteria(self, table_name, columns = ["*"], criteria:Optional[str] = None) -> Dict: '''Retrieve an object from the database.''' - result = self.db_manager.select_and_return_records_from_table(self.table_name, columns, criteria) + result = self.db_manager.select_and_return_records_from_table(table_name, columns, criteria) if result: return result else: @@ -51,13 +62,13 @@ def get_objects_from_management_table_by_criteria(self, object_id: int, columns # rachel-8511, ShaniStrassProg, Riki7649255 - def delete_object_from_management_table(self, table_name, criteria) -> None: + def _delete_object_from_management_table(self, table_name, criteria) -> None: '''Delete an object from the database.''' self.db_manager.delete_data_from_table(table_name, criteria) - def delete_object_from_management_table_by_id(self, table_name, object_id) -> None: + def _delete_object_from_management_table_by_id(self, pk_col, table_name, object_id) -> None: '''Delete an object from the database.''' - self.db_manager.delete_data_from_table(table_name, criteria= f'object_id = {object_id}') + self.db_manager.delete_data_from_table(table_name, criteria= f'{pk_col} = \'{object_id}\'') # rachel-8511, ShaniStrassProg is it needed? @@ -72,11 +83,11 @@ def delete_object_from_management_table_by_id(self, table_name, object_id) -> No # return self.db_manager.describe(self.table_name) - def convert_object_name_to_management_table_name(object_name): + def _convert_object_name_to_management_table_name(object_name): return f'mng_{object_name}s' - def is_management_table_exist(self, table_name): + def _is_management_table_exist(self, table_name): # Check if table exists by querying the sqlite_master table query = f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}'" return self.db_manager.execute_query_with_single_result(query) @@ -88,36 +99,30 @@ def save_in_memory(self, table_name, object): # insert object info into management table mng_{object_name}s # for exmple: object db_instance will be saved in table mng_db_instances - self.insert_object_to_management_table(table_name, object) + self._insert_object_to_management_table(table_name, object) - def delete_from_memory(self, pk_col, pk_val, table_name:str, criteria='default'): + def delete_from_memory_by_id(self, pk_col, pk_val, table_name:str): # pk_val is the object id # if criteria not sent- use PK for deletion - if criteria == 'default': - criteria = f'{pk_col} = \'{pk_val}\'' - + criteria = f'{pk_col} = \'{pk_val}\'' self.db_manager.delete_data_from_table(table_name, criteria) - - def update_in_memory(self, pktable_name, updates, criteria='default', object_id:Optional[str] = None): - - # if criteria not sent- use PK for deletion - if criteria == 'default': - if not object_id: - raise ValueError('must be or criteria or object id') - criteria = f'object_id = {object_id}' - - + def update_in_memory_by_criteria(self,table_name:str, updates:Dict, criteria): + self._update_object_in_management_table_by_criteria(table_name, updates, criteria) + + def update_in_memory_by_id(self, pk_col, table_name, updates, object_id:Optional[str]): + if not object_id: + raise ValueError('must be or criteria or object id') + criteria = f'{pk_col} = \'{object_id}\'' + self.update_in_memory_by_criteria(table_name, updates, criteria) - def get_from_memory(self, object_name, columns = ["*"], object_id = None, criteria = None): + def get_from_memory_by_id(self, pk_col, table_name, object_id, columns = ["*"]): """get records from memory by criteria or id""" - table_name = self.convert_object_name_to_management_table_name(object_name) - if object_id: - criteria = f'object_id = {object_id}' - self.get_objects_from_management_table_by_criteria(table_name, columns, criteria) + criteria = f'{pk_col} = \'{object_id}\'' + return self._get_objects_from_management_table_by_criteria(table_name, columns, criteria) def convert_object_attributes_to_dictionary(**kwargs): diff --git a/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py b/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py index ccb0bff7..7ed37365 100644 --- a/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py +++ b/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py @@ -23,6 +23,7 @@ class DBSubnetGroup: def __init__(self, **kwargs): try: + print(kwargs) self.db_subnet_group_name = kwargs['db_subnet_group_name'] self.db_subnet_group_description = kwargs['db_subnet_group_description'] self.vpc_id = kwargs['vpc_id'] diff --git a/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py b/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py index 0c42b29e..8fc33513 100644 --- a/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py +++ b/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py @@ -57,8 +57,7 @@ def get_db_subnet_group(self, db_subnet_group_name: str) -> DBSubnetGroup: def modify_db_subnet_group(self, db_subnet_group_name: str, updates: Dict[str, Any]) -> DBSubnetGroup: if not db_subnet_group_name: raise ValueError('Missing required argument db_subnet_group_name') - - + if updates.get('description') and not is_length_in_range(updates['description'], 1, 255): raise ValueError("invalid length for subnet group description: " + len(updates['description'])) From ac36839912017dc6b7df483de92b45789de620aa Mon Sep 17 00:00:00 2001 From: Tem-M Date: Mon, 16 Sep 2024 15:02:39 +0300 Subject: [PATCH 07/11] commit --- DB/NEW_KT_DB/DataAccess/ObjectManager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DB/NEW_KT_DB/DataAccess/ObjectManager.py b/DB/NEW_KT_DB/DataAccess/ObjectManager.py index ec3b2ece..ce24835f 100644 --- a/DB/NEW_KT_DB/DataAccess/ObjectManager.py +++ b/DB/NEW_KT_DB/DataAccess/ObjectManager.py @@ -83,7 +83,7 @@ def _delete_object_from_management_table_by_id(self, pk_col, table_name, object_ # return self.db_manager.describe(self.table_name) - def _convert_object_name_to_management_table_name(object_name): + def _convert_object_name_to_management_table_name(self, object_name): return f'mng_{object_name}s' From dad5c89ffed7a15a1cee199f97c6e432ed110c5d Mon Sep 17 00:00:00 2001 From: tamar koledetzky Date: Wed, 18 Sep 2024 17:36:49 +0300 Subject: [PATCH 08/11] functions and tests for DBClusterParameterGroup --- .../DBClusterParameterGroupController.py | 6 +- .../DBClusterParameterGroupManager.py | 36 +-- DB/NEW_KT_DB/DataAccess/ObjectManager.py | 5 +- .../Models/DBClusterParameterGroupModel.py | 47 ++-- .../Classes/DBClusterParameterGroupService.py | 92 ++++--- .../Test/DBClusterParameterGroupTests.py | 246 +++++++++++++----- DB/NEW_KT_DB/Test/GeneralTests.py | 34 +-- 7 files changed, 297 insertions(+), 169 deletions(-) diff --git a/DB/NEW_KT_DB/Controller/DBClusterParameterGroupController.py b/DB/NEW_KT_DB/Controller/DBClusterParameterGroupController.py index 1b44ff11..cac47782 100644 --- a/DB/NEW_KT_DB/Controller/DBClusterParameterGroupController.py +++ b/DB/NEW_KT_DB/Controller/DBClusterParameterGroupController.py @@ -6,13 +6,13 @@ def __init__(self, service: DBClusterParameterGroupService): self.service = service def create_db_cluster_parameter_group(self, group_name: str, group_family: str, description: Optional[str]=None): - self.service.create(group_name, group_family, description) + return self.service.create(group_name, group_family, description) def delete_db_cluste_parameter_group(self, group_name: str): self.service.delete(group_name) - def describe_db_cluste_parameter_group(self, group_name: str) -> Dict: - return self.service.describe_group('DBClusterParameterGroup', group_name) + def describe_db_cluste_parameter_group(self, group_name: str = None, max_records: int = 100, marker: str = None) -> Dict: + return self.service.describe_group('DBClusterParameterGroup', group_name, max_records, marker) def modify_db_cluste_parameter_group(self, group_name: str, parameters: list[Dict[str, any]]): self.service.modify('DBClusterParameterGroup', group_name, parameters) \ No newline at end of file diff --git a/DB/NEW_KT_DB/DataAccess/DBClusterParameterGroupManager.py b/DB/NEW_KT_DB/DataAccess/DBClusterParameterGroupManager.py index 2bdb85af..a0ce1b93 100644 --- a/DB/NEW_KT_DB/DataAccess/DBClusterParameterGroupManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBClusterParameterGroupManager.py @@ -2,38 +2,38 @@ import json import sqlite3 from NEW_KT_DB.DataAccess.ObjectManager import ObjectManager +from NEW_KT_DB.Models.DBClusterParameterGroupModel import DBClusterParameterGroup class DBClusterParameterGroupManager: def __init__(self, db_file: str): '''Initialize ObjectManager with the database connection.''' self.object_manager = ObjectManager(db_file) - # self.table_name ='cluster_managment' - # self.create_table() + self.object_manager.create_management_table( + DBClusterParameterGroup.get_object_name(), DBClusterParameterGroup.table_schema, 'TEXT') - def createInMemoryDBCluster(self, data, id): - self.object_manager.save_in_memory(self.__class__.__name__[:-len("Manager")], data, id) + def createInMemoryDBCluster(self, data): + self.object_manager.save_in_memory(self.__class__.__name__[:-len("Manager")], data) - def deleteInMemoryDBCluster(self, id): - self.object_manager.delete_from_memory(self.__class__.__name__[:-len("Manager")], object_id= id) + def deleteInMemoryDBCluster(self, group_name): + self.object_manager.delete_from_memory_by_pk(self.__class__.__name__[:-len("Manager")], pk_column=DBClusterParameterGroup.pk_column, pk_value=group_name) - - def describeDBCluster(self, id): - self.object_manager.get_from_memory(self.__class__.__name__[:-len("Manager")],object_id= id) - - - def modifyDBCluster(self, id, data): - self.object_manager.update_in_memory(self.__class__.__name__[:-len("Manager")], data, object_id= id) + def modifyDBCluster(self, group_name, data): + self.object_manager.update_in_memory(self.__class__.__name__[:-len("Manager")], updates=data, criteria=f'{DBClusterParameterGroup.pk_column} = "{group_name}"') - def get(self, id): - return self.object_manager.get_from_memory(self.__class__.__name__[:-len("Manager")],object_id= id) + def get(self, group_name): + return self.object_manager.get_from_memory(self.__class__.__name__[:-len("Manager")], columns='*', criteria=f'{DBClusterParameterGroup.pk_column} = "{group_name}"') def get_all_groups(self): - return self.object_manager.get_from_memory(self.__class__.__name__[:-len("Manager")]) + return self.object_manager.get_all_objects_from_memory(self.__class__.__name__[:-len("Manager")]) + + def is_identifier_exist(self, group_name): + result= self.object_manager.get_from_memory(self.__class__.__name__[:-len("Manager")], columns='*', criteria=f'{DBClusterParameterGroup.pk_column} = "{group_name}"') + if result !=[]: + return True + return False - def is_identifier_exist(self, id): - return self.object_manager.is_object_exist(self.__class__.__name__[:-len("Manager")],object_id= id) diff --git a/DB/NEW_KT_DB/DataAccess/ObjectManager.py b/DB/NEW_KT_DB/DataAccess/ObjectManager.py index b75b2373..a79eec5e 100644 --- a/DB/NEW_KT_DB/DataAccess/ObjectManager.py +++ b/DB/NEW_KT_DB/DataAccess/ObjectManager.py @@ -1,7 +1,10 @@ from typing import Dict, Any, Optional import json import sqlite3 -from DBManager import DBManager +import os +import sys +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../..'))) +from NEW_KT_DB.DataAccess.DBManager import DBManager class ObjectManager: def __init__(self, db_file: str): diff --git a/DB/NEW_KT_DB/Models/DBClusterParameterGroupModel.py b/DB/NEW_KT_DB/Models/DBClusterParameterGroupModel.py index 596bd6a7..c8389164 100644 --- a/DB/NEW_KT_DB/Models/DBClusterParameterGroupModel.py +++ b/DB/NEW_KT_DB/Models/DBClusterParameterGroupModel.py @@ -4,6 +4,14 @@ class DBClusterParameterGroup: + + pk_column = 'group_name' + table_schema = """ + group_name TEXT PRIMARY KEY, + group_family TEXT, + description TEXT, + parameters TEXT + """ def __init__(self, group_name: str, group_family: str, description: Optional[str] = None, tags: Optional[List[str]] = None, pk_column: str='DBClusterParameterGroupName', pk_value: str= None ): self.group_name = group_name self.group_family = group_family @@ -25,8 +33,6 @@ def load_default_parameters(self): parameters.append(Parameter('backup_retention_period', 7).to_dict()) parameters.append(Parameter('preferred_backup_window', '03:00-03:30').to_dict()) parameters.append(Parameter('preferred_maintenance_window', 'Mon:00:00-Mon:00:30').to_dict()) - # for p in parameters: - # p.save_to_db(conn) return parameters @@ -36,38 +42,41 @@ def to_dict(self) -> Dict: group_family= self.group_family, description= self.description, parameters= self.parameters, - tags= self.tags, - pk_column=self.pk_column, - pk_value=self.pk_value + # tags= self.tags, + # pk_column=self.pk_column, + # pk_value=self.pk_value ) + @staticmethod + def get_object_name(): + return DBClusterParameterGroup.__name__ from typing import Optional, List, Dict class Parameter: - def __init__(self, parameter_name: str, parameter_value: str, description: str = '', source: str = 'engine-default', apply_type: str = '', data_type: str = '', allowed_values: str = '', is_modifiable: bool = True, minimum_engine_version: str = '', apply_method: str = '', supported_engine_modes: Optional[List[str]] = None): + def __init__(self, parameter_name: str, parameter_value: str, description: str = '', source: str = 'engine-default', apply_method: str = '', is_modifiable: bool = True):#, apply_type: str = '', data_type: str = '', allowed_values: str = '', is_modifiable: bool = True, minimum_engine_version: str = '', apply_method: str = '', supported_engine_modes: Optional[List[str]] = None): self.parameter_name = parameter_name self.parameter_value = parameter_value self.description = description - self.source = source - self.apply_type = apply_type - self.data_type = data_type - self.allowed_values = allowed_values + # self.source = source + # self.apply_type = apply_type + # self.data_type = data_type + # self.allowed_values = allowed_values self.is_modifiable = is_modifiable - self.minimum_engine_version = minimum_engine_version + # self.minimum_engine_version = minimum_engine_version self.apply_method = apply_method - self.supported_engine_modes = supported_engine_modes + # self.supported_engine_modes = supported_engine_modes def to_dict(self) -> Dict: return ObjectManager.convert_object_attributes_to_dictionary( parameter_name= self.parameter_name, parameter_value= self.parameter_value, description= self.description, - source= self.source, - apply_type=self.apply_type, - data_type= self.data_type, - allowed_values=self.allowed_values, + # source= self.source, + # apply_type=self.apply_type, + # data_type= self.data_type, + # allowed_values=self.allowed_values, is_modifiable= self.is_modifiable, - minimum_engine_version= self.minimum_engine_version, - apply_method= self.apply_method, - supported_engine_modes= self.supported_engine_modes + # minimum_engine_version= self.minimum_engine_version, + apply_method= self.apply_method + # supported_engine_modes= self.supported_engine_modes ) diff --git a/DB/NEW_KT_DB/Service/Classes/DBClusterParameterGroupService.py b/DB/NEW_KT_DB/Service/Classes/DBClusterParameterGroupService.py index 32986566..d8a843bc 100644 --- a/DB/NEW_KT_DB/Service/Classes/DBClusterParameterGroupService.py +++ b/DB/NEW_KT_DB/Service/Classes/DBClusterParameterGroupService.py @@ -7,7 +7,7 @@ from NEW_KT_DB.Service.Abc.DBO import DBO # from DB.KT_DB.Models.ParameterGroupModel import ParameterGroupModel from NEW_KT_DB.Validation.GeneralValidations import is_valid_user_group_name, is_valid -from NEW_KT_DB.Models import DBClusterParameterGroupModel +from NEW_KT_DB.Models.DBClusterParameterGroupModel import DBClusterParameterGroup from NEW_KT_DB.DataAccess import DBClusterManager#, DBClusterParameterGroupManager from NEW_KT_DB.DataAccess import DBClusterParameterGroupManager sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..'))) @@ -17,6 +17,12 @@ class DBClusterParameterGroupService(DBO): """ Service class for managing generic parameter groups. """ + column_index_mapping = { + 'group_name': 0, + 'group_family': 1, + 'description': 2, + 'parameters': 3 + } def __init__(self, dal:DBClusterParameterGroupManager, dal_cluster: DBClusterManager, storage_manager: StorageManager): """ @@ -43,15 +49,21 @@ def create(self, group_name: str, group_family: str, description: Optional[str] raise ValueError(f"group_name {group_name} is not valid") if self.dal.is_identifier_exist(group_name): raise ValueError(f"ParameterGroup with NAME '{group_name}' already exists.") - group = DBClusterParameterGroupModel(group_name, group_family, description) + group = DBClusterParameterGroup(group_name, group_family, description) parameter_group_dict=group.to_dict() - self.dal.create(parameter_group_dict, group_name) + data_tuple = ( + parameter_group_dict['group_name'], + parameter_group_dict['group_family'], + parameter_group_dict.get('description', None), + json.dumps(parameter_group_dict['parameters']) + ) + self.dal.createInMemoryDBCluster(data_tuple) file_name=f'db_cluster_parameter_groups/db_cluster_parameter_group_{group_name}.json' + self.storage_manager.create_directory('db_cluster_parameter_groups') self.storage_manager.create_file(file_name, json.dumps(parameter_group_dict)) - # with open(file_name, 'w') as json_file: - # json.dump(parameter_group_dict, json_file, indent=4) print(f"Creating parameter group '{group_name}' in family '{group_family}' with description '{description}'") - return self.describe(group_name) + group_tuple=self.get(group_name) + return self.describe(group_tuple) def delete(self, group_name: str): """ @@ -64,18 +76,15 @@ def delete(self, group_name: str): raise ValueError("You can't delete a default parameter group") if not self.dal.is_identifier_exist(group_name): raise ValueError(f"Parameter Group '{group_name}' does not exist.") - data = self.dal_cluster.get_all_clusters() - clusters = list(data.values()) + clusters = self.dal_cluster.get_all_clusters() for c in clusters: - if c['group_name'] == group_name: + if c[6] == group_name: raise ValueError("Can't delete parameter group associated with any DB clusters") - self.dal.delete(group_name) + self.dal.deleteInMemoryDBCluster(group_name) file_name = f'db_cluster_parameter_groups/db_cluster_parameter_group_{group_name}.json' self.storage_manager.delete_file(file_name) - # if os.path.exists(file_name): - # os.remove(file_name) print(f"Deleting parameter group '{group_name}'") - + def describe_group(self, title: str, parameter_group_name: str = None, max_records: int = 100, marker: str = None) -> Dict: """ Describe a specific parameter group. @@ -88,25 +97,24 @@ def describe_group(self, title: str, parameter_group_name: str = None, max_recor """ parameter_groups_local = [] if parameter_group_name is not None: - data = self.dal.get(parameter_group_name) + data = self.get(parameter_group_name) parameter_groups_local.append(self.describe(data)) else: - result = self.dal.get_all_groups() - parameter_groups = list(result.values()) + parameter_groups = self.dal.get_all_groups() count = 0 for p in parameter_groups: - if p['group_name'] == marker or marker is None: + if p[DBClusterParameterGroupService.column_index_mapping['group_name']] == marker or marker is None: marker = None count += 1 if count <= max_records: parameter_groups_local.append(self.describe(p)) else: - marker = p['group_name'] - if marker is None: + marker = p[DBClusterParameterGroupService.column_index_mapping['group_name']] + if marker is None: return {title: parameter_groups_local} return {'Marker': marker, title: parameter_groups_local} - def camel_to_snake_case(name: str) -> str: + def camel_to_snake_case(self, name: str) -> str: """ Convert a CamelCase string to snake_case. @@ -133,30 +141,33 @@ def modify(self, title: str, group_name: str, parameters: Optional[list[Dict[str :param parameters: A list of dictionaries with updates to apply to the parameter group. :return: A dictionary containing details about the modified parameter group. """ - update = [] - parameter_group = self.dal.get(group_name) + parameter_group = self.get(group_name) + parameters_in_parameter_group=parameter_group[DBClusterParameterGroupService.column_index_mapping['parameters']] + parameters_in_parameter_group=json.loads(parameters_in_parameter_group) + # print(f"parameter_group{parameter_group}") + for new_parameter in parameters: is_valid(new_parameter['IsModifiable'], [True, False], 'IsModifiable') is_valid(new_parameter['ApplyMethod'], ['immediate', 'pending-reboot'], 'ApplyMethod') - for old_parameter in parameter_group['parameters']: + for idx, old_parameter in enumerate(parameters_in_parameter_group): if new_parameter['ParameterName'] == old_parameter['parameter_name']: - if old_parameter['IsModifiable'] == False: + if old_parameter['is_modifiable'] == False: raise ValueError(f"You can't modify the parameter {old_parameter['parameter_name']}") new_parameter_updates = self.convert_dict_keys_to_snake_case(new_parameter) - update.append(new_parameter_updates) - else: - update.append(old_parameter) - new_data = {'parameters': update} - updated_data = {**parameter_group, **new_data} - self.dal.update(group_name, updated_data) + updated_parameter = {**old_parameter, **new_parameter_updates} + parameters_in_parameter_group[idx] = updated_parameter + self.dal.modifyDBCluster(group_name, f"parameters='{json.dumps(parameters_in_parameter_group)}'") file_name=f'db_cluster_parameter_groups/db_cluster_parameter_group_{group_name}.json' - self.storage_manager.create_file(file_name, json.dump(updated_data)) - # with open(file_name, 'w') as json_file: - # json.dump(updated_data, json_file, indent=4) + group_family=parameter_group[DBClusterParameterGroupService.column_index_mapping['group_family']] + description=parameter_group[DBClusterParameterGroupService.column_index_mapping['description']] + group = DBClusterParameterGroup(group_name, group_family, description) + parameter_group_dict=group.to_dict() + parameter_group_dict['parameters']=parameters_in_parameter_group + self.storage_manager.write_to_file(file_name, json.dumps(parameter_group_dict)) return {title: group_name} # @abstractmethod - def describe(self, data: Dict) -> Dict: + def describe(self, data: tuple) -> Dict: """ Abstract method to describe a parameter group. @@ -166,12 +177,15 @@ def describe(self, data: Dict) -> Dict: :return: A dictionary containing the description of the parameter group. """ describe = { - 'DBClusterParameterGroupName': data['group_name'], - 'DBParameterGroupFamily': data['group_family'], - 'Description': data['description'], - 'DBClusterParameterGroupArn': f'arn:aws:rds:region:account:dbcluster-parameter_group/{data["group_name"]}' + 'DBClusterParameterGroupName': data[DBClusterParameterGroupService.column_index_mapping['group_name']], + 'DBParameterGroupFamily': data[DBClusterParameterGroupService.column_index_mapping['group_family']], + 'Description': data[DBClusterParameterGroupService.column_index_mapping['description']], + 'DBClusterParameterGroupArn': f'arn:aws:rds:region:account:dbcluster-parameter_group/{data[DBClusterParameterGroupService.column_index_mapping["group_name"]]}' } return describe def get(self, group_name: str)->Dict: - return self.dal.get(group_name) \ No newline at end of file + result= self.dal.get(group_name) + if result == []: + raise ValueError(f"Parameter Group '{group_name}' does not exist.") + return result[0] \ No newline at end of file diff --git a/DB/NEW_KT_DB/Test/DBClusterParameterGroupTests.py b/DB/NEW_KT_DB/Test/DBClusterParameterGroupTests.py index c8be1d93..86d332ac 100644 --- a/DB/NEW_KT_DB/Test/DBClusterParameterGroupTests.py +++ b/DB/NEW_KT_DB/Test/DBClusterParameterGroupTests.py @@ -2,7 +2,8 @@ import os import sys import pytest - +from unittest.mock import Mock, patch +from unittest.mock import MagicMock sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) from NEW_KT_DB.DataAccess.DBClusterManager import DBClusterManager from NEW_KT_DB.Controller.DBClusterParameterGroupController import DBClusterParameterGroupController @@ -10,18 +11,23 @@ from NEW_KT_DB.DataAccess.DBClusterParameterGroupManager import DBClusterParameterGroupManager sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..'))) from Storage.NEW_KT_Storage.DataAccess.StorageManager import StorageManager +from GeneralTests import * @pytest.fixture def parameter_group_manager(): return DBClusterParameterGroupManager(':memory:') -@pytest.fixture -def cluster_manager(): - return DBClusterManager(':memory:') +# @pytest.fixture +# def cluster_manager(): +# return DBClusterManager(':memory:') @pytest.fixture -def storage_manager(): - return StorageManager('test') +def cluster_manager(): + # Create a mock for DBClusterManager and its method get_all_clusters + mock_cluster_manager = Mock(spec=DBClusterManager) + # Set the return value of get_all_clusters + mock_cluster_manager.get_all_clusters.return_value = {} + return mock_cluster_manager @pytest.fixture def parameter_group_service(parameter_group_manager, cluster_manager, storage_manager): @@ -35,27 +41,31 @@ def parameter_group_controller(parameter_group_service): def create_parameter_group(controller, group_name, group_family, description): return controller.create_db_cluster_parameter_group(group_name, group_family, description) -# Generic function to check if a file exists -def assert_file_exists(file_name): - assert os.path.exists(file_name), f"Expected file {file_name} was not created." - -# Generic function to delete a file -def delete_file_if_exists(file_name): - if os.path.exists(file_name): - os.remove(file_name) - -# Generic function to load JSON file and assert its content -def assert_json_content(file_name, expected_data): - with open(file_name, 'r') as json_file: - data = json.load(json_file) - for key, value in expected_data.items(): - assert data[key] == value, f"Expected {key} to be {value}, but got {data[key]}" - # Generic function for file name def generate_file_name_for_group (group_name): return f'db_cluster_parameter_groups/db_cluster_parameter_group_{group_name}.json' -def test_create_parameter_group(parameter_group_controller): +# Generic function to assert the parameter group's details +def assert_parameter_group_details(result, index, expected_group_name, expected_family, expected_description): + """ + Assert the details of a specific DBClusterParameterGroup in the result. + + :param result: The result dictionary returned from the describe_db_cluste_parameter_group function. + :param index: The index of the parameter group in the result list to check. + :param expected_group_name: The expected DBClusterParameterGroupName value. + :param expected_family: The expected DBParameterGroupFamily value. + :param expected_description: The expected Description value. + """ + parameter_group = result['DBClusterParameterGroup'][index] + + assert parameter_group['DBClusterParameterGroupName'] == expected_group_name, \ + f"Expected DBClusterParameterGroupName to be '{expected_group_name}' but got '{parameter_group['DBClusterParameterGroupName']}'" + assert parameter_group['DBParameterGroupFamily'] == expected_family, \ + f"Expected DBParameterGroupFamily to be '{expected_family}' but got '{parameter_group['DBParameterGroupFamily']}'" + assert parameter_group['Description'] == expected_description, \ + f"Expected Description to be '{expected_description}' but got '{parameter_group['Description']}'" + +def test_create_parameter_group(parameter_group_controller, storage_manager): group_name = "TestGroup" group_family = "TestFamily" description = "Test Description" @@ -66,16 +76,17 @@ def test_create_parameter_group(parameter_group_controller): assert result['DBClusterParameterGroupName'] == group_name assert result['DBParameterGroupFamily'] == group_family assert result['Description'] == description - + full_path = os.path.abspath(file_name) + print(f"Full path of the file: {full_path}") # Check if the correct file was created - assert_file_exists(file_name) + assert_file_exists(storage_manager, file_name) # Check if the file content matches the expected result expected_data = {'group_name': group_name, 'group_family': group_family, 'description': description} - assert_json_content(file_name, expected_data) + assert_json_content(storage_manager, file_name, expected_data) # Cleanup - delete_file_if_exists(file_name) + delete_file_if_exists(storage_manager, file_name) def test_create_existing_parameter_group(parameter_group_controller): group_name = "TestGroup" @@ -95,7 +106,23 @@ def test_create_parameter_group_with_invalid_name(parameter_group_controller): with pytest.raises(ValueError, match=f"group_name {invalid_group_name} is not valid"): create_parameter_group(parameter_group_controller, invalid_group_name, "ValidFamily", "Valid Description") -def test_delete_parameter_group(parameter_group_controller): +# def test_delete_parameter_group(parameter_group_controller): +# group_name = "TestGroup" +# file_name = generate_file_name_for_group(group_name) + +# # Create the parameter group +# create_parameter_group(parameter_group_controller, group_name, "TestFamily", "Test Description") + +# # Ensure the file exists before deletion +# assert_file_exists(file_name) + +# # Delete the parameter group +# parameter_group_controller.delete_db_cluste_parameter_group(group_name) + +# # Check if the file was deleted +# assert not os.path.exists(file_name), f"Expected file {file_name} was not deleted." + +def test_delete_parameter_group(parameter_group_controller, storage_manager): group_name = "TestGroup" file_name = generate_file_name_for_group(group_name) @@ -103,7 +130,7 @@ def test_delete_parameter_group(parameter_group_controller): create_parameter_group(parameter_group_controller, group_name, "TestFamily", "Test Description") # Ensure the file exists before deletion - assert_file_exists(file_name) + assert_file_exists(storage_manager, file_name) # Delete the parameter group parameter_group_controller.delete_db_cluste_parameter_group(group_name) @@ -111,31 +138,48 @@ def test_delete_parameter_group(parameter_group_controller): # Check if the file was deleted assert not os.path.exists(file_name), f"Expected file {file_name} was not deleted." -def test_delete_nonexistent_parameter_group(parameter_group_controller): - group_name = "NonExistentGroup" - - # Test if exception is raised when trying to delete a non-existent group - with pytest.raises(ValueError, match=f"Parameter Group '{group_name}' does not exist."): - parameter_group_controller.delete_db_cluste_parameter_group(group_name) - -def test_delete_parameter_group_with_associated_cluster(parameter_group_controller, cluster_manager): +def test_delete_parameter_group_with_associated_cluster(parameter_group_controller, cluster_manager, storage_manager): group_name = "TestGroup" file_name = generate_file_name_for_group(group_name) # Create the parameter group create_parameter_group(parameter_group_controller, group_name, "TestFamily", "Test Description") - # Associate the parameter group with a cluster - cluster_manager.create_cluster({"cluster_id": "TestCluster", "group_name": group_name}) + # Mock get_all_clusters to return a cluster associated with the parameter group + cluster_manager.get_all_clusters.return_value =[("","","","","","",group_name)] #{"TestCluster": {"group_name": group_name}} # Attempt to delete the parameter group, expect an exception due to association with cluster with pytest.raises(ValueError, match="Can't delete parameter group associated with any DB clusters"): parameter_group_controller.delete_db_cluste_parameter_group(group_name) # Cleanup - delete_file_if_exists(file_name) + delete_file_if_exists(storage_manager, file_name) -def test_delete_default_parameter_group(parameter_group_controller): +def test_delete_nonexistent_parameter_group(parameter_group_controller): + group_name = "NonExistentGroup" + + # Test if exception is raised when trying to delete a non-existent group + with pytest.raises(ValueError, match=f"Parameter Group '{group_name}' does not exist."): + parameter_group_controller.delete_db_cluste_parameter_group(group_name) + +# def test_delete_parameter_group_with_associated_cluster(parameter_group_controller, cluster_manager): +# group_name = "TestGroup" +# file_name = generate_file_name_for_group(group_name) + +# # Create the parameter group +# create_parameter_group(parameter_group_controller, group_name, "TestFamily", "Test Description") + +# # Associate the parameter group with a cluster +# cluster_manager.create_cluster({"cluster_id": "TestCluster", "group_name": group_name}) + +# # Attempt to delete the parameter group, expect an exception due to association with cluster +# with pytest.raises(ValueError, match="Can't delete parameter group associated with any DB clusters"): +# parameter_group_controller.delete_db_cluste_parameter_group(group_name) + +# # Cleanup +# delete_file_if_exists(file_name) + +def test_delete_default_parameter_group(parameter_group_controller, storage_manager): group_name = "default" file_name = generate_file_name_for_group(group_name) @@ -147,9 +191,9 @@ def test_delete_default_parameter_group(parameter_group_controller): parameter_group_controller.delete_db_cluste_parameter_group(group_name) # Cleanup - delete_file_if_exists(file_name) + delete_file_if_exists(storage_manager, file_name) -def test_modify_parameter_group(parameter_group_controller): +def test_modify_parameter_group(parameter_group_controller, storage_manager): group_name = "TestGroup" file_name = generate_file_name_for_group(group_name) @@ -158,26 +202,30 @@ def test_modify_parameter_group(parameter_group_controller): # Modify the parameter group with new parameters parameters = [ - {'ParameterName': 'backup_retention_period', 'ParameterValue': '14', 'IsModifiable': True, 'ApplyMethod': 'immediate'} + {'ParameterName': 'backup_retention_period', 'ParameterValue': 14, 'IsModifiable': True, 'ApplyMethod': 'immediate'} ] parameter_group_controller.modify_db_cluste_parameter_group(group_name, parameters) # Check if the modifications were applied - expected_parameters = {'parameters': [{'parameter_name': 'backup_retention_period', 'parameter_value': '14'}]} - assert_json_content(file_name, expected_parameters) + expected_parameters = {'parameters': [{'parameter_name': 'backup_retention_period', 'parameter_value': 14, 'description': '', + 'is_modifiable': True, 'apply_method': 'immediate'}, {'parameter_name': 'preferred_backup_window', 'parameter_value': '03:00-03:30', + 'description': '', 'is_modifiable': True, 'apply_method': ''}, {'parameter_name': 'preferred_maintenance_window', + 'parameter_value': 'Mon:00:00-Mon:00:30', 'description': '', 'is_modifiable': True, 'apply_method': ''}]} + + assert_json_content(storage_manager, file_name, expected_parameters) # Cleanup - delete_file_if_exists(file_name) + delete_file_if_exists(storage_manager, file_name) def test_modify_nonexistent_parameter_group(parameter_group_controller): group_name = "NonExistentGroup" - parameters = [{'ParameterName': 'backup_retention_period', 'ParameterValue': '14', 'IsModifiable': True, 'ApplyMethod': 'immediate'}] + parameters = [{'ParameterName': 'backup_retention_period', 'ParameterValue': 14, 'IsModifiable': True, 'ApplyMethod': 'immediate'}] # Test if exception is raised when trying to modify a non-existent group - with pytest.raises(KeyError, match=f"Parameter Group '{group_name}' does not exist."): + with pytest.raises(ValueError, match=f"Parameter Group '{group_name}' does not exist."): parameter_group_controller.modify_db_cluste_parameter_group(group_name, parameters) -def test_modify_non_modifiable_parameter(parameter_group_controller): +def test_modify_non_modifiable_parameter(parameter_group_controller, storage_manager): group_name = "TestGroup" file_name = generate_file_name_for_group(group_name) @@ -186,21 +234,21 @@ def test_modify_non_modifiable_parameter(parameter_group_controller): # Define a non-modifiable parameter parameters = [ - {'ParameterName': 'max_connections', 'ParameterValue': '100', 'IsModifiable': False, 'ApplyMethod': 'immediate'} + {'ParameterName': 'backup_retention_period', 'ParameterValue': 5, 'IsModifiable': False, 'ApplyMethod': 'immediate'} ] parameter_group_controller.modify_db_cluste_parameter_group( group_name, parameters) # Attempt to change a non-modifiable parameter, expect an exception - with pytest.raises(ValueError, match="You can't modify the parameter max_connections"): + with pytest.raises(ValueError, match="You can't modify the parameter backup_retention_period"): new_parameters = [ - {'ParameterName': 'max_connections', 'ParameterValue': '200', 'IsModifiable': False, 'ApplyMethod': 'immediate'} + {'ParameterName': 'backup_retention_period', 'ParameterValue': 14, 'IsModifiable': False, 'ApplyMethod': 'immediate'} ] parameter_group_controller.modify_db_cluste_parameter_group(group_name, new_parameters) # Cleanup - delete_file_if_exists(file_name) + delete_file_if_exists(storage_manager, file_name) -def test_modify_with_invalid_is_modifiable(parameter_group_controller): +def test_modify_with_invalid_is_modifiable(parameter_group_controller, storage_manager): group_name = "TestGroup" file_name = generate_file_name_for_group(group_name) @@ -217,9 +265,9 @@ def test_modify_with_invalid_is_modifiable(parameter_group_controller): parameter_group_controller.modify_db_cluste_parameter_group(group_name, invalid_parameters) # Cleanup - delete_file_if_exists(file_name) + delete_file_if_exists(storage_manager, file_name) -def test_modify_with_invalid_apply_method(parameter_group_controller): +def test_modify_with_invalid_apply_method(parameter_group_controller, storage_manager): group_name = "TestGroup" file_name = generate_file_name_for_group(group_name) @@ -234,29 +282,93 @@ def test_modify_with_invalid_apply_method(parameter_group_controller): parameter_group_controller.modify_db_cluste_parameter_group(group_name, invalid_parameters) # Cleanup - delete_file_if_exists(file_name) + delete_file_if_exists(storage_manager, file_name) -def test_describe_parameter_group(parameter_group_controller): +def test_describe_parameter_group(parameter_group_controller, storage_manager): group_name = "TestGroup" + family="TestFamily" + description="Test Description" file_name = generate_file_name_for_group(group_name) # Create a parameter group - create_parameter_group(parameter_group_controller, group_name, "TestFamily", "Test Description") - + create_parameter_group(parameter_group_controller, group_name, family, description) + # Describe the parameter group result = parameter_group_controller.describe_db_cluste_parameter_group(group_name) - # Check the result contains the correct description - assert result['Test Title'][0]['DBClusterParameterGroupName'] == group_name - assert result['Test Title'][0]['DBParameterGroupFamily'] == "TestFamily" - assert result['Test Title'][0]['Description'] == "Test Description" + assert_parameter_group_details(result, 0, group_name, family, description) + result = parameter_group_controller.describe_db_cluste_parameter_group() + # Check the result contains the correct description + assert_parameter_group_details(result, 0, group_name, family, description) + # Cleanup - delete_file_if_exists(file_name) + delete_file_if_exists(storage_manager, file_name) def test_describe_nonexistent_parameter_group(parameter_group_controller): group_name = "NonExistentGroup" # Test if exception is raised when trying to describe a non-existent group - with pytest.raises(KeyError, match=f"Parameter Group '{group_name}' does not exist."): + with pytest.raises(ValueError, match=f"Parameter Group '{group_name}' does not exist."): parameter_group_controller.describe_db_cluste_parameter_group(group_name) + +def test_describe_group_without_parameter_group_name(parameter_group_controller, storage_manager): + max_records = 2 + marker = None + + + # Mock the return of get_all_groups method to simulate multiple parameter groups + mock_parameter_groups = { + "Group1": {"group_name": "Group1", "family": "TestFamily1", "description": "Description 1"}, + "Group2": {"group_name": "Group2", "family": "TestFamily2", "description": "Description 2"}, + "Group3": {"group_name": "Group3", "family": "TestFamily3", "description": "Description 3"}, + } + for p in mock_parameter_groups.values(): + create_parameter_group(parameter_group_controller, p['group_name'], p['family'], p['description']) + # with patch.object(parameter_group_controller.service.dal, 'get_all_groups', return_value=mock_parameter_groups): + # result = parameter_group_controller.describe_db_cluste_parameter_group() + # parameter_group_controller.dal.get_all_groups = lambda: mock_parameter_groups + + # Call the describe_group without a parameter_group_name + result = parameter_group_controller.describe_db_cluste_parameter_group(max_records=max_records, marker=marker) + + # Check that the correct number of parameter groups are returned based on max_records + assert len(result["DBClusterParameterGroup"]) == max_records + for idx, p in enumerate(mock_parameter_groups.values()): + if idx >= max_records: + break + assert_parameter_group_details(result, idx, p['group_name'], p['family'], p['description']) + + + + # Check if pagination marker is returned + assert 'Marker' in result + assert result['Marker'] == "Group3" + for p in mock_parameter_groups.values(): + file_name=generate_file_name_for_group(p['group_name']) + delete_file_if_exists(storage_manager, file_name) + + +# def test_describe_group_without_any_parameters(parameter_group_controller): +# # title = "Default Title" + +# # Mock the return of get_all_groups method to simulate multiple parameter groups +# mock_parameter_groups = { +# "Group1": {"group_name": "Group1", "family": "TestFamily1", "description": "Description 1"}, +# "Group2": {"group_name": "Group2", "family": "TestFamily2", "description": "Description 2"}, +# } +# # for p in mock_parameter_groups.values(): +# # create_parameter_group(parameter_group_controller, p['group_name'], p['family'], p['description']) + +# parameter_group_controller.get_all_groups = lambda: mock_parameter_groups + +# # Call describe_group without any parameters (using default values) +# result = parameter_group_controller.describe_db_cluste_parameter_group() + +# # Verify that all parameter groups are returned (up to max_records default which is 100) +# assert len(result["DBClusterParameterGroup"]) == len(mock_parameter_groups) +# assert result["DBClusterParameterGroup"][0]['DBClusterParameterGroupName'] == "Group1" +# assert result["DBClusterParameterGroup"][1]['DBClusterParameterGroupName'] == "Group2" + +# # Check that marker is not returned since there are less than 100 records +# assert 'Marker' not in result diff --git a/DB/NEW_KT_DB/Test/GeneralTests.py b/DB/NEW_KT_DB/Test/GeneralTests.py index 2f5ad49b..641d447b 100644 --- a/DB/NEW_KT_DB/Test/GeneralTests.py +++ b/DB/NEW_KT_DB/Test/GeneralTests.py @@ -1,37 +1,27 @@ import json import os +import sys import pytest -from Storage.NEW_KT_Storage.DataAccess import StorageManager - +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..'))) +from Storage.NEW_KT_Storage.DataAccess.StorageManager import StorageManager +@pytest.fixture def storage_manager(): - """Fixture to create an instance of OptionGroup.""" + """Fixture to create an instance of StorageManager.""" return StorageManager('test') -def test_file_exists(file_name): - assert storage_manager.is_file_exist(file_name) - -def assert_file_exists(file_name): +def assert_file_exists(storage_manager, file_name): assert storage_manager.is_file_exist(file_name), f"Expected file {file_name} was not created." # Generic function to delete a file -def delete_file_if_exists(storage_manager,file_name): +def delete_file_if_exists(storage_manager, file_name): storage_manager.delete_file(file_name) # Generic function to load JSON file and assert its content -def assert_json_content(file_name, expected_data): - with open(file_name, 'r') as json_file: +def assert_json_content(storage_manager, file_name, expected_data): + full_path = os.path.join(storage_manager.base_directory, file_name) + with open(full_path, 'r') as json_file: data = json.load(json_file) for key, value in expected_data.items(): - assert data[key] == value, f"Expected {key} to be {value}, but got {data[key]}" - -# @pytest.fixture -# def storage_manager(): -# """Fixture to create an instance of StorageManager.""" -# return StorageManager('test') - - -# def test_file_exists(storage_manager, file_name): # Pass storage_manager as a parameter - -# # Use storage_manager to check if the file exists -# assert storage_manager.is_file_exist(file_name), f"File {file_name} does not exist." \ No newline at end of file + print(value) + assert data[key] == value, f"Expected {key} to be {value}, but got {data[key]}" \ No newline at end of file From 9d26a54e59cf940a74e020f4b6adc712a110c6f0 Mon Sep 17 00:00:00 2001 From: tamar koledetzky Date: Wed, 18 Sep 2024 17:38:49 +0300 Subject: [PATCH 09/11] document the get method --- .../Classes/DBClusterParameterGroupService.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/DB/NEW_KT_DB/Service/Classes/DBClusterParameterGroupService.py b/DB/NEW_KT_DB/Service/Classes/DBClusterParameterGroupService.py index d8a843bc..65af796b 100644 --- a/DB/NEW_KT_DB/Service/Classes/DBClusterParameterGroupService.py +++ b/DB/NEW_KT_DB/Service/Classes/DBClusterParameterGroupService.py @@ -166,7 +166,6 @@ def modify(self, title: str, group_name: str, parameters: Optional[list[Dict[str self.storage_manager.write_to_file(file_name, json.dumps(parameter_group_dict)) return {title: group_name} - # @abstractmethod def describe(self, data: tuple) -> Dict: """ Abstract method to describe a parameter group. @@ -184,8 +183,19 @@ def describe(self, data: tuple) -> Dict: } return describe - def get(self, group_name: str)->Dict: - result= self.dal.get(group_name) + def get(self, group_name: str) -> Dict: + """ + Retrieve a parameter group by its name. + + :param group_name: The name of the parameter group to retrieve. + :return: A dictionary representing the parameter group. + :raises ValueError: If the parameter group does not exist. + + This method queries the data access layer (DAL) to retrieve the parameter group with the specified name. + If no parameter group is found, it raises a ValueError indicating that the parameter group does not exist. + Otherwise, it returns the first result as a dictionary. + """ + result = self.dal.get(group_name) if result == []: - raise ValueError(f"Parameter Group '{group_name}' does not exist.") - return result[0] \ No newline at end of file + raise ValueError(f"Parameter Group '{group_name}' does not exist.") + return result[0] \ No newline at end of file From 1bcf88430c1413288d773df2920e36fdc18c4261 Mon Sep 17 00:00:00 2001 From: tamar koledetzky Date: Thu, 19 Sep 2024 09:29:07 +0300 Subject: [PATCH 10/11] delete_subnet_group --- .../Controller/DBSubnetGroupController.py | 24 -- .../DataAccess/DBSubnetGroupManager.py | 38 --- DB/NEW_KT_DB/Models/DBSubnetGroupModel.py | 71 ------ .../Service/Classes/DBSubnetGroupService.py | 91 ------- .../Test/DBClusterParameterGroupTests.py | 4 +- DB/NEW_KT_DB/Test/DBSubnetGroupTests.py | 228 ------------------ 6 files changed, 2 insertions(+), 454 deletions(-) delete mode 100644 DB/NEW_KT_DB/Controller/DBSubnetGroupController.py delete mode 100644 DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py delete mode 100644 DB/NEW_KT_DB/Models/DBSubnetGroupModel.py delete mode 100644 DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py delete mode 100644 DB/NEW_KT_DB/Test/DBSubnetGroupTests.py diff --git a/DB/NEW_KT_DB/Controller/DBSubnetGroupController.py b/DB/NEW_KT_DB/Controller/DBSubnetGroupController.py deleted file mode 100644 index 027dbcc9..00000000 --- a/DB/NEW_KT_DB/Controller/DBSubnetGroupController.py +++ /dev/null @@ -1,24 +0,0 @@ -from Service.Classes import DBSubnetGroupService - -class DBSubnetGroupController: - def __init__(self, service: DBSubnetGroupService): - self.service = service - - - def create_db_subnet_group(self, **kwargs): - self.service.create_db_subnet_group(**kwargs) - - - def delete_db_subnet_group(self, name): - self.service.delete_db_subnet_group(name) - - - def modify_db_subnet_group(self, name, updates): - self.service.modify_db_subnet_group(name, updates) - - def get_db_subnet_group(self, name): - return self.service.get_db_subnet_group(name) - - def describe_db_subnet_group(self, name): - return self.service.describe_db_subnet_group(name) - \ No newline at end of file diff --git a/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py b/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py deleted file mode 100644 index 68c8af52..00000000 --- a/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py +++ /dev/null @@ -1,38 +0,0 @@ -from typing import Dict, Any, List - -import sys -import os -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - -from Models.DBSubnetGroupModel import DBSubnetGroup - -class DBSubnetGroupManager: - def __init__(self, object_manager): - self.object_manager = object_manager - self.object_manager._create_management_table(DBSubnetGroup.table_name, DBSubnetGroup.table_structure) - - def create(self, subnet_group: DBSubnetGroup): - self.object_manager.save_in_memory(DBSubnetGroup.table_name, subnet_group) - - def get(self, name: str): - data = self.object_manager.get_from_memory_by_id(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, name) - if data: - data_mapping = {'db_subnet_group_name':name} - for key, value in data[name].items(): - data_mapping[key] = value - return DBSubnetGroup(**data_mapping) - else: - raise ValueError(f"subnet group with name '{name}' not found") - - - def delete(self, name: str): - self.object_manager.delete_from_memory_by_id(DBSubnetGroup.pk_column, name, DBSubnetGroup.table_name) - - def describe(self, name: str): - return self.get(name).to_dict() - - def modify(self, subnet_group: DBSubnetGroup): - updates = subnet_group.to_dict() - del updates['db_subnet_group_name'] - self.object_manager.update_in_memory_by_id(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, updates, subnet_group.db_subnet_group_name) diff --git a/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py b/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py deleted file mode 100644 index 7ed37365..00000000 --- a/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py +++ /dev/null @@ -1,71 +0,0 @@ -from typing import List, Dict, Any -import ast -import json -import sys -import os -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - -from DataAccess.ObjectManager import ObjectManager - -class DBSubnetGroup: - - pk_column = 'db_subnet_group_name' - table_name = 'db_subnet_groups' - table_structure = f""" - db_subnet_group_name primary key not null, - db_subnet_group_description TEXT NOT NULL, - vpc_id VARCHAR(255) NOT NULL, - subnets JSONB DEFAULT '{{}}', - db_subnet_group_arn VARCHAR(255), - status VARCHAR(50) DEFAULT 'pending' - """ - - def __init__(self, **kwargs): - try: - print(kwargs) - self.db_subnet_group_name = kwargs['db_subnet_group_name'] - self.db_subnet_group_description = kwargs['db_subnet_group_description'] - self.vpc_id = kwargs['vpc_id'] - self.subnets = kwargs.get('subnets', None) - if not self.subnets: - self.subnets = [] - if type(self.subnets) is not list: - self.subnets = ast.literal_eval(self.subnets) - self.db_subnet_group_arn = kwargs.get('db_subnet_group_arn', None) - - except KeyError as e: - raise ValueError(f"Missing required attribute for DBSubnetGroup: {str(e)}") - - # Ideally: - # self.db_subnet_group_arn should be dynamically created according to vpc-id, account-id and - # subnet-group-name, and then dynamically added to the routing table - - self.status = 'pending' - self.pk_value = self.db_subnet_group_name - - def to_dict(self) -> Dict[str, Any]: - return ObjectManager.convert_object_attributes_to_dictionary( - db_subnet_group_name=self.db_subnet_group_name, - db_subnet_group_description = self.db_subnet_group_description, - vpc_id = self.vpc_id, - subnets = self.subnets, - db_subnet_group_arn = self.db_subnet_group_arn, - status = self.status - ) - - def to_bytes(self): - bytes = json.dumps(self.to_dict()) - bytes = bytes.encode('utf-8') - return bytes - - def from_bytes_to_dict(bytes): - return json.loads(bytes.decode('utf-8')) - - - def to_sql(self): - # Convert the model instance to a dictionary - data_dict = self.to_dict() - values = '(' + ", ".join(f'\'{json.dumps(v)}\'' if isinstance(v, dict) or isinstance(v, list) else f'\'{v}\'' if isinstance(v, str) else f'\'{str(v)}\'' - for v in data_dict.values()) + ')' - return values \ No newline at end of file diff --git a/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py b/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py deleted file mode 100644 index 8fc33513..00000000 --- a/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py +++ /dev/null @@ -1,91 +0,0 @@ -from sqlite3 import IntegrityError -from typing import List, Dict, Any - -import sys -import os -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..'))) -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - - -from Models.DBSubnetGroupModel import DBSubnetGroup -from DataAccess.DBSubnetGroupManager import DBSubnetGroupManager -from Storage.KT_Storage.DataAccess.StorageManager import StorageManager -from Storage.KT_Storage.DataAccess.VersionManager import VersionManager -from Validation.GeneralValidations import * -class DBSubnetGroupService: - def __init__(self, db_subnet_group_manager: DBSubnetGroupManager): - self.manager = db_subnet_group_manager - self.bucket = 'db_subnet_groups' - self.storage_manager = StorageManager() - self.storage_manager.create_bucket(self.bucket) - self.version_manager = VersionManager() - self.subnet_groups = dict() - - def create_db_subnet_group(self, **kwargs): - # object - if not kwargs.get('db_subnet_group_name'): - raise ValueError('Missing required argument db_subnet_group_name') - - if not is_length_in_range(kwargs['db_subnet_group_name'], 1, 255): - raise ValueError("invalid length for subnet group db_subnet_group_name: " + len(kwargs['db_subnet_group_name'])) - - if kwargs['db_subnet_group_name'] in self.subnet_groups: - raise ValueError(f"db_subnet_group_name {kwargs['db_subnet_group_name']} already exists") - - if kwargs.get('description') and not is_length_in_range('description', 1, 255): - raise ValueError("invalid length for subnet group description: " + len(kwargs['description'])) - - subnet_group = DBSubnetGroup(**kwargs) - # save in management table - # in try except block in case the server was shut down and re-run and local collection doesn't include all subnetGroups - try: - self.manager.create(subnet_group) - except IntegrityError as e: - raise ValueError(f"db_subnet_group_name {kwargs['db_subnet_group_name']} already exists") - - # physical object - # version = 0 assume created for the first time - self.storage_manager.create(self.bucket, subnet_group.db_subnet_group_name, subnet_group.to_bytes(), '0') - # save in local collection (hash table) for quick access - self.subnet_groups[kwargs['db_subnet_group_name']] = subnet_group - - def get_db_subnet_group(self, db_subnet_group_name: str) -> DBSubnetGroup: - data = self.manager.get(db_subnet_group_name) - return data - - def modify_db_subnet_group(self, db_subnet_group_name: str, updates: Dict[str, Any]) -> DBSubnetGroup: - if not db_subnet_group_name: - raise ValueError('Missing required argument db_subnet_group_name') - - if updates.get('description') and not is_length_in_range(updates['description'], 1, 255): - raise ValueError("invalid length for subnet group description: " + len(updates['description'])) - - subnet_group = self.get_db_subnet_group(db_subnet_group_name) - - for key, value in updates.items(): - setattr(subnet_group, key, value) - - self.manager.modify(subnet_group) - # version = str(int(self.version_manager.get(self.bucket, subnet_group.db_subnet_group_name).version_id)+1) - # for now we override the basic version, when the latest version id can be retrieved, we will make a new version as old_version_id + 1 - self.storage_manager.create(self.bucket, db_subnet_group_name, subnet_group.to_bytes(), '0') - - def delete_db_subnet_group(self, db_subnet_group_name: str) -> None: - if not db_subnet_group_name: - raise ValueError('Missing required argument db_subnet_group_name') - - # delete from management table - self.manager.delete(db_subnet_group_name) - # for now version id is 0 - # delete physical object from storage - self.storage_manager.delete_by_name(bucket_name=self.bucket, version_id='0', key=db_subnet_group_name) - # delete from local collection (hash table) - if db_subnet_group_name in self.subnet_groups: - del self.subnet_groups[db_subnet_group_name] - - def describe_db_subnet_group(self, db_subnet_group_name: str) -> Dict: - if not db_subnet_group_name: - raise ValueError('Missing required argument db_subnet_group_name') - - return self.manager.describe(db_subnet_group_name) \ No newline at end of file diff --git a/DB/NEW_KT_DB/Test/DBClusterParameterGroupTests.py b/DB/NEW_KT_DB/Test/DBClusterParameterGroupTests.py index 86d332ac..88bb6825 100644 --- a/DB/NEW_KT_DB/Test/DBClusterParameterGroupTests.py +++ b/DB/NEW_KT_DB/Test/DBClusterParameterGroupTests.py @@ -138,7 +138,7 @@ def test_delete_parameter_group(parameter_group_controller, storage_manager): # Check if the file was deleted assert not os.path.exists(file_name), f"Expected file {file_name} was not deleted." -def test_delete_parameter_group_with_associated_cluster(parameter_group_controller, cluster_manager, storage_manager): +def test_delete_parameter_group_with_associated_cluster(parameter_group_controller, storage_manager): group_name = "TestGroup" file_name = generate_file_name_for_group(group_name) @@ -146,7 +146,7 @@ def test_delete_parameter_group_with_associated_cluster(parameter_group_controll create_parameter_group(parameter_group_controller, group_name, "TestFamily", "Test Description") # Mock get_all_clusters to return a cluster associated with the parameter group - cluster_manager.get_all_clusters.return_value =[("","","","","","",group_name)] #{"TestCluster": {"group_name": group_name}} + parameter_group_controller.service.dal_cluster.get_all_clusters.return_value =[("","","","","","",group_name)] #{"TestCluster": {"group_name": group_name}} # Attempt to delete the parameter group, expect an exception due to association with cluster with pytest.raises(ValueError, match="Can't delete parameter group associated with any DB clusters"): diff --git a/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py b/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py deleted file mode 100644 index b2fe7674..00000000 --- a/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py +++ /dev/null @@ -1,228 +0,0 @@ -from sqlite3 import IntegrityError -import pytest -import json - -import sys -import os -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '.'))) - -from Service.Classes.DBSubnetGroupService import DBSubnetGroupService -from DataAccess.DBSubnetGroupManager import DBSubnetGroupManager -from Controller.DBSubnetGroupController import DBSubnetGroupController -from Storage.KT_Storage.DataAccess.StorageManager import StorageManager -from DataAccess.ObjectManager import ObjectManager -from Models.DBSubnetGroupModel import DBSubnetGroup -import sqlite3 - -object_manager = ObjectManager('../object_management_db.db') -manager = DBSubnetGroupManager(object_manager=object_manager) -service = DBSubnetGroupService(manager) -controller = DBSubnetGroupController(service) -storage_manager = StorageManager() - -@pytest.fixture -def clear_table(): - # Connect to the SQLite database - conn = conn = sqlite3.connect('../object_management_db.db') - cursor = conn.cursor() - - # Clear the table if it exists - table_name = "db_subnet_groups" - cursor.execute(f"DELETE FROM {table_name};") - - # Commit changes and close the connection - conn.commit() - conn.close() - - # Yield to allow tests to run - yield - -def test_create(clear_table): - - # remove existing subnet group from previous tests - controller.create_db_subnet_group( - db_subnet_group_name='subnet_group_1', - subnets=[ - {'subnet_id': 'subnet-12345678'}, - {'subnet_id': 'subnet-87654321'} - ], - db_subnet_group_description='Test subnet group', - vpc_id='vpc-12345678', - db_subnet_group_arn='arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' - ) - - # check that file was created (no error raised on get) - storage_manager.get('db_subnet_groups', 'subnet_group_1', '0') - # check that object was saved to management table (no error raised on get) - controller.get_db_subnet_group('subnet_group_1') - - # check that file content is correct - from_storage = DBSubnetGroup(**DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', 'subnet_group_1', '0')['content'])) - from_db = controller.get_db_subnet_group('subnet_group_1') - # check values were stored correctly in management table as well as storage - - # group name - assert from_storage.db_subnet_group_name == 'subnet_group_1' - assert from_db.db_subnet_group_name == 'subnet_group_1' - - # subnets - for subnet in from_storage.subnets: - assert subnet in [ - {'subnet_id': 'subnet-12345678'}, - {'subnet_id': 'subnet-87654321'} - ] - for subnet in from_db.subnets: - assert subnet in [ - {'subnet_id': 'subnet-12345678'}, - {'subnet_id': 'subnet-87654321'} - ] - - # description - assert from_storage.db_subnet_group_description == 'Test subnet group' - assert from_db.db_subnet_group_description == 'Test subnet group' - - # vpc_id - assert from_storage.vpc_id == 'vpc-12345678' - assert from_db.vpc_id == 'vpc-12345678' - - # arn - assert from_storage.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' - assert from_db.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' - - -def test_unique_constraint(): - with pytest.raises(ValueError): - controller.create_db_subnet_group( - db_subnet_group_name='subnet_group_1', - subnets=[ - {'subnet_id': 'subnet-87654321'}, - {'subnet_id': 'subnet-12345678'} - ], - db_subnet_group_description='Another subnet group with same name', - vpc_id='vpc-87654321', - db_subnet_group_arn='arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' - ) - - -def test_get(): - subnet_group_1 = controller.get_db_subnet_group('subnet_group_1') - assert subnet_group_1 != None - assert subnet_group_1.db_subnet_group_name == 'subnet_group_1' - assert subnet_group_1.db_subnet_group_description == 'Test subnet group' - assert subnet_group_1.vpc_id == 'vpc-12345678' - assert subnet_group_1.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' - assert subnet_group_1.status == 'pending' - for subnet in subnet_group_1.subnets: - assert subnet in [ - {"subnet_id": "subnet-12345678"}, - {"subnet_id": "subnet-87654321"} - ] - -def test_modify(): - controller.modify_db_subnet_group( - name='subnet_group_1', - updates= {'subnets':[ - {'subnet_id': 'subnet-12345988'}, - {'subnet_id': 'subnet-876543881'}]} - ) - - from_storage = DBSubnetGroup(**DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', 'subnet_group_1', '0')['content'])) - from_db = controller.get_db_subnet_group('subnet_group_1') - assert from_storage.db_subnet_group_name == from_db.db_subnet_group_name - for subnet in from_storage.subnets: - assert subnet in from_db.subnets - assert from_storage.db_subnet_group_description == from_db.db_subnet_group_description - assert from_storage.vpc_id == from_db.vpc_id - assert from_storage.db_subnet_group_arn == from_db.db_subnet_group_arn - assert from_storage.status == from_db.status - -def test_describe(): - subnet_group_1 = controller.describe_db_subnet_group('subnet_group_1') - assert type(subnet_group_1) == dict - assert type(subnet_group_1['subnets']) == list - assert type(subnet_group_1['subnets'][0]) == dict - subnet_group_1 = DBSubnetGroup(**subnet_group_1) - assert subnet_group_1 != None - assert subnet_group_1.db_subnet_group_name == 'subnet_group_1' - assert subnet_group_1.db_subnet_group_description == 'Test subnet group' - assert subnet_group_1.vpc_id == 'vpc-12345678' - assert subnet_group_1.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' - assert subnet_group_1.status == 'pending' - for subnet in subnet_group_1.subnets: - assert subnet in [ - {'subnet_id': 'subnet-12345988'}, - {'subnet_id': 'subnet-876543881'} - ] - - -def test_delete(): - controller.delete_db_subnet_group('subnet_group_1') - with pytest.raises(FileNotFoundError): - storage_manager.get('db_subnet_groups', 'subnet_group_1', '0') - with pytest.raises(Exception): - controller.get_db_subnet_group('subnet_group_1') - - -@pytest.mark.parametrize("index", range(20)) -def test_insert_many(index): - controller.create_db_subnet_group( - db_subnet_group_name=f'subnet_group_{index}', - subnets=[ - {'subnet_id': f'subnet-1234567{index}'}, - {'subnet_id': f'subnet-8765432{index}'} - ], - db_subnet_group_description=f'Test subnet group {index}', - vpc_id=f'vpc-1234567{index}', - db_subnet_group_arn=f'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_{index}' - ) - - # check that file was created (no error raised on get) - storage_manager.get('db_subnet_groups', f'subnet_group_{index}', '0') - # check that object was saved to management table (no error raised on get) - controller.get_db_subnet_group(f'subnet_group_{index}') - - # check that file content is correct - from_storage = DBSubnetGroup(**DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', f'subnet_group_{index}', '0')['content'])) - from_db = controller.get_db_subnet_group(f'subnet_group_{index}') - # check values were stored correctly in management table as well as storage - - # group name - assert from_storage.db_subnet_group_name == f'subnet_group_{index}' - assert from_db.db_subnet_group_name == f'subnet_group_{index}' - - # subnets - for subnet in from_storage.subnets: - assert subnet in [ - {'subnet_id': f'subnet-1234567{index}'}, - {'subnet_id': f'subnet-8765432{index}'} - ] - for subnet in from_db.subnets: - assert subnet in [ - {'subnet_id': f'subnet-1234567{index}'}, - {'subnet_id': f'subnet-8765432{index}'} - ] - - # description - assert from_storage.db_subnet_group_description == f'Test subnet group {index}' - assert from_db.db_subnet_group_description == f'Test subnet group {index}' - - # vpc_id - assert from_storage.vpc_id == f'vpc-1234567{index}' - assert from_db.vpc_id == f'vpc-1234567{index}' - - # arn - assert from_storage.db_subnet_group_arn == f'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_{index}' - assert from_db.db_subnet_group_arn == f'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_{index}' - - - -@pytest.mark.parametrize("index", range(20)) -def test_delete_many_from_prev_test(index): - db_subnet_group_name=f'subnet_group_{index}' - controller.delete_db_subnet_group(db_subnet_group_name) - with pytest.raises(FileNotFoundError): - storage_manager.get('db_subnet_groups', db_subnet_group_name, '0') - with pytest.raises(Exception): - controller.get_db_subnet_group(db_subnet_group_name) From ccd057dfced1a9ea53d46ad93b56b0af1f76fd4f Mon Sep 17 00:00:00 2001 From: tamar koledetzky Date: Sun, 22 Sep 2024 09:32:54 +0300 Subject: [PATCH 11/11] changes finish --- .../DBClusterParameterGroupManager.py | 2 +- .../Models/DBClusterParameterGroupModel.py | 2 +- .../Classes/DBClusterParameterGroupService.py | 8 +- .../Test/DBClusterParameterGroupTests.py | 124 +++--------------- .../db_cluster_parameter_group_default.json | 1 + 5 files changed, 26 insertions(+), 111 deletions(-) create mode 100644 test/db_cluster_parameter_groups/db_cluster_parameter_group_default.json diff --git a/DB/NEW_KT_DB/DataAccess/DBClusterParameterGroupManager.py b/DB/NEW_KT_DB/DataAccess/DBClusterParameterGroupManager.py index a0ce1b93..8a8dc90a 100644 --- a/DB/NEW_KT_DB/DataAccess/DBClusterParameterGroupManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBClusterParameterGroupManager.py @@ -9,7 +9,7 @@ def __init__(self, db_file: str): '''Initialize ObjectManager with the database connection.''' self.object_manager = ObjectManager(db_file) self.object_manager.create_management_table( - DBClusterParameterGroup.get_object_name(), DBClusterParameterGroup.table_schema, 'TEXT') + DBClusterParameterGroup.get_object_name(), DBClusterParameterGroup.table_structure, 'TEXT') def createInMemoryDBCluster(self, data): diff --git a/DB/NEW_KT_DB/Models/DBClusterParameterGroupModel.py b/DB/NEW_KT_DB/Models/DBClusterParameterGroupModel.py index c8389164..cad089c1 100644 --- a/DB/NEW_KT_DB/Models/DBClusterParameterGroupModel.py +++ b/DB/NEW_KT_DB/Models/DBClusterParameterGroupModel.py @@ -6,7 +6,7 @@ class DBClusterParameterGroup: pk_column = 'group_name' - table_schema = """ + table_structure = """ group_name TEXT PRIMARY KEY, group_family TEXT, description TEXT, diff --git a/DB/NEW_KT_DB/Service/Classes/DBClusterParameterGroupService.py b/DB/NEW_KT_DB/Service/Classes/DBClusterParameterGroupService.py index 65af796b..959725f5 100644 --- a/DB/NEW_KT_DB/Service/Classes/DBClusterParameterGroupService.py +++ b/DB/NEW_KT_DB/Service/Classes/DBClusterParameterGroupService.py @@ -114,7 +114,7 @@ def describe_group(self, title: str, parameter_group_name: str = None, max_recor return {title: parameter_groups_local} return {'Marker': marker, title: parameter_groups_local} - def camel_to_snake_case(self, name: str) -> str: + def convert_camel_case_string_to_snake(self, name: str) -> str: """ Convert a CamelCase string to snake_case. @@ -123,14 +123,14 @@ def camel_to_snake_case(self, name: str) -> str: """ return ''.join(['_' + c.lower() if c.isupper() else c for c in name]).lstrip('_') - def convert_dict_keys_to_snake_case(self, input_dict: Dict) -> Dict: + def convert_dict_keys_from_camel_case_to_snake(self, input_dict: Dict) -> Dict: """ Convert all keys in a dictionary from CamelCase to snake_case. :param input_dict: The input dictionary with CamelCase keys. :return: A new dictionary with snake_case keys. """ - return {self.camel_to_snake_case(key): value for key, value in input_dict.items()} + return {self.convert_camel_case_string_to_snake(key): value for key, value in input_dict.items()} def modify(self, title: str, group_name: str, parameters: Optional[list[Dict[str, any]]] = None): """ @@ -153,7 +153,7 @@ def modify(self, title: str, group_name: str, parameters: Optional[list[Dict[str if new_parameter['ParameterName'] == old_parameter['parameter_name']: if old_parameter['is_modifiable'] == False: raise ValueError(f"You can't modify the parameter {old_parameter['parameter_name']}") - new_parameter_updates = self.convert_dict_keys_to_snake_case(new_parameter) + new_parameter_updates = self.convert_dict_keys_from_camel_case_to_snake(new_parameter) updated_parameter = {**old_parameter, **new_parameter_updates} parameters_in_parameter_group[idx] = updated_parameter self.dal.modifyDBCluster(group_name, f"parameters='{json.dumps(parameters_in_parameter_group)}'") diff --git a/DB/NEW_KT_DB/Test/DBClusterParameterGroupTests.py b/DB/NEW_KT_DB/Test/DBClusterParameterGroupTests.py index 88bb6825..d1cfe403 100644 --- a/DB/NEW_KT_DB/Test/DBClusterParameterGroupTests.py +++ b/DB/NEW_KT_DB/Test/DBClusterParameterGroupTests.py @@ -13,14 +13,19 @@ from Storage.NEW_KT_Storage.DataAccess.StorageManager import StorageManager from GeneralTests import * +# Generic function for file name +def generate_file_name_for_group (group_name): + return f'db_cluster_parameter_groups/db_cluster_parameter_group_{group_name}.json' + +group_name = "TestGroup" +group_family = "TestFamily" +description = "Test Description" +file_name = generate_file_name_for_group(group_name) + @pytest.fixture def parameter_group_manager(): return DBClusterParameterGroupManager(':memory:') -# @pytest.fixture -# def cluster_manager(): -# return DBClusterManager(':memory:') - @pytest.fixture def cluster_manager(): # Create a mock for DBClusterManager and its method get_all_clusters @@ -41,10 +46,6 @@ def parameter_group_controller(parameter_group_service): def create_parameter_group(controller, group_name, group_family, description): return controller.create_db_cluster_parameter_group(group_name, group_family, description) -# Generic function for file name -def generate_file_name_for_group (group_name): - return f'db_cluster_parameter_groups/db_cluster_parameter_group_{group_name}.json' - # Generic function to assert the parameter group's details def assert_parameter_group_details(result, index, expected_group_name, expected_family, expected_description): """ @@ -66,11 +67,6 @@ def assert_parameter_group_details(result, index, expected_group_name, expected_ f"Expected Description to be '{expected_description}' but got '{parameter_group['Description']}'" def test_create_parameter_group(parameter_group_controller, storage_manager): - group_name = "TestGroup" - group_family = "TestFamily" - description = "Test Description" - file_name = generate_file_name_for_group(group_name) - # Create the parameter group result = create_parameter_group(parameter_group_controller, group_name, group_family, description) assert result['DBClusterParameterGroupName'] == group_name @@ -89,10 +85,7 @@ def test_create_parameter_group(parameter_group_controller, storage_manager): delete_file_if_exists(storage_manager, file_name) def test_create_existing_parameter_group(parameter_group_controller): - group_name = "TestGroup" - group_family = "TestFamily" - - # Ensure the group exists + # Ensure the group exists create_parameter_group(parameter_group_controller, group_name, group_family, "Test Description") # Test if exception is raised when trying to create an existing group @@ -106,26 +99,7 @@ def test_create_parameter_group_with_invalid_name(parameter_group_controller): with pytest.raises(ValueError, match=f"group_name {invalid_group_name} is not valid"): create_parameter_group(parameter_group_controller, invalid_group_name, "ValidFamily", "Valid Description") -# def test_delete_parameter_group(parameter_group_controller): -# group_name = "TestGroup" -# file_name = generate_file_name_for_group(group_name) - -# # Create the parameter group -# create_parameter_group(parameter_group_controller, group_name, "TestFamily", "Test Description") - -# # Ensure the file exists before deletion -# assert_file_exists(file_name) - -# # Delete the parameter group -# parameter_group_controller.delete_db_cluste_parameter_group(group_name) - -# # Check if the file was deleted -# assert not os.path.exists(file_name), f"Expected file {file_name} was not deleted." - def test_delete_parameter_group(parameter_group_controller, storage_manager): - group_name = "TestGroup" - file_name = generate_file_name_for_group(group_name) - # Create the parameter group create_parameter_group(parameter_group_controller, group_name, "TestFamily", "Test Description") @@ -139,10 +113,7 @@ def test_delete_parameter_group(parameter_group_controller, storage_manager): assert not os.path.exists(file_name), f"Expected file {file_name} was not deleted." def test_delete_parameter_group_with_associated_cluster(parameter_group_controller, storage_manager): - group_name = "TestGroup" - file_name = generate_file_name_for_group(group_name) - - # Create the parameter group + # Create the parameter group create_parameter_group(parameter_group_controller, group_name, "TestFamily", "Test Description") # Mock get_all_clusters to return a cluster associated with the parameter group @@ -162,27 +133,9 @@ def test_delete_nonexistent_parameter_group(parameter_group_controller): with pytest.raises(ValueError, match=f"Parameter Group '{group_name}' does not exist."): parameter_group_controller.delete_db_cluste_parameter_group(group_name) -# def test_delete_parameter_group_with_associated_cluster(parameter_group_controller, cluster_manager): -# group_name = "TestGroup" -# file_name = generate_file_name_for_group(group_name) - -# # Create the parameter group -# create_parameter_group(parameter_group_controller, group_name, "TestFamily", "Test Description") - -# # Associate the parameter group with a cluster -# cluster_manager.create_cluster({"cluster_id": "TestCluster", "group_name": group_name}) - -# # Attempt to delete the parameter group, expect an exception due to association with cluster -# with pytest.raises(ValueError, match="Can't delete parameter group associated with any DB clusters"): -# parameter_group_controller.delete_db_cluste_parameter_group(group_name) - -# # Cleanup -# delete_file_if_exists(file_name) - def test_delete_default_parameter_group(parameter_group_controller, storage_manager): group_name = "default" - file_name = generate_file_name_for_group(group_name) - + # Create the default parameter group create_parameter_group(parameter_group_controller, group_name, "DefaultFamily", "Default group description") @@ -194,11 +147,8 @@ def test_delete_default_parameter_group(parameter_group_controller, storage_mana delete_file_if_exists(storage_manager, file_name) def test_modify_parameter_group(parameter_group_controller, storage_manager): - group_name = "TestGroup" - file_name = generate_file_name_for_group(group_name) - # Create a parameter group - create_parameter_group(parameter_group_controller, group_name, "TestFamily", "Test Description") + create_parameter_group(parameter_group_controller, group_name, group_family, description) # Modify the parameter group with new parameters parameters = [ @@ -226,11 +176,8 @@ def test_modify_nonexistent_parameter_group(parameter_group_controller): parameter_group_controller.modify_db_cluste_parameter_group(group_name, parameters) def test_modify_non_modifiable_parameter(parameter_group_controller, storage_manager): - group_name = "TestGroup" - file_name = generate_file_name_for_group(group_name) - - # Create the parameter group - create_parameter_group(parameter_group_controller, group_name, "TestFamily", "Test Description") + # Create the parameter group + create_parameter_group(parameter_group_controller, group_name, group_family, description) # Define a non-modifiable parameter parameters = [ @@ -249,11 +196,8 @@ def test_modify_non_modifiable_parameter(parameter_group_controller, storage_man delete_file_if_exists(storage_manager, file_name) def test_modify_with_invalid_is_modifiable(parameter_group_controller, storage_manager): - group_name = "TestGroup" - file_name = generate_file_name_for_group(group_name) - # Create the parameter group - create_parameter_group(parameter_group_controller, group_name, "TestFamily", "Test Description") + create_parameter_group(parameter_group_controller, group_name, group_family, description) @@ -285,21 +229,16 @@ def test_modify_with_invalid_apply_method(parameter_group_controller, storage_ma delete_file_if_exists(storage_manager, file_name) def test_describe_parameter_group(parameter_group_controller, storage_manager): - group_name = "TestGroup" - family="TestFamily" - description="Test Description" - file_name = generate_file_name_for_group(group_name) - # Create a parameter group - create_parameter_group(parameter_group_controller, group_name, family, description) + create_parameter_group(parameter_group_controller, group_name, group_family, description) # Describe the parameter group result = parameter_group_controller.describe_db_cluste_parameter_group(group_name) # Check the result contains the correct description - assert_parameter_group_details(result, 0, group_name, family, description) + assert_parameter_group_details(result, 0, group_name, group_family, description) result = parameter_group_controller.describe_db_cluste_parameter_group() # Check the result contains the correct description - assert_parameter_group_details(result, 0, group_name, family, description) + assert_parameter_group_details(result, 0, group_name, group_family, description) # Cleanup @@ -347,28 +286,3 @@ def test_describe_group_without_parameter_group_name(parameter_group_controller, for p in mock_parameter_groups.values(): file_name=generate_file_name_for_group(p['group_name']) delete_file_if_exists(storage_manager, file_name) - - -# def test_describe_group_without_any_parameters(parameter_group_controller): -# # title = "Default Title" - -# # Mock the return of get_all_groups method to simulate multiple parameter groups -# mock_parameter_groups = { -# "Group1": {"group_name": "Group1", "family": "TestFamily1", "description": "Description 1"}, -# "Group2": {"group_name": "Group2", "family": "TestFamily2", "description": "Description 2"}, -# } -# # for p in mock_parameter_groups.values(): -# # create_parameter_group(parameter_group_controller, p['group_name'], p['family'], p['description']) - -# parameter_group_controller.get_all_groups = lambda: mock_parameter_groups - -# # Call describe_group without any parameters (using default values) -# result = parameter_group_controller.describe_db_cluste_parameter_group() - -# # Verify that all parameter groups are returned (up to max_records default which is 100) -# assert len(result["DBClusterParameterGroup"]) == len(mock_parameter_groups) -# assert result["DBClusterParameterGroup"][0]['DBClusterParameterGroupName'] == "Group1" -# assert result["DBClusterParameterGroup"][1]['DBClusterParameterGroupName'] == "Group2" - -# # Check that marker is not returned since there are less than 100 records -# assert 'Marker' not in result diff --git a/test/db_cluster_parameter_groups/db_cluster_parameter_group_default.json b/test/db_cluster_parameter_groups/db_cluster_parameter_group_default.json new file mode 100644 index 00000000..1dba230b --- /dev/null +++ b/test/db_cluster_parameter_groups/db_cluster_parameter_group_default.json @@ -0,0 +1 @@ +{"group_name": "default", "group_family": "DefaultFamily", "description": "Default group description", "parameters": [{"parameter_name": "backup_retention_period", "parameter_value": 7, "description": "", "is_modifiable": true, "apply_method": ""}, {"parameter_name": "preferred_backup_window", "parameter_value": "03:00-03:30", "description": "", "is_modifiable": true, "apply_method": ""}, {"parameter_name": "preferred_maintenance_window", "parameter_value": "Mon:00:00-Mon:00:30", "description": "", "is_modifiable": true, "apply_method": ""}]} \ No newline at end of file