From 48600fc417f2b2865426f65955558679e7e1407b Mon Sep 17 00:00:00 2001 From: Tem-M Date: Sun, 15 Sep 2024 21:47:40 +0300 Subject: [PATCH 01/16] implementation complete - fixes needed --- .../Controller/DBSubnetGroupController.py | 24 ++++ .../DataAccess/DBSubnetGroupManager.py | 34 +++++ DB/NEW_KT_DB/Models/DBSubnetGroupModel.py | 67 ++++++++++ .../Service/Classes/DBSubnetGroupService.py | 74 +++++++++++ DB/NEW_KT_DB/Test/DBSubnetGroupTests.py | 121 ++++++++++++++++++ 5 files changed, 320 insertions(+) create mode 100644 DB/NEW_KT_DB/Controller/DBSubnetGroupController.py create mode 100644 DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py create mode 100644 DB/NEW_KT_DB/Models/DBSubnetGroupModel.py create mode 100644 DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py create mode 100644 DB/NEW_KT_DB/Test/DBSubnetGroupTests.py diff --git a/DB/NEW_KT_DB/Controller/DBSubnetGroupController.py b/DB/NEW_KT_DB/Controller/DBSubnetGroupController.py new file mode 100644 index 00000000..027dbcc9 --- /dev/null +++ b/DB/NEW_KT_DB/Controller/DBSubnetGroupController.py @@ -0,0 +1,24 @@ +from Service.Classes import DBSubnetGroupService + +class DBSubnetGroupController: + def __init__(self, service: DBSubnetGroupService): + self.service = service + + + def create_db_subnet_group(self, **kwargs): + self.service.create_db_subnet_group(**kwargs) + + + def delete_db_subnet_group(self, name): + self.service.delete_db_subnet_group(name) + + + def modify_db_subnet_group(self, name, updates): + self.service.modify_db_subnet_group(name, updates) + + def get_db_subnet_group(self, name): + return self.service.get_db_subnet_group(name) + + def describe_db_subnet_group(self, name): + return self.service.describe_db_subnet_group(name) + \ No newline at end of file diff --git a/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py b/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py new file mode 100644 index 00000000..d6cc671d --- /dev/null +++ b/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py @@ -0,0 +1,34 @@ +from typing import Dict, Any, List + +import sys +import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +from Models.DBSubnetGroupModel import DBSubnetGroup + +class DBSubnetGroupManager: + def __init__(self, db_manager, object_manager): + self.db_manager = db_manager + self.object_manager = object_manager + self.db_manager.create_table(DBSubnetGroup.table_name, DBSubnetGroup.table_structure) + + def create(self, subnet_group: DBSubnetGroup): + self.object_manager.save_in_memory(DBSubnetGroup.table_name, subnet_group.to_sql()) + + def get(self, name: str): + cols = ['db_subnet_group_name', 'db_subnet_group_description', 'vpc_id', 'subnets', 'db_subnet_group_arn', 'status'] + data = self.object_manager.get_from_memory(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, name, cols = cols) + return data + + def delete(self, name: str): + self.object_manager.delete_from_memory(name) + + def describe(self, name: str): + cols = ['db_subnet_group_name', 'db_subnet_group_description', 'vpc_id', 'subnets', 'db_subnet_group_arn', 'status'] + return self.object_manager.get_from_memory(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, name, cols) + + def modify(self, subnet_group: DBSubnetGroup): + updates = subnet_group.to_dict() + del updates['db_subnet_group_name'] + self.object_manager.update_in_memory(DBSubnetGroup.pk_column, subnet_group.db_subnet_group_name, DBSubnetGroup.table_name, updates) \ No newline at end of file diff --git a/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py b/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py new file mode 100644 index 00000000..9b933bea --- /dev/null +++ b/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py @@ -0,0 +1,67 @@ +from typing import List, Dict, Any +import json +import sys +import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +from DataAccess.ObjectManager import ObjectManager + +class DBSubnetGroup: + + pk_column = 'db_subnet_group_name' + table_name = 'db_subnet_groups' + table_structure = f""" + db_subnet_group_name primary key not null, + db_subnet_group_description TEXT NOT NULL, + vpc_id VARCHAR(255) NOT NULL, + subnets JSONB DEFAULT '{{}}', + db_subnet_group_arn VARCHAR(255), + status VARCHAR(50) DEFAULT 'pending' + """ + + def __init__(self, **kwargs): + try: + self.db_subnet_group_name = kwargs['db_subnet_group_name'] + self.db_subnet_group_description = kwargs['db_subnet_group_description'] + self.vpc_id = kwargs['vpc_id'] + self.subnets = kwargs.get('subnets', None) + if not self.subnets: + self.subnets = dict() + self.db_subnet_group_arn = kwargs.get('db_subnet_group_arn', None) + + except KeyError as e: + raise ValueError(f"Missing required attribute for DBSubnetGroup: {str(e)}") + + # Ideally: + # self.db_subnet_group_arn should be dynamically created according to vpc-id, account-id and + # subnet-group-name, and then dynamically added to the routing table + + self.status = 'pending' + self.pk_value = self.db_subnet_group_name + + def to_dict(self) -> Dict[str, Any]: + return ObjectManager.convert_object_attributes_to_dictionary( + db_subnet_group_name=self.db_subnet_group_name, + db_subnet_group_description = self.db_subnet_group_description, + vpc_id = self.vpc_id, + subnets = self.subnets, + db_subnet_group_arn = self.db_subnet_group_arn, + status = self.status + ) + + def to_bytes(self): + bytes = json.dumps(self.to_dict()) + bytes = bytes.encode('utf-8') + return bytes + + def from_bytes_to_dict(bytes): + return json.loads(bytes.decode('utf-8')) + + + def to_sql(self): + # Convert the model instance to a dictionary + data_dict = self.to_dict() + values = '(' + ", ".join(f'\'{json.dumps(v)}\'' if isinstance(v, dict) or isinstance(v, list) else f'\'{v}\'' if isinstance(v, str) else f'\'{str(v)}\'' + for v in data_dict.values()) + ')' + return values \ No newline at end of file diff --git a/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py b/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py new file mode 100644 index 00000000..1c84896b --- /dev/null +++ b/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py @@ -0,0 +1,74 @@ +from typing import List, Dict, Any + +import sys +import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + + +from Models.DBSubnetGroupModel import DBSubnetGroup +from DataAccess.DBSubnetGroupManager import DBSubnetGroupManager +from Storage.KT_Storage.DataAccess.StorageManager import StorageManager +from Storage.KT_Storage.DataAccess.VersionManager import VersionManager +from Validation.GeneralValidations import * +class DBSubnetGroupService: + def __init__(self, db_subnet_group_manager: DBSubnetGroupManager): + self.manager = db_subnet_group_manager + self.bucket = 'db_subnet_groups' + self.storage_manager = StorageManager() + self.storage_manager.create_bucket(self.bucket) + self.version_manager = VersionManager() + + def create_db_subnet_group(self, **kwargs): + # object + if not kwargs.get('db_subnet_group_name'): + raise ValueError('Missing required argument db_subnet_group_name') + + if not is_length_in_range(kwargs['db_subnet_group_name'], 1, 255): + raise ValueError("invalid length for subnet group db_subnet_group_name: " + len(kwargs['db_subnet_group_name'])) + + if kwargs.get('description') and not is_length_in_range('description', 1, 255): + raise ValueError("invalid length for subnet group description: " + len(kwargs['description'])) + + subnet_group = DBSubnetGroup(**kwargs) + # physical object + # version = 0 assume created for the first time + self.storage_manager.create(self.bucket, subnet_group.db_subnet_group_name, subnet_group.to_bytes(), '0') + # save in memory + self.manager.create(subnet_group) + + def get_db_subnet_group(self, db_subnet_group_name: str) -> DBSubnetGroup: + data = self.manager.get(db_subnet_group_name) + return DBSubnetGroup(**data) + + def modify_db_subnet_group(self, db_subnet_group_name: str, updates: Dict[str, Any]) -> DBSubnetGroup: + if not db_subnet_group_name: + raise ValueError('Missing required argument db_subnet_group_name') + + + if updates.get('description') and not is_length_in_range(updates['description'], 1, 255): + raise ValueError("invalid length for subnet group description: " + len(updates['description'])) + + subnet_group = self.get_db_subnet_group(db_subnet_group_name) + + for key, value in updates.items(): + setattr(subnet_group, key, value) + + self.manager.modify(subnet_group) + # version = str(int(self.version_manager.get(self.bucket, subnet_group.db_subnet_group_name).version_id)+1) + # for now we override the basic version, when the latest version id can be retrieved, we will make a new version as old_version_id + 1 + self.storage_manager.create(self.bucket, db_subnet_group_name, subnet_group.to_bytes(), '0') + + def delete_db_subnet_group(self, db_subnet_group_name: str) -> None: + if not db_subnet_group_name: + raise ValueError('Missing required argument db_subnet_group_name') + + self.manager.delete(db_subnet_group_name) + self.storage_manager.delete_by_db_subnet_group_name(bucket_db_subnet_group_name=self.bucket, version_id=self.version_manager.get(self.bucket, db_subnet_group_name).version_id, key=db_subnet_group_name) + + def describe_db_subnet_group(self, db_subnet_group_name: str) -> Dict: + if not db_subnet_group_name: + raise ValueError('Missing required argument db_subnet_group_name') + + return self.manager.describe(db_subnet_group_name) \ No newline at end of file diff --git a/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py b/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py new file mode 100644 index 00000000..93fa618a --- /dev/null +++ b/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py @@ -0,0 +1,121 @@ +from sqlite3 import IntegrityError +import pytest +import json + +import sys +import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '.'))) + +from Service.Classes.DBSubnetGroupService import DBSubnetGroupService +from DataAccess.DBSubnetGroupManager import DBSubnetGroupManager +from Controller.DBSubnetGroupController import DBSubnetGroupController +from Storage.KT_Storage.DataAccess.StorageManager import StorageManager +from DataAccess.ObjectManager import ObjectManager +from DataAccess.DBManager import DBManager +from Models.DBSubnetGroupModel import DBSubnetGroup + +db_manager = DBManager('../object_management_db.db') +object_manager = ObjectManager('../object_management_db.db') +manager = DBSubnetGroupManager(db_manager=db_manager, object_manager=object_manager) +service = DBSubnetGroupService(manager) +controller = DBSubnetGroupController(service) +storage_manager = StorageManager() + +def test_create(): + controller.create_db_subnet_group( + db_subnet_group_name='subnet_group_1', + subnets=[ + {'subnet_id': 'subnet-12345678'}, + {'subnet_id': 'subnet-87654321'} + ], + db_subnet_group_description='Test subnet group', + vpc_id='vpc-12345678', + db_subnet_group_arn='arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' + ) + + # assert bucket was created + # check that file was created + assert storage_manager.get('db_subnet_groups', 'subnet_group_1', '0') != None + # check that object was saved to management table + assert controller.get_db_subnet_group('subnet_group_1') != None + # check that file content is correct + from_storage = DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', 'subnet_group_1', '0')['content']) + from_db = controller.get_db_subnet_group('subnet_group_1').to_dict() + assert from_storage['db_subnet_group_name'] == from_db['db_subnet_group_name'] + for subnet in from_storage['subnets']: + assert subnet in json.loads(from_db['subnets']) + assert from_storage['db_subnet_group_description'] == from_db['db_subnet_group_description'] + assert from_storage['vpc_id'] == from_db['vpc_id'] + assert from_storage['db_subnet_group_arn'] == from_db['db_subnet_group_arn'] + assert from_storage['status'] == from_db['status'] + +def test_unique_constraint(): + with pytest.raises(IntegrityError, match="UNIQUE constraint failed"): + controller.create_db_subnet_group( + db_subnet_group_name='subnet_group_1', + subnets=[ + {'subnet_id': 'subnet-87654321'}, + {'subnet_id': 'subnet-12345678'} + ], + db_subnet_group_description='Another subnet group with same name', + vpc_id='vpc-87654321', + db_subnet_group_arn='arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' + ) + +def test_modify(): + controller.modify_db_subnet_group( + name='subnet_group_1', + updates= {'subnets':[ + {'subnet_id': 'subnet-12345988'}, + {'subnet_id': 'subnet-876543881'}]} + ) + + from_storage = DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', 'subnet_group_1', '0')['content']) + from_db = controller.get_db_subnet_group('subnet_group_1') + assert from_storage['db_subnet_group_name'] == from_db.db_subnet_group_name + for subnet in from_storage['subnets']: + assert str(subnet) in from_db.subnets + assert from_storage['db_subnet_group_description'] == from_db.db_subnet_group_description + assert from_storage['vpc_id'] == from_db.vpc_id + assert from_storage['db_subnet_group_arn'] == from_db.db_subnet_group_arn + assert from_storage['status'] == from_db.status + +def test_get(): + subnet_group_1 = controller.get_db_subnet_group('subnet_group_1') + assert subnet_group_1 != None + assert subnet_group_1.db_subnet_group_name == 'subnet_group_1' + assert subnet_group_1.db_subnet_group_description == 'Test subnet group' + assert subnet_group_1.vpc_id == 'vpc-12345678' + assert subnet_group_1.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' + assert subnet_group_1.status == 'pending' + for subnet in json.loads(subnet_group_1.subnets): + assert subnet in [ + {"subnet_id": "subnet-12345678"}, + {"subnet_id": "subnet-87654321"} + ] + + +def test_describe(): + subnet_group_1 = controller.describe_db_subnet_group('subnet_group_1') + assert subnet_group_1 != None + assert subnet_group_1.db_subnet_group_name == 'subnet_group_1' + assert subnet_group_1.db_subnet_group_description == 'Test subnet group' + assert subnet_group_1.vpc_id == 'vpc-12345678' + assert subnet_group_1.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' + assert subnet_group_1.status == 'pending' + for subnet in json.loads(subnet_group_1.subnets): + assert subnet in json.dumps([ + {'subnet_id': 'subnet-12345678'}, + {'subnet_id': 'subnet-87654321'} + ]) + + +def test_delete(): + controller.delete_db_subnet_group('subnet_group_1') + assert storage_manager.get('db_subnet_groups', 'subnet_group_1', '0') == None + assert controller.get_db_subnet_group('subnet_group_1') == None + + + \ No newline at end of file From 7c103e8ff62a54f0266e59d065a1a8f8bdc6483f Mon Sep 17 00:00:00 2001 From: Tem-M Date: Mon, 16 Sep 2024 11:18:33 +0300 Subject: [PATCH 02/16] tests pass --- .../DataAccess/DBSubnetGroupManager.py | 26 +-- DB/NEW_KT_DB/Models/DBSubnetGroupModel.py | 5 +- .../Service/Classes/DBSubnetGroupService.py | 28 ++- DB/NEW_KT_DB/Test/DBSubnetGroupTests.py | 187 +++++++++++++----- 4 files changed, 183 insertions(+), 63 deletions(-) diff --git a/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py b/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py index d6cc671d..d601ff4f 100644 --- a/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py @@ -8,27 +8,31 @@ from Models.DBSubnetGroupModel import DBSubnetGroup class DBSubnetGroupManager: - def __init__(self, db_manager, object_manager): - self.db_manager = db_manager + def __init__(self, object_manager): self.object_manager = object_manager - self.db_manager.create_table(DBSubnetGroup.table_name, DBSubnetGroup.table_structure) + self.object_manager.create_management_table(DBSubnetGroup.table_name, DBSubnetGroup.table_structure) def create(self, subnet_group: DBSubnetGroup): - self.object_manager.save_in_memory(DBSubnetGroup.table_name, subnet_group.to_sql()) + self.object_manager.insert_object_to_management_table(DBSubnetGroup.table_name, subnet_group.to_sql()) def get(self, name: str): - cols = ['db_subnet_group_name', 'db_subnet_group_description', 'vpc_id', 'subnets', 'db_subnet_group_arn', 'status'] - data = self.object_manager.get_from_memory(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, name, cols = cols) - return data + data = self.object_manager.get_object_from_management_table(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, name) + if data: + data_mapping = {'db_subnet_group_name':name} + for key, value in data[name].items(): + data_mapping[key] = value + return DBSubnetGroup(**data_mapping) + else: + raise ValueError(f"subnet group with name '{name}' not found") + def delete(self, name: str): - self.object_manager.delete_from_memory(name) + self.object_manager.delete_object_from_management_table(DBSubnetGroup.table_name, f"{DBSubnetGroup.pk_column}='{name}'") def describe(self, name: str): - cols = ['db_subnet_group_name', 'db_subnet_group_description', 'vpc_id', 'subnets', 'db_subnet_group_arn', 'status'] - return self.object_manager.get_from_memory(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, name, cols) + return self.get(name).to_dict() def modify(self, subnet_group: DBSubnetGroup): updates = subnet_group.to_dict() del updates['db_subnet_group_name'] - self.object_manager.update_in_memory(DBSubnetGroup.pk_column, subnet_group.db_subnet_group_name, DBSubnetGroup.table_name, updates) \ No newline at end of file + self.object_manager.update_object_in_management_table_by_criteria(table_name = DBSubnetGroup.table_name, updates = updates, criteria = f"{DBSubnetGroup.pk_column}='{subnet_group.db_subnet_group_name}'") \ No newline at end of file diff --git a/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py b/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py index 9b933bea..ccb0bff7 100644 --- a/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py +++ b/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py @@ -1,4 +1,5 @@ from typing import List, Dict, Any +import ast import json import sys import os @@ -27,7 +28,9 @@ def __init__(self, **kwargs): self.vpc_id = kwargs['vpc_id'] self.subnets = kwargs.get('subnets', None) if not self.subnets: - self.subnets = dict() + self.subnets = [] + if type(self.subnets) is not list: + self.subnets = ast.literal_eval(self.subnets) self.db_subnet_group_arn = kwargs.get('db_subnet_group_arn', None) except KeyError as e: diff --git a/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py b/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py index 1c84896b..0c42b29e 100644 --- a/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py +++ b/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py @@ -1,3 +1,4 @@ +from sqlite3 import IntegrityError from typing import List, Dict, Any import sys @@ -19,6 +20,7 @@ def __init__(self, db_subnet_group_manager: DBSubnetGroupManager): self.storage_manager = StorageManager() self.storage_manager.create_bucket(self.bucket) self.version_manager = VersionManager() + self.subnet_groups = dict() def create_db_subnet_group(self, **kwargs): # object @@ -28,19 +30,29 @@ def create_db_subnet_group(self, **kwargs): if not is_length_in_range(kwargs['db_subnet_group_name'], 1, 255): raise ValueError("invalid length for subnet group db_subnet_group_name: " + len(kwargs['db_subnet_group_name'])) + if kwargs['db_subnet_group_name'] in self.subnet_groups: + raise ValueError(f"db_subnet_group_name {kwargs['db_subnet_group_name']} already exists") + if kwargs.get('description') and not is_length_in_range('description', 1, 255): raise ValueError("invalid length for subnet group description: " + len(kwargs['description'])) subnet_group = DBSubnetGroup(**kwargs) + # save in management table + # in try except block in case the server was shut down and re-run and local collection doesn't include all subnetGroups + try: + self.manager.create(subnet_group) + except IntegrityError as e: + raise ValueError(f"db_subnet_group_name {kwargs['db_subnet_group_name']} already exists") + # physical object # version = 0 assume created for the first time self.storage_manager.create(self.bucket, subnet_group.db_subnet_group_name, subnet_group.to_bytes(), '0') - # save in memory - self.manager.create(subnet_group) + # save in local collection (hash table) for quick access + self.subnet_groups[kwargs['db_subnet_group_name']] = subnet_group def get_db_subnet_group(self, db_subnet_group_name: str) -> DBSubnetGroup: data = self.manager.get(db_subnet_group_name) - return DBSubnetGroup(**data) + return data def modify_db_subnet_group(self, db_subnet_group_name: str, updates: Dict[str, Any]) -> DBSubnetGroup: if not db_subnet_group_name: @@ -63,9 +75,15 @@ def modify_db_subnet_group(self, db_subnet_group_name: str, updates: Dict[str, A def delete_db_subnet_group(self, db_subnet_group_name: str) -> None: if not db_subnet_group_name: raise ValueError('Missing required argument db_subnet_group_name') - + + # delete from management table self.manager.delete(db_subnet_group_name) - self.storage_manager.delete_by_db_subnet_group_name(bucket_db_subnet_group_name=self.bucket, version_id=self.version_manager.get(self.bucket, db_subnet_group_name).version_id, key=db_subnet_group_name) + # for now version id is 0 + # delete physical object from storage + self.storage_manager.delete_by_name(bucket_name=self.bucket, version_id='0', key=db_subnet_group_name) + # delete from local collection (hash table) + if db_subnet_group_name in self.subnet_groups: + del self.subnet_groups[db_subnet_group_name] def describe_db_subnet_group(self, db_subnet_group_name: str) -> Dict: if not db_subnet_group_name: diff --git a/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py b/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py index 93fa618a..8dc69fcb 100644 --- a/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py +++ b/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py @@ -13,17 +13,23 @@ from Controller.DBSubnetGroupController import DBSubnetGroupController from Storage.KT_Storage.DataAccess.StorageManager import StorageManager from DataAccess.ObjectManager import ObjectManager -from DataAccess.DBManager import DBManager from Models.DBSubnetGroupModel import DBSubnetGroup +import sqlite3 -db_manager = DBManager('../object_management_db.db') object_manager = ObjectManager('../object_management_db.db') -manager = DBSubnetGroupManager(db_manager=db_manager, object_manager=object_manager) +manager = DBSubnetGroupManager(object_manager=object_manager) service = DBSubnetGroupService(manager) controller = DBSubnetGroupController(service) storage_manager = StorageManager() def test_create(): + + # remove existing subnet group from previous tests + conn = sqlite3.connect('../object_management_db.db') + conn.execute("delete from db_subnet_groups where db_subnet_group_name = 'subnet_group_1'") + conn.commit() + conn.close() + controller.create_db_subnet_group( db_subnet_group_name='subnet_group_1', subnets=[ @@ -35,24 +41,47 @@ def test_create(): db_subnet_group_arn='arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' ) - # assert bucket was created - # check that file was created - assert storage_manager.get('db_subnet_groups', 'subnet_group_1', '0') != None - # check that object was saved to management table - assert controller.get_db_subnet_group('subnet_group_1') != None + # check that file was created (no error raised on get) + storage_manager.get('db_subnet_groups', 'subnet_group_1', '0') + # check that object was saved to management table (no error raised on get) + controller.get_db_subnet_group('subnet_group_1') + # check that file content is correct - from_storage = DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', 'subnet_group_1', '0')['content']) - from_db = controller.get_db_subnet_group('subnet_group_1').to_dict() - assert from_storage['db_subnet_group_name'] == from_db['db_subnet_group_name'] - for subnet in from_storage['subnets']: - assert subnet in json.loads(from_db['subnets']) - assert from_storage['db_subnet_group_description'] == from_db['db_subnet_group_description'] - assert from_storage['vpc_id'] == from_db['vpc_id'] - assert from_storage['db_subnet_group_arn'] == from_db['db_subnet_group_arn'] - assert from_storage['status'] == from_db['status'] + from_storage = DBSubnetGroup(**DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', 'subnet_group_1', '0')['content'])) + from_db = controller.get_db_subnet_group('subnet_group_1') + # check values were stored correctly in management table as well as storage + + # group name + assert from_storage.db_subnet_group_name == 'subnet_group_1' + assert from_db.db_subnet_group_name == 'subnet_group_1' + + # subnets + for subnet in from_storage.subnets: + assert subnet in [ + {'subnet_id': 'subnet-12345678'}, + {'subnet_id': 'subnet-87654321'} + ] + for subnet in from_db.subnets: + assert subnet in [ + {'subnet_id': 'subnet-12345678'}, + {'subnet_id': 'subnet-87654321'} + ] + + # description + assert from_storage.db_subnet_group_description == 'Test subnet group' + assert from_db.db_subnet_group_description == 'Test subnet group' + + # vpc_id + assert from_storage.vpc_id == 'vpc-12345678' + assert from_db.vpc_id == 'vpc-12345678' + + # arn + assert from_storage.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' + assert from_db.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' + def test_unique_constraint(): - with pytest.raises(IntegrityError, match="UNIQUE constraint failed"): + with pytest.raises(ValueError): controller.create_db_subnet_group( db_subnet_group_name='subnet_group_1', subnets=[ @@ -63,24 +92,7 @@ def test_unique_constraint(): vpc_id='vpc-87654321', db_subnet_group_arn='arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' ) - -def test_modify(): - controller.modify_db_subnet_group( - name='subnet_group_1', - updates= {'subnets':[ - {'subnet_id': 'subnet-12345988'}, - {'subnet_id': 'subnet-876543881'}]} - ) - - from_storage = DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', 'subnet_group_1', '0')['content']) - from_db = controller.get_db_subnet_group('subnet_group_1') - assert from_storage['db_subnet_group_name'] == from_db.db_subnet_group_name - for subnet in from_storage['subnets']: - assert str(subnet) in from_db.subnets - assert from_storage['db_subnet_group_description'] == from_db.db_subnet_group_description - assert from_storage['vpc_id'] == from_db.vpc_id - assert from_storage['db_subnet_group_arn'] == from_db.db_subnet_group_arn - assert from_storage['status'] == from_db.status + def test_get(): subnet_group_1 = controller.get_db_subnet_group('subnet_group_1') @@ -90,32 +102,115 @@ def test_get(): assert subnet_group_1.vpc_id == 'vpc-12345678' assert subnet_group_1.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' assert subnet_group_1.status == 'pending' - for subnet in json.loads(subnet_group_1.subnets): + for subnet in subnet_group_1.subnets: assert subnet in [ {"subnet_id": "subnet-12345678"}, {"subnet_id": "subnet-87654321"} ] - +def test_modify(): + controller.modify_db_subnet_group( + name='subnet_group_1', + updates= {'subnets':[ + {'subnet_id': 'subnet-12345988'}, + {'subnet_id': 'subnet-876543881'}]} + ) + + from_storage = DBSubnetGroup(**DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', 'subnet_group_1', '0')['content'])) + from_db = controller.get_db_subnet_group('subnet_group_1') + assert from_storage.db_subnet_group_name == from_db.db_subnet_group_name + for subnet in from_storage.subnets: + assert subnet in from_db.subnets + assert from_storage.db_subnet_group_description == from_db.db_subnet_group_description + assert from_storage.vpc_id == from_db.vpc_id + assert from_storage.db_subnet_group_arn == from_db.db_subnet_group_arn + assert from_storage.status == from_db.status + def test_describe(): subnet_group_1 = controller.describe_db_subnet_group('subnet_group_1') + assert type(subnet_group_1) == dict + assert type(subnet_group_1['subnets']) == list + assert type(subnet_group_1['subnets'][0]) == dict + subnet_group_1 = DBSubnetGroup(**subnet_group_1) assert subnet_group_1 != None assert subnet_group_1.db_subnet_group_name == 'subnet_group_1' assert subnet_group_1.db_subnet_group_description == 'Test subnet group' assert subnet_group_1.vpc_id == 'vpc-12345678' assert subnet_group_1.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' assert subnet_group_1.status == 'pending' - for subnet in json.loads(subnet_group_1.subnets): - assert subnet in json.dumps([ - {'subnet_id': 'subnet-12345678'}, - {'subnet_id': 'subnet-87654321'} - ]) + for subnet in subnet_group_1.subnets: + assert subnet in [ + {'subnet_id': 'subnet-12345988'}, + {'subnet_id': 'subnet-876543881'} + ] def test_delete(): controller.delete_db_subnet_group('subnet_group_1') - assert storage_manager.get('db_subnet_groups', 'subnet_group_1', '0') == None - assert controller.get_db_subnet_group('subnet_group_1') == None + with pytest.raises(FileNotFoundError): + storage_manager.get('db_subnet_groups', 'subnet_group_1', '0') + with pytest.raises(Exception): + controller.get_db_subnet_group('subnet_group_1') + + +@pytest.mark.parametrize("index", range(20)) +def test_insert_many(index): + controller.create_db_subnet_group( + db_subnet_group_name=f'subnet_group_{index}', + subnets=[ + {'subnet_id': f'subnet-1234567{index}'}, + {'subnet_id': f'subnet-8765432{index}'} + ], + db_subnet_group_description=f'Test subnet group {index}', + vpc_id=f'vpc-1234567{index}', + db_subnet_group_arn=f'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_{index}' + ) + + # check that file was created (no error raised on get) + storage_manager.get('db_subnet_groups', f'subnet_group_{index}', '0') + # check that object was saved to management table (no error raised on get) + controller.get_db_subnet_group(f'subnet_group_{index}') + + # check that file content is correct + from_storage = DBSubnetGroup(**DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', f'subnet_group_{index}', '0')['content'])) + from_db = controller.get_db_subnet_group(f'subnet_group_{index}') + # check values were stored correctly in management table as well as storage + + # group name + assert from_storage.db_subnet_group_name == f'subnet_group_{index}' + assert from_db.db_subnet_group_name == f'subnet_group_{index}' + + # subnets + for subnet in from_storage.subnets: + assert subnet in [ + {'subnet_id': f'subnet-1234567{index}'}, + {'subnet_id': f'subnet-8765432{index}'} + ] + for subnet in from_db.subnets: + assert subnet in [ + {'subnet_id': f'subnet-1234567{index}'}, + {'subnet_id': f'subnet-8765432{index}'} + ] + + # description + assert from_storage.db_subnet_group_description == f'Test subnet group {index}' + assert from_db.db_subnet_group_description == f'Test subnet group {index}' + + # vpc_id + assert from_storage.vpc_id == f'vpc-1234567{index}' + assert from_db.vpc_id == f'vpc-1234567{index}' + + # arn + assert from_storage.db_subnet_group_arn == f'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_{index}' + assert from_db.db_subnet_group_arn == f'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_{index}' + - \ No newline at end of file +@pytest.mark.parametrize("index", range(20)) +def test_delete_many_from_prev_test(index): + db_subnet_group_name=f'subnet_group_{index}' + controller.delete_db_subnet_group(db_subnet_group_name) + with pytest.raises(FileNotFoundError): + storage_manager.get('db_subnet_groups', db_subnet_group_name, '0') + with pytest.raises(Exception): + controller.get_db_subnet_group(db_subnet_group_name) From 2f5fdf8aac752d95625a51244ed370f0ac3549ff Mon Sep 17 00:00:00 2001 From: Tem-M Date: Mon, 16 Sep 2024 12:09:43 +0300 Subject: [PATCH 03/16] add cleanup fixture to tests --- DB/NEW_KT_DB/Test/DBSubnetGroupTests.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py b/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py index 8dc69fcb..b2fe7674 100644 --- a/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py +++ b/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py @@ -21,15 +21,27 @@ service = DBSubnetGroupService(manager) controller = DBSubnetGroupController(service) storage_manager = StorageManager() - -def test_create(): - - # remove existing subnet group from previous tests - conn = sqlite3.connect('../object_management_db.db') - conn.execute("delete from db_subnet_groups where db_subnet_group_name = 'subnet_group_1'") + +@pytest.fixture +def clear_table(): + # Connect to the SQLite database + conn = conn = sqlite3.connect('../object_management_db.db') + cursor = conn.cursor() + + # Clear the table if it exists + table_name = "db_subnet_groups" + cursor.execute(f"DELETE FROM {table_name};") + + # Commit changes and close the connection conn.commit() conn.close() + + # Yield to allow tests to run + yield + +def test_create(clear_table): + # remove existing subnet group from previous tests controller.create_db_subnet_group( db_subnet_group_name='subnet_group_1', subnets=[ From b7b97a3e576873342b201a05ca78e1cc4af76326 Mon Sep 17 00:00:00 2001 From: Tem-M Date: Mon, 16 Sep 2024 12:22:19 +0300 Subject: [PATCH 04/16] technical commit --- DB/NEW_KT_DB/DataAccess/DBManager.py | 80 ++++++++++--------- DB/NEW_KT_DB/DataAccess/ObjectManager.py | 16 ++-- .../KT_Storage/DataAccess/StorageManager.py | 10 +-- .../KT_Storage/DataAccess/VersionManager.py | 10 ++- Storage/KT_Storage/Models/VesionModel.py | 4 +- 5 files changed, 66 insertions(+), 54 deletions(-) diff --git a/DB/NEW_KT_DB/DataAccess/DBManager.py b/DB/NEW_KT_DB/DataAccess/DBManager.py index dd0dde58..79a4ec0d 100644 --- a/DB/NEW_KT_DB/DataAccess/DBManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBManager.py @@ -11,21 +11,21 @@ def __init__(self, db_file: str): # rachel-8511, ShaniStrassProg def close(self): - '''Close the database connection.''' - self.connection.close() + '''Close the database connection.''' + self.connection.close() # saraNoigershel def execute_query_with_multiple_results(self, query: str) -> Optional[List[Tuple]]: - '''Execute a given query and return the results.''' - try: - c = self.connection.cursor() - c.execute(query) - results = c.fetchall() - # self.connection.commit() ??? - return results if results else None - except OperationalError as e: - raise Exception(f'Error executing query {query}: {e}') + '''Execute a given query and return the results.''' + try: + c = self.connection.cursor() + c.execute(query) + results = c.fetchall() + # self.connection.commit() ??? + return results if results else None + except OperationalError as e: + raise Exception(f'Error executing query {query}: {e}') # ShaniStrassProg @@ -56,42 +56,41 @@ def execute_query_without_results(self, query: str): def create_table(self, table_name, table_structure): '''create a table in a given db by given table_structure''' create_statement = f'''CREATE TABLE IF NOT EXISTS {table_name} ({table_structure})''' - execute_query_without_results(create_statement) + self.execute_query_without_results(create_statement) # Riki7649255 based on rachel-8511, ShaniStrassProg def insert_data_into_table(self, table_name, data): insert_statement = f'''INSERT INTO {table_name} VALUES {data}''' - execute_query_without_results(insert_statement) + self.execute_query_without_results(insert_statement) # Riki7649255 based on rachel-8511, Shani def update_records_in_table(self, table_name: str, updates: Dict[str, Any], criteria: str) -> None: - '''Update records in the specified table based on criteria.''' - - # add documentation here - set_clause = ', '.join([f'{k} = ?' for k in updates.keys()]) - values = list(updates.values()) - - update_statement = f''' - UPDATE {table_name} - SET {set_clause} - WHERE {criteria} - ''' - - execute_query_without_results(update_statement) + '''Update records in the specified table based on criteria.''' + + # add documentation here + set_clause = '(' + ', '.join([f'{k}' for k in updates.keys()]) + ') = (' + ', '.join([f'"{v}"' for v in updates.values()]) + ')' + + update_statement = f''' + UPDATE {table_name} + SET {set_clause} + WHERE {criteria} + ''' + + self.execute_query_without_results(update_statement) # Riki7649255 based on rachel-8511 def delete_data_from_table(self, table_name: str, criteria: str) -> None: - '''Delete a record from the specified table based on criteria.''' - - delete_statement = f''' - DELETE FROM {table_name} - WHERE {criteria} - ''') - - execute_query_without_results(delete_statement) + '''Delete a record from the specified table based on criteria.''' + + delete_statement = f''' + DELETE FROM {table_name} + WHERE {criteria} + ''' + + self.execute_query_without_results(delete_statement) # rachel-8511, Riki7649255 @@ -104,14 +103,21 @@ def select_and_return_records_from_table(self, table_name: str, columns: List[st Returns: Dict[int, Dict[str, Any]]: A dictionary where keys are object_ids and values are metadata. ''' + + if columns == ['*']: + columns = [res[1] for res in self.connection.execute(f'PRAGMA table_info({table_name});').fetchall()] + columns_clause = ', '.join(columns) query = f'SELECT {columns_clause} FROM {table_name}' if criteria: - query += f' WHERE {criteria}' + query += f' WHERE {criteria};' + try: - results = execute_query_with_multiple_results(query) - return {result[0]: dict(zip(columns, result[1:])) for result in results} + results = self.execute_query_with_multiple_results(query) + return {result[0]: dict(zip(columns[1:], result[1:])) for result in results} except OperationalError as e: + raise Exception(f'Error selecting from {table_name}: {e}') + except TypeError as e: raise Exception(f'Error selecting from {table_name}: {e}') diff --git a/DB/NEW_KT_DB/DataAccess/ObjectManager.py b/DB/NEW_KT_DB/DataAccess/ObjectManager.py index 56c6948f..21baa349 100644 --- a/DB/NEW_KT_DB/DataAccess/ObjectManager.py +++ b/DB/NEW_KT_DB/DataAccess/ObjectManager.py @@ -1,7 +1,7 @@ from typing import Dict, Any import json import sqlite3 -from DBManager import DBManager +from .DBManager import DBManager class ObjectManager: def __init__(self, db_file: str): @@ -12,7 +12,7 @@ def __init__(self, db_file: str): # for internal use only: # Riki7649255 based on rachel-8511 - def create_management_table(self, table_name, table_structure='object_id INTEGER PRIMARY KEY AUTOINCREMENT,type_object TEXT NOT NULL,metadata TEXT NOT NULL') + def create_management_table(self, table_name, table_structure='object_id INTEGER PRIMARY KEY AUTOINCREMENT,type_object TEXT NOT NULL,metadata TEXT NOT NULL'): self.db_manager.create_table(table_name, table_structure) @@ -27,11 +27,11 @@ def update_object_in_management_table_by_criteria(self, table_name, updates, cri # rachel-8511, Riki7649255 - def get_object_from_management_table(self, object_id: int) -> Dict[str, Any]: + def get_object_from_management_table(self, pk_col, table_name, object_id: int) -> Dict[str, Any]: '''Retrieve an object from the database.''' - result = self.db_manager.select_and_return_records_from_table(self.table_name, ['type_object', 'metadata'], f'object_id = {object_id}') + result = self.db_manager.select_and_return_records_from_table(table_name=table_name, criteria=f'{pk_col} = \'{object_id}\'') if result: - return result[object_id] + return result else: raise FileNotFoundError(f'Object with ID {object_id} not found.') @@ -68,7 +68,7 @@ def save_in_memory(self, object): # insert object info into management table mng_{object_name}s # for exmple: object db_instance will be saved in table mng_db_instances - table_name = convert_object_name_to_management_table_name(self.object_name) + table_name = ObjectManager.convert_object_name_to_management_table_name(self.object_name) if not is_management_table_exist(table_name): create_management_table(table_name) @@ -82,7 +82,7 @@ def delete_from_memory(self,criteria='default'): if criteria == 'default': criteria = f'{self.pk_column} = {self.pk_value}' - table_name = convert_object_name_to_management_table_name(self.object_name) + table_name = ObjectManager.convert_object_name_to_management_table_name(self.object_name) delete_data_from_table(table_name, criteria) @@ -93,7 +93,7 @@ def update_in_memory(self, updates, criteria='default'): if criteria == 'default': criteria = f'{self.pk_column} = {self.pk_value}' - table_name = convert_object_name_to_management_table_name(self.object_name) + table_name = ObjectManager.convert_object_name_to_management_table_name(self.object_name) update_object_in_management_table_by_criteria(table_name, updates, criteria) diff --git a/Storage/KT_Storage/DataAccess/StorageManager.py b/Storage/KT_Storage/DataAccess/StorageManager.py index eee45534..94f7e5d2 100644 --- a/Storage/KT_Storage/DataAccess/StorageManager.py +++ b/Storage/KT_Storage/DataAccess/StorageManager.py @@ -3,9 +3,9 @@ import os import aiofiles import shutil -from Crypto.Cipher import AES -from Crypto.Util.Padding import pad -import base6import +from Cryptodome.Cipher import AES +from Cryptodome.Util.Padding import pad +# import base6import URL_SERVER = 's3/KT_cloud/Storage/server' @@ -48,7 +48,7 @@ def get(self, bucket, key , version_id) -> Dict[str, Any]: file_path = os.path.join(self.server_path, bucket, versioned_file_name) if not os.path.exists(file_path): - return {'error': 'File not found'} + raise FileNotFoundError(f"File '{key}' with version '{version_id}' not found in bucket '{bucket}'.") if os.path.isdir(file_path): # If the object is a directory, return its metadata and list of contents @@ -76,7 +76,7 @@ def get(self, bucket, key , version_id) -> Dict[str, Any]: def delete_by_name(self, bucket_name, version_id, key) -> None: """Delete a specified file or directory by name in a bucket and version.""" file_name, file_extension = os.path.splitext(key) - file_name_path = f"{file_name}{file_extension}" + file_name_path = f"{file_name}.v{version_id}{file_extension}" file_path = os.path.join(self.server_path, bucket_name, file_name_path) if os.path.exists(file_path): diff --git a/Storage/KT_Storage/DataAccess/VersionManager.py b/Storage/KT_Storage/DataAccess/VersionManager.py index dfccf931..d7aca55c 100644 --- a/Storage/KT_Storage/DataAccess/VersionManager.py +++ b/Storage/KT_Storage/DataAccess/VersionManager.py @@ -1,10 +1,16 @@ -from DataAccess import StorageManager +# from DataAccess import StorageManager from typing import Dict, Any import json import aiofiles import os -from Models.VesionModel import Version +import sys +import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '.'))) + +from ..Models.VesionModel import Version from .StorageManager import StorageManager class VersionManager: def __init__(self, metadata_file="s3 project/KT_Cloud/Storage/server/metadata.json"): diff --git a/Storage/KT_Storage/Models/VesionModel.py b/Storage/KT_Storage/Models/VesionModel.py index 6bbf2ad5..5cdb8b00 100644 --- a/Storage/KT_Storage/Models/VesionModel.py +++ b/Storage/KT_Storage/Models/VesionModel.py @@ -1,7 +1,7 @@ from datetime import datetime import hashlib -from AclModel import Acl -from Tag import Tag +from .AclModel import Acl +from .Tag import Tag # domain/versioning.py From a7c69c7234756cb11a66aa8804828cff40d62183 Mon Sep 17 00:00:00 2001 From: Tem-M Date: Mon, 16 Sep 2024 14:38:01 +0300 Subject: [PATCH 05/16] fix db_manager and object_manager --- DB/NEW_KT_DB/DataAccess/DBManager.py | 32 +++++-- .../DataAccess/DBSubnetGroupManager.py | 10 +-- DB/NEW_KT_DB/DataAccess/ObjectManager.py | 83 ++++++++++--------- DB/NEW_KT_DB/Models/DBSubnetGroupModel.py | 1 + .../Service/Classes/DBSubnetGroupService.py | 3 +- 5 files changed, 76 insertions(+), 53 deletions(-) diff --git a/DB/NEW_KT_DB/DataAccess/DBManager.py b/DB/NEW_KT_DB/DataAccess/DBManager.py index c1d7580f..08daddf0 100644 --- a/DB/NEW_KT_DB/DataAccess/DBManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBManager.py @@ -67,7 +67,7 @@ def update_records_in_table(self, table_name: str, updates: Dict[str, Any], crit # add documentation here set_clause = ', '.join([f'{k} = ?' for k in updates.keys()]) - values = list(updates.values()) + values = tuple(updates.values()) update_statement = f''' UPDATE {table_name} @@ -90,7 +90,25 @@ def delete_data_from_table(self, table_name: str, criteria: str) -> None: self.execute_query_without_results(delete_statement) - + # Tem-M + def get_columns_from_table(self, table_name): + '''Get the columns from the specified table.''' + try: + get_columns_query = f"""PRAGMA table_info({table_name});""" + cols = self.execute_query_with_multiple_results(get_columns_query) + return [col[1] for col in cols] + except Exception as e: + print(f"Error occurred while fetching columns from table {table_name}: {e}") + return [] + + def get_all_data_from_table(self, table_name): + try: + get_all_data_query = f"""SELECT * FROM {table_name}""" + return self.execute_query_with_multiple_results(get_all_data_query) + except Exception as e: + print(f"Error occurred while fetching data from table {table_name}: {e}") + return [] + # rachel-8511, Riki7649255 def select_and_return_records_from_table(self, table_name: str, columns: List[str] = ['*'], criteria: Optional[str] = None) -> Dict[int, Dict[str, Any]]: '''Select records from the specified table based on criteria. @@ -101,18 +119,18 @@ def select_and_return_records_from_table(self, table_name: str, columns: List[st Returns: Dict[int, Dict[str, Any]]: A dictionary where keys are object_ids and values are metadata. ''' - - if columns == ['*']: - columns = [res[1] for res in self.connection.execute(f'PRAGMA table_info({table_name});').fetchall()] + cols = columns + if cols == ['*']: + cols = self.get_columns_from_table(table_name) - columns_clause = ', '.join(columns) + columns_clause = ', '.join(cols) query = f'SELECT {columns_clause} FROM {table_name}' if criteria: query += f' WHERE {criteria};' try: results = self.execute_query_with_multiple_results(query) - return {result[0]: dict(zip(columns, result[1:])) for result in results} + return {result[0]: dict(zip(cols if columns != ['*'] else cols[1:], result[1:])) for result in results} except OperationalError as e: raise Exception(f'Error selecting from {table_name}: {e}') except TypeError as e: diff --git a/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py b/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py index d601ff4f..68c8af52 100644 --- a/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py @@ -10,13 +10,13 @@ class DBSubnetGroupManager: def __init__(self, object_manager): self.object_manager = object_manager - self.object_manager.create_management_table(DBSubnetGroup.table_name, DBSubnetGroup.table_structure) + self.object_manager._create_management_table(DBSubnetGroup.table_name, DBSubnetGroup.table_structure) def create(self, subnet_group: DBSubnetGroup): - self.object_manager.insert_object_to_management_table(DBSubnetGroup.table_name, subnet_group.to_sql()) + self.object_manager.save_in_memory(DBSubnetGroup.table_name, subnet_group) def get(self, name: str): - data = self.object_manager.get_object_from_management_table(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, name) + data = self.object_manager.get_from_memory_by_id(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, name) if data: data_mapping = {'db_subnet_group_name':name} for key, value in data[name].items(): @@ -27,7 +27,7 @@ def get(self, name: str): def delete(self, name: str): - self.object_manager.delete_object_from_management_table(DBSubnetGroup.table_name, f"{DBSubnetGroup.pk_column}='{name}'") + self.object_manager.delete_from_memory_by_id(DBSubnetGroup.pk_column, name, DBSubnetGroup.table_name) def describe(self, name: str): return self.get(name).to_dict() @@ -35,4 +35,4 @@ def describe(self, name: str): def modify(self, subnet_group: DBSubnetGroup): updates = subnet_group.to_dict() del updates['db_subnet_group_name'] - self.object_manager.update_object_in_management_table_by_criteria(table_name = DBSubnetGroup.table_name, updates = updates, criteria = f"{DBSubnetGroup.pk_column}='{subnet_group.db_subnet_group_name}'") \ No newline at end of file + self.object_manager.update_in_memory_by_id(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, updates, subnet_group.db_subnet_group_name) diff --git a/DB/NEW_KT_DB/DataAccess/ObjectManager.py b/DB/NEW_KT_DB/DataAccess/ObjectManager.py index 46400dfe..ec3b2ece 100644 --- a/DB/NEW_KT_DB/DataAccess/ObjectManager.py +++ b/DB/NEW_KT_DB/DataAccess/ObjectManager.py @@ -12,28 +12,39 @@ def __init__(self, db_file: str): # for internal use only: # Riki7649255 based on rachel-8511 - def create_management_table(self, table_name, table_structure='object_id INTEGER PRIMARY KEY AUTOINCREMENT,type_object TEXT NOT NULL,metadata TEXT NOT NULL'): + def _create_management_table(self, table_name, table_structure='object_id INTEGER PRIMARY KEY AUTOINCREMENT,type_object TEXT NOT NULL,metadata TEXT NOT NULL'): + """ + creates a management table with the name and the structure you specify + make sure to keep track of the table name you send here - you will use it whenever you want to access the table + you created - this function should only be called from within the specific manager you created (i.e. DBInstanceManager) + """ self.db_manager.create_table(table_name, table_structure) - # Riki7649255 based on saraNoigershel - def insert_object_to_management_table(self, table_name, object): - self.db_manager.insert_data_into_table(table_name, object) - - - + # Riki7649255 based on saraNoigershel, Tem-M + def _insert_object_to_management_table(self, table_name, object): + """ + inserts an object to the management table you specified, the object should be sent as is! not converted to a tuple or dictionary! + if the table does not exist, the function will abort and raise an error + the table should be created within the __init__ function of the manager you created (i.e. DBInstanceManager) + """ + columns = self.db_manager.get_columns_from_table(table_name) + values = tuple([str(getattr(object, column)) for column in columns]) + self.db_manager.insert_data_into_table(table_name, columns, values) + + # Malki1844 + def _get_all_data_from_table(self, table_name): + self.db_manager.get_all_data_from_table(table_name) # Riki7649255 based on rachel-8511 - def update_object_in_management_table_by_criteria(self, table_name, updates, criteria): + def _update_object_in_management_table_by_criteria(self, table_name, updates, criteria): + updates = {k: str(v) for k, v in updates.items()} self.db_manager.update_records_in_table(table_name, updates, criteria) - def update_object_in_management_table_by_id(self, pk_col, table_name, object_id, updates): - self.db_manager.update_records_in_table(table_name, updates, f'{pk_col} = \'{object_id}\'') - # rachel-8511, Riki7649255 - def get_object_from_management_table(self, pk_col, table_name, object_id: int) -> Dict[str, Any]: + def _get_object_from_management_table(self, pk_col, table_name, object_id: int) -> Dict[str, Any]: '''Retrieve an object from the database.''' result = self.db_manager.select_and_return_records_from_table(table_name=table_name, criteria=f'{pk_col} = \'{object_id}\'') if result: @@ -41,9 +52,9 @@ def get_object_from_management_table(self, pk_col, table_name, object_id: int) - else: raise FileNotFoundError(f'Object with ID {object_id} not found.') - def get_objects_from_management_table_by_criteria(self, object_id: int, columns = ["*"], criteria:Optional[str] = None) -> Dict: + def _get_objects_from_management_table_by_criteria(self, table_name, columns = ["*"], criteria:Optional[str] = None) -> Dict: '''Retrieve an object from the database.''' - result = self.db_manager.select_and_return_records_from_table(self.table_name, columns, criteria) + result = self.db_manager.select_and_return_records_from_table(table_name, columns, criteria) if result: return result else: @@ -51,13 +62,13 @@ def get_objects_from_management_table_by_criteria(self, object_id: int, columns # rachel-8511, ShaniStrassProg, Riki7649255 - def delete_object_from_management_table(self, table_name, criteria) -> None: + def _delete_object_from_management_table(self, table_name, criteria) -> None: '''Delete an object from the database.''' self.db_manager.delete_data_from_table(table_name, criteria) - def delete_object_from_management_table_by_id(self, table_name, object_id) -> None: + def _delete_object_from_management_table_by_id(self, pk_col, table_name, object_id) -> None: '''Delete an object from the database.''' - self.db_manager.delete_data_from_table(table_name, criteria= f'object_id = {object_id}') + self.db_manager.delete_data_from_table(table_name, criteria= f'{pk_col} = \'{object_id}\'') # rachel-8511, ShaniStrassProg is it needed? @@ -72,11 +83,11 @@ def delete_object_from_management_table_by_id(self, table_name, object_id) -> No # return self.db_manager.describe(self.table_name) - def convert_object_name_to_management_table_name(object_name): + def _convert_object_name_to_management_table_name(object_name): return f'mng_{object_name}s' - def is_management_table_exist(self, table_name): + def _is_management_table_exist(self, table_name): # Check if table exists by querying the sqlite_master table query = f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}'" return self.db_manager.execute_query_with_single_result(query) @@ -88,36 +99,30 @@ def save_in_memory(self, table_name, object): # insert object info into management table mng_{object_name}s # for exmple: object db_instance will be saved in table mng_db_instances - self.insert_object_to_management_table(table_name, object) + self._insert_object_to_management_table(table_name, object) - def delete_from_memory(self, pk_col, pk_val, table_name:str, criteria='default'): + def delete_from_memory_by_id(self, pk_col, pk_val, table_name:str): # pk_val is the object id # if criteria not sent- use PK for deletion - if criteria == 'default': - criteria = f'{pk_col} = \'{pk_val}\'' - + criteria = f'{pk_col} = \'{pk_val}\'' self.db_manager.delete_data_from_table(table_name, criteria) - - def update_in_memory(self, pktable_name, updates, criteria='default', object_id:Optional[str] = None): - - # if criteria not sent- use PK for deletion - if criteria == 'default': - if not object_id: - raise ValueError('must be or criteria or object id') - criteria = f'object_id = {object_id}' - - + def update_in_memory_by_criteria(self,table_name:str, updates:Dict, criteria): + self._update_object_in_management_table_by_criteria(table_name, updates, criteria) + + def update_in_memory_by_id(self, pk_col, table_name, updates, object_id:Optional[str]): + if not object_id: + raise ValueError('must be or criteria or object id') + criteria = f'{pk_col} = \'{object_id}\'' + self.update_in_memory_by_criteria(table_name, updates, criteria) - def get_from_memory(self, object_name, columns = ["*"], object_id = None, criteria = None): + def get_from_memory_by_id(self, pk_col, table_name, object_id, columns = ["*"]): """get records from memory by criteria or id""" - table_name = self.convert_object_name_to_management_table_name(object_name) - if object_id: - criteria = f'object_id = {object_id}' - self.get_objects_from_management_table_by_criteria(table_name, columns, criteria) + criteria = f'{pk_col} = \'{object_id}\'' + return self._get_objects_from_management_table_by_criteria(table_name, columns, criteria) def convert_object_attributes_to_dictionary(**kwargs): diff --git a/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py b/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py index ccb0bff7..7ed37365 100644 --- a/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py +++ b/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py @@ -23,6 +23,7 @@ class DBSubnetGroup: def __init__(self, **kwargs): try: + print(kwargs) self.db_subnet_group_name = kwargs['db_subnet_group_name'] self.db_subnet_group_description = kwargs['db_subnet_group_description'] self.vpc_id = kwargs['vpc_id'] diff --git a/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py b/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py index 0c42b29e..8fc33513 100644 --- a/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py +++ b/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py @@ -57,8 +57,7 @@ def get_db_subnet_group(self, db_subnet_group_name: str) -> DBSubnetGroup: def modify_db_subnet_group(self, db_subnet_group_name: str, updates: Dict[str, Any]) -> DBSubnetGroup: if not db_subnet_group_name: raise ValueError('Missing required argument db_subnet_group_name') - - + if updates.get('description') and not is_length_in_range(updates['description'], 1, 255): raise ValueError("invalid length for subnet group description: " + len(updates['description'])) From 109e4c6660393f00db4b1c1fab11a2fce2d0b145 Mon Sep 17 00:00:00 2001 From: sara-lea Date: Mon, 16 Sep 2024 14:50:00 +0300 Subject: [PATCH 06/16] Implementing the createDBCluster function --- .../Untitled-checkpoint.ipynb | 6 + DB/ELTS/Untitled.ipynb | 52 ++++++ DB/NEW_KT_DB/DataAccess/DBClusterManager.py | 7 +- DB/NEW_KT_DB/DataAccess/DBManager.py | 76 ++++---- DB/NEW_KT_DB/DataAccess/ObjectManager.py | 35 ++-- DB/NEW_KT_DB/Models/DBClusterModel.py | 114 +++++++----- .../Service/Classes/DBClusterService.py | 98 +++++++++- .../Validation/DBClusterValiditions.py | 174 +++++++++++++++++- 8 files changed, 460 insertions(+), 102 deletions(-) create mode 100644 DB/ELTS/.ipynb_checkpoints/Untitled-checkpoint.ipynb create mode 100644 DB/ELTS/Untitled.ipynb diff --git a/DB/ELTS/.ipynb_checkpoints/Untitled-checkpoint.ipynb b/DB/ELTS/.ipynb_checkpoints/Untitled-checkpoint.ipynb new file mode 100644 index 00000000..363fcab7 --- /dev/null +++ b/DB/ELTS/.ipynb_checkpoints/Untitled-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/DB/ELTS/Untitled.ipynb b/DB/ELTS/Untitled.ipynb new file mode 100644 index 00000000..d389bf14 --- /dev/null +++ b/DB/ELTS/Untitled.ipynb @@ -0,0 +1,52 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 40, + "id": "d05cb473", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Get the desktop path\n", + "desktop_path = os.path.join(os.path.expanduser('~'), 'Desktop')\n", + "db_cluster_identifier= \"cluster\"\n", + "# Create the full directory path on the desktop\n", + "cluster_directory = os.path.join(desktop_path, f'clusters/{db_cluster_identifier}')\n", + "\n", + "# Create the directory (and any necessary intermediate directories)\n", + "os.makedirs(cluster_directory, exist_ok=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "76a40da7", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/DB/NEW_KT_DB/DataAccess/DBClusterManager.py b/DB/NEW_KT_DB/DataAccess/DBClusterManager.py index 802c229f..e35c36a9 100644 --- a/DB/NEW_KT_DB/DataAccess/DBClusterManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBClusterManager.py @@ -8,11 +8,11 @@ def __init__(self, db_file: str): '''Initialize ObjectManager with the database connection.''' self.object_manager = ObjectManager(db_file) self.table_name ='cluster_managment' - self.create_table() + # self.create_table() - def createInMemoryDBCluster(self): - self.object_manager.save_in_memory() + def createInMemoryDBCluster(self, cluster_to_save): + self.object_manager.save_in_memory(cluster_to_save) def deleteInMemoryDBCluster(self): @@ -25,4 +25,3 @@ def describeDBCluster(self): def modifyDBCluster(self): self.object_manager.update_in_memory() - diff --git a/DB/NEW_KT_DB/DataAccess/DBManager.py b/DB/NEW_KT_DB/DataAccess/DBManager.py index dd0dde58..af0766b3 100644 --- a/DB/NEW_KT_DB/DataAccess/DBManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBManager.py @@ -11,21 +11,21 @@ def __init__(self, db_file: str): # rachel-8511, ShaniStrassProg def close(self): - '''Close the database connection.''' - self.connection.close() + '''Close the database connection.''' + self.connection.close() # saraNoigershel def execute_query_with_multiple_results(self, query: str) -> Optional[List[Tuple]]: - '''Execute a given query and return the results.''' - try: - c = self.connection.cursor() - c.execute(query) - results = c.fetchall() - # self.connection.commit() ??? - return results if results else None - except OperationalError as e: - raise Exception(f'Error executing query {query}: {e}') + '''Execute a given query and return the results.''' + try: + c = self.connection.cursor() + c.execute(query) + results = c.fetchall() + # self.connection.commit() ??? + return results if results else None + except OperationalError as e: + raise Exception(f'Error executing query {query}: {e}') # ShaniStrassProg @@ -64,34 +64,46 @@ def insert_data_into_table(self, table_name, data): insert_statement = f'''INSERT INTO {table_name} VALUES {data}''' execute_query_without_results(insert_statement) + # # Riki7649255 based on rachel-8511, Shani + # def update_records_in_table(self, table_name: str, updates: Dict[str, Any], criteria: str) -> None: + # '''Update records in the specified table based on criteria.''' + + # # add documentation here + # set_clause = ', '.join([f'{k} = ?' for k in updates.keys()]) + # values = list(updates.values()) + + # update_statement = f''' + # UPDATE {table_name} + # SET {set_clause} + # WHERE {criteria} + # ''' + + # execute_query_without_results(update_statement) - # Riki7649255 based on rachel-8511, Shani def update_records_in_table(self, table_name: str, updates: Dict[str, Any], criteria: str) -> None: - '''Update records in the specified table based on criteria.''' - - # add documentation here - set_clause = ', '.join([f'{k} = ?' for k in updates.keys()]) - values = list(updates.values()) - - update_statement = f''' - UPDATE {table_name} - SET {set_clause} - WHERE {criteria} - ''' - - execute_query_without_results(update_statement) + '''Update records in the specified table based on criteria.''' + + set_clause = '(' + ', '.join([f'{k}' for k in updates.keys()]) + ') = (' + ', '.join([f'"{v}"' for v in updates.values()]) + ')' + + update_statement = f''' + UPDATE {table_name} + SET {set_clause} + WHERE {criteria} + ''' + + self.execute_query_without_results(update_statement) # Riki7649255 based on rachel-8511 def delete_data_from_table(self, table_name: str, criteria: str) -> None: - '''Delete a record from the specified table based on criteria.''' - - delete_statement = f''' - DELETE FROM {table_name} - WHERE {criteria} - ''') + '''Delete a record from the specified table based on criteria.''' - execute_query_without_results(delete_statement) + delete_statement = f''' + DELETE FROM {table_name} + WHERE {criteria} + ''' + + execute_query_without_results(delete_statement) # rachel-8511, Riki7649255 diff --git a/DB/NEW_KT_DB/DataAccess/ObjectManager.py b/DB/NEW_KT_DB/DataAccess/ObjectManager.py index 56c6948f..0cf00f4f 100644 --- a/DB/NEW_KT_DB/DataAccess/ObjectManager.py +++ b/DB/NEW_KT_DB/DataAccess/ObjectManager.py @@ -12,7 +12,7 @@ def __init__(self, db_file: str): # for internal use only: # Riki7649255 based on rachel-8511 - def create_management_table(self, table_name, table_structure='object_id INTEGER PRIMARY KEY AUTOINCREMENT,type_object TEXT NOT NULL,metadata TEXT NOT NULL') + def create_management_table(self, table_name, table_structure='object_id INTEGER PRIMARY KEY AUTOINCREMENT,type_object TEXT NOT NULL,metadata TEXT NOT NULL'): self.db_manager.create_table(table_name, table_structure) @@ -60,21 +60,28 @@ def convert_object_name_to_management_table_name(object_name): def is_management_table_exist(table_name): # check if table exists using single result query - return db_manager.execute_query_with_single_result(f'desc table {table_name}') + return self.db_manager.execute_query_with_single_result(f'desc table {table_name}') # for outer use: - def save_in_memory(self, object): + # def save_in_memory(self, object): - # insert object info into management table mng_{object_name}s - # for exmple: object db_instance will be saved in table mng_db_instances - table_name = convert_object_name_to_management_table_name(self.object_name) + # # insert object info into management table mng_{object_name}s + # # for exmple: object db_instance will be saved in table mng_db_instances + # table_name = self.convert_object_name_to_management_table_name(self.object_name) - if not is_management_table_exist(table_name): - create_management_table(table_name) + # if not self.is_management_table_exist(table_name): + # self.create_management_table(table_name) - insert_object_to_management_table(table_name, object) + # self.insert_object_to_management_table(table_name, object) + def save_in_memory(self, object): + # insert object info into management table mng_{object_name}s + # for exmple: object db_instance will be saved in table mng_db_instances + table_name = str(object.__class__.__name__) + if not self.is_management_table_exist(table_name): + self.create_management_table(table_name) + self.insert_object_to_management_table(table_name, object) def delete_from_memory(self,criteria='default'): @@ -82,9 +89,9 @@ def delete_from_memory(self,criteria='default'): if criteria == 'default': criteria = f'{self.pk_column} = {self.pk_value}' - table_name = convert_object_name_to_management_table_name(self.object_name) + table_name = self.convert_object_name_to_management_table_name(self.object_name) - delete_data_from_table(table_name, criteria) + self.delete_data_from_table(table_name, criteria) def update_in_memory(self, updates, criteria='default'): @@ -93,13 +100,13 @@ def update_in_memory(self, updates, criteria='default'): if criteria == 'default': criteria = f'{self.pk_column} = {self.pk_value}' - table_name = convert_object_name_to_management_table_name(self.object_name) + table_name = self.convert_object_name_to_management_table_name(self.object_name) - update_object_in_management_table_by_criteria(table_name, updates, criteria) + self.update_object_in_management_table_by_criteria(table_name, updates, criteria) def get_from_memory(self): - get_object_from_management_table(self.object_id) + self.get_object_from_management_table(self.object_id) def convert_object_attributes_to_dictionary(**kwargs): diff --git a/DB/NEW_KT_DB/Models/DBClusterModel.py b/DB/NEW_KT_DB/Models/DBClusterModel.py index 23170019..846d9989 100644 --- a/DB/NEW_KT_DB/Models/DBClusterModel.py +++ b/DB/NEW_KT_DB/Models/DBClusterModel.py @@ -1,75 +1,105 @@ from datetime import datetime from typing import Dict from DataAccess import ObjectManager - +import os class Cluster: def __init__(self, **kwargs): self.db_cluster_identifier = kwargs['db_cluster_identifier'] self.engine = kwargs['engine'] - self.availability_zones = kwargs.get('availability_zones', None) + self.allocated_storage = kwargs['allocated_storage'] self.copy_tags_to_snapshot = kwargs.get('copy_tags_to_snapshot', False) + self.db_cluster_instance_class = kwargs.get('db_cluster_instance_class', False) self.database_name = kwargs.get('database_name', None) self.db_cluster_parameter_group_name = kwargs.get('db_cluster_parameter_group_name', None) self.db_subnet_group_name = kwargs.get('db_subnet_group_name', None) self.deletion_protection = kwargs.get('deletion_protection', False) - self.enable_cloudwatch_logs_exports = kwargs.get('enable_cloudwatch_logs_exports', None) - self.enable_global_write_forwarding = kwargs.get('enable_global_write_forwarding', False) - self.enable_http_endpoint = kwargs.get('enable_http_endpoint', False) - self.enable_limitless_database = kwargs.get('enable_limitless_database', False) - self.enable_local_write_forwarding = kwargs.get('enable_local_write_forwarding', False) self.engine_version = kwargs.get('engine_version', None) - self.global_cluster_identifier = kwargs.get('global_cluster_identifier', None) + self.master_username = kwargs.get('master_username',None) + self.master_user_password = kwargs.get('master_user_password',None) + self.manage_master_user_password = kwargs.get('manage_master_user_password',False) self.option_group_name = kwargs.get('option_group_name', None) self.port = kwargs.get('port', None) #handle defuelt values self.replication_source_identifier = kwargs.get('replication_source_identifier', None) - self.scaling_configuration = kwargs.get('scaling_configuration', None) self.storage_encrypted = kwargs.get('storage_encrypted', None) self.storage_type = kwargs.get('storage_type', 'aurora') self.tags = kwargs.get('tags', None) self.created_at = datetime.now() self.status = 'available' - self.instances = {} - self.primary_endpoint = None - # self.primary_writer_instance = create db instance - # self.reader_instances = replica of the instance in different azs + self.primary_writer_instance = None + self.reader_instances = [] + self.cluster_endpoint = None + self.instances_endpoints = {} # Added attribute to store endpoints self.pk_column = kwargs.get('pk_column', 'ClusterID') - self.pk_value = kwargs.get('pk_value', None) + self.pk_value = kwargs.get('pk_value', self.db_cluster_identifier) def to_dict(self) -> Dict: '''Retrieve the data of the DB cluster as a dictionary.''' return ObjectManager.convert_object_attributes_to_dictionary( - db_cluster_identifier=self.db_cluster_identifier, - engine=self.engine, - availability_zones=self.availability_zones, - copy_tags_to_snapshot=self.copy_tags_to_snapshot, - database_name=self.database_name, - db_cluster_parameter_group_name=self.db_cluster_parameter_group_name, - db_subnet_group_name=self.db_subnet_group_name, - deletion_protection=self.deletion_protection, - enable_cloudwatch_logs_exports=self.enable_cloudwatch_logs_exports, - enable_global_write_forwarding=self.enable_global_write_forwarding, - enable_http_endpoint=self.enable_http_endpoint, - enable_limitless_database=self.enable_limitless_database, - enable_local_write_forwarding=self.enable_local_write_forwarding, - engine_version=self.engine_version, - global_cluster_identifier=self.global_cluster_identifier, - option_group_name=self.option_group_name, - port=self.port, - replication_source_identifier=self.replication_source_identifier, - scaling_configuration=self.scaling_configuration, - storage_encrypted=self.storage_encrypted, - storage_type=self.storage_type, - tags=self.tags, - created_at=self.created_at, - status=self.status, - instances=self.instances, - # primary_writer_instance =self.primary_writer_instance, - # reader_instances =self.reader_instances + db_cluster_identifier=self.db_cluster_identifier, + engine=self.engine, + allocated_storage=self.allocated_storage, + copy_tags_to_snapshot=self.copy_tags_to_snapshot, + db_cluster_instance_class=self.db_cluster_instance_class, + database_name=self.database_name, + db_cluster_parameter_group_name=self.db_cluster_parameter_group_name, + db_subnet_group_name=self.db_subnet_group_name, + deletion_protection=self.deletion_protection, + engine_version=self.engine_version, + master_username = self.master_username, + master_user_password = self.master_user_password, + manage_master_user_password = self.manage_master_user_password, + option_group_name=self.option_group_name, + port=self.port, + replication_source_identifier=self.replication_source_identifier, + storage_encrypted=self.storage_encrypted, + storage_type=self.storage_type, + tags=self.tags, + created_at=self.created_at, + status=self.status, + primary_writer_instance=self.primary_writer_instance, + reader_instances=self.reader_instances, + cluster_endpoint = self.cluster_endpoint, + instances_endpoints=self.instances_endpoints, pk_column=self.pk_column, pk_value=self.pk_value - ) \ No newline at end of file + ) + + # def to_dict(self) -> Dict: + # '''Retrieve the data of the DB cluster as a dictionary.''' + + # return ObjectManager.convert_object_attributes_to_dictionary( + # db_cluster_identifier=self.db_cluster_identifier, + # engine=self.engine, + # availability_zones=self.availability_zones, + # copy_tags_to_snapshot=self.copy_tags_to_snapshot, + # database_name=self.database_name, + # db_cluster_parameter_group_name=self.db_cluster_parameter_group_name, + # db_subnet_group_name=self.db_subnet_group_name, + # deletion_protection=self.deletion_protection, + # enable_cloudwatch_logs_exports=self.enable_cloudwatch_logs_exports, + # enable_global_write_forwarding=self.enable_global_write_forwarding, + # enable_http_endpoint=self.enable_http_endpoint, + # enable_limitless_database=self.enable_limitless_database, + # enable_local_write_forwarding=self.enable_local_write_forwarding, + # engine_version=self.engine_version, + # global_cluster_identifier=self.global_cluster_identifier, + # option_group_name=self.option_group_name, + # port=self.port, + # replication_source_identifier=self.replication_source_identifier, + # scaling_configuration=self.scaling_configuration, + # storage_encrypted=self.storage_encrypted, + # storage_type=self.storage_type, + # tags=self.tags, + # created_at=self.created_at, + # status=self.status, + # primary_writer_instance = self.primary_writer_instance, + # reader_instances = self.reader_instances, + # endpoints=self.endpoints, + # pk_column=self.pk_column, + # pk_value=self.pk_value + # ) \ No newline at end of file diff --git a/DB/NEW_KT_DB/Service/Classes/DBClusterService.py b/DB/NEW_KT_DB/Service/Classes/DBClusterService.py index 65acc86d..b10f5ffe 100644 --- a/DB/NEW_KT_DB/Service/Classes/DBClusterService.py +++ b/DB/NEW_KT_DB/Service/Classes/DBClusterService.py @@ -2,23 +2,107 @@ from DataAccess import ClusterManager from Models import DBClusterModel from Abc import DBO -from Validation import Validation +from Validation import DBClusterValiditions from DataAccess import DBClusterManager +from DBInstanceService import DBInstanceService +import os +import json +from DBClusterValiditions import ( + validate_db_cluster_identifier, + validate_engine, + validate_database_name, + validate_db_cluster_parameter_group_name, + validate_db_subnet_group_name, + validate_port, + check_required_params, + validate_master_user_password, + validate_master_username +) class DBClusterService(DBO): - def __init__(self, dal: ClusterManager): + def __init__(self, dal: DBClusterManager): self.dal = dal # validations here - def create(self, **attributes): + + def create(self, **kwargs): + '''Create a new DBCluster.''' - # create object in code using DBClusterModel.init()- assign all **attributes - # create physical object as described in task - # save in memory using DBClusterManager.createInMemoryDBCluster() function - pass + + # Validate required parameters + required_params = ['db_cluster_identifier', 'engine', 'db_subnet_group_name'] + if not check_required_params(required_params, **kwargs): + raise ValueError("Missing required parameters") + + # Perform validations + if not validate_db_cluster_identifier(kwargs.get('db_cluster_identifier', '')): + raise ValueError(f"Invalid DBClusterIdentifier: {kwargs.get('db_cluster_identifier')}") + + if not validate_engine(kwargs.get('engine', '')): + raise ValueError(f"Invalid engine: {kwargs.get('engine')}") + + if 'database_name' in kwargs and kwargs['database_name'] and not validate_database_name(kwargs['database_name']): + raise ValueError(f"Invalid DatabaseName: {kwargs['database_name']}") + + if 'db_cluster_parameter_group_name' in kwargs and kwargs['db_cluster_parameter_group_name'] and not validate_db_cluster_parameter_group_name(kwargs['db_cluster_parameter_group_name']): + raise ValueError(f"Invalid DBClusterParameterGroupName: {kwargs['db_cluster_parameter_group_name']}") + + if kwargs.get('db_subnet_group_name') and not validate_db_subnet_group_name(kwargs.get('db_subnet_group_name')): + raise ValueError(f"Invalid DBSubnetGroupName: {kwargs['db_subnet_group_name']}") + + if 'port' in kwargs and kwargs['port'] and not validate_port(kwargs['port']): + raise ValueError(f"Invalid port: {kwargs['port']}. Valid range is 1150-65535.") + + if 'master_username' in kwargs and not validate_master_username(kwargs['master_username']): + raise ValueError("Invalid master username") + + if 'master_user_password' in kwargs and not validate_master_user_password(kwargs['master_user_password'], kwargs.get('manage_master_user_password', False)): + raise ValueError("Invalid master user password") + + # Create the cluster object + cluster = DBClusterModel.Cluster(**kwargs) + + # Create physical folder structure + desktop_path = os.path.join(os.path.expanduser('~'), 'Desktop') + cluster_directory = os.path.join(desktop_path, f'Clusters/{cluster.db_cluster_identifier}') + os.makedirs(cluster_directory, exist_ok=True) + + # Set cluster endpoint + cluster.cluster_endpoint = cluster_directory + + # Create the primary writer instance + primary_instance_name = f'{cluster.db_cluster_identifier}-primary' + primary_instance = self.DBInstanceService( + instance_name=primary_instance_name, + cluster_identifier=cluster.db_cluster_identifier, + allocated_storage=cluster.allocated_storage, + master_username=cluster.master_username, + master_user_password=cluster.master_user_password + ) + + # Retrieve primary instance details + primary_instance_json_string = primary_instance.get("DBInstance") + primary_instance_json_data = json.loads(primary_instance_json_string) + cluster.instances_endpoints["primary_instance"] = primary_instance_json_data.get("endpoint") + cluster.primary_writer_instance = primary_instance_json_data.get('db_instance_identifier') + + # Create configuration file + cluster_config_path = os.path.join(cluster_directory, 'cluster_config.json') + cluster_dict = cluster.to_dict() + try: + with open(cluster_config_path, 'w') as file: + json.dump(cluster_dict, file, indent=4) + except IOError as e: + raise RuntimeError(f"Failed to write configuration file: {e}") + + # Store the cluster information in the database + self.dal.createInMemoryDBCluster(cluster) + + return {"DBCluster": cluster_dict} + def delete(self): diff --git a/DB/NEW_KT_DB/Validation/DBClusterValiditions.py b/DB/NEW_KT_DB/Validation/DBClusterValiditions.py index fecf15d2..3ffcbf6f 100644 --- a/DB/NEW_KT_DB/Validation/DBClusterValiditions.py +++ b/DB/NEW_KT_DB/Validation/DBClusterValiditions.py @@ -1,6 +1,174 @@ import re -from GeneralValidations import +import GeneralValidations +# def is_db_cluster_name_valid(cluster_name): +# return is_length_in_range(cluster_name, 5, 20) -def is_db_cluster_name_valid(cluster_name): - return is_length_in_range(cluster_name, 5, 20) +def validate_db_cluster_identifier(identifier: str) -> bool: + """ + Validates the DBClusterIdentifier based on the cluster type. + Returns True if valid, False otherwise. + """ + length_constraint = 52 + pattern = r'^[a-zA-Z][a-zA-Z0-9\-]{0,' + str(length_constraint - 1) + r'}(? bool: + # """ + # Validates the engine type. + # Returns True if valid, False otherwise. + # """ + # valid_engines = ['aurora-mysql', 'aurora-postgresql', 'mysql', 'postgres', 'neptune'] + # return engine in valid_engines + """ + Validates the engine type. + Reuses `string_in_dict` for engine validation. + """ + valid_engines = ['mysql', 'postgres'] + return GeneralValidations.string_in_dict(engine, dict.fromkeys(valid_engines, True)) + +def validate_database_name(database_name: str) -> bool: + """ + Validates the database name (if provided). + Returns True if valid, False otherwise. + """ + # if not database_name: + # return True # No validation needed if not provided + # return re.match(r'^[a-zA-Z0-9]{1,64}$', database_name) is not None + if not database_name: + return True # No validation needed if not provided + return GeneralValidations.is_length_in_range(database_name, 1, 64) and GeneralValidations.is_string_matches_regex(database_name, r'^[a-zA-Z0-9]+$') + +def validate_db_cluster_parameter_group_name(group_name: str) -> bool: + """ + Validates the DBClusterParameterGroupName. + Returns True if valid, False otherwise. + """ + if not group_name: + return True # No validation needed if not provided + return re.match(r'^[a-zA-Z0-9\-]+$', group_name) is not None + +def validate_db_subnet_group_name(subnet_group_name: str) -> bool: + """ + Validates the DBSubnetGroupName. + Returns True if valid, False otherwise. + """ + if not subnet_group_name: + return True # No validation needed if not provided + return re.match(r'^[a-zA-Z0-9\-]+$', subnet_group_name) is not None + +def validate_port(port: int) -> bool: + """ + Validates the port number. + Returns True if valid, False otherwise. + """ + # return 1150 <= port <= 65535 + + return GeneralValidations.is_valid_number(port, 1150, 65535) +# from general_validations import ( +# is_length_in_range, +# is_string_matches_regex +# ) + +def validate_master_username(username: str) -> bool: + """ + Validates the MasterUsername. + Constraints: + - Must be 1 to 16 letters or numbers. + - First character must be a letter. + - Can't be a reserved word for the chosen database engine. + """ + if not username: + return True # Not required, so no validation needed if not provided + if not GeneralValidations.is_length_in_range(username, 1, 16): + return False + if not username[0].isalpha(): # First character must be a letter + return False + # Optional: Add a list of reserved words for specific engines and check against that + reserved_words = [] # Define reserved words for the engine if applicable + if username.lower() in reserved_words: + return False + return GeneralValidations.is_string_matches_regex(username, r'^[a-zA-Z0-9]+$') # Letters and numbers only + +def validate_master_user_password(password: str, manage_master_user_password: bool) -> bool: + """ + Validates the MasterUserPassword. + Constraints: + - Must contain from 8 to 41 characters. + - Can contain any printable ASCII character except "/", "\"", or "@". + - Can't be specified if ManageMasterUserPassword is turned on. + """ + if not password: + return True # Not required, so no validation needed if not provided + if manage_master_user_password: + return False # Can't specify password if ManageMasterUserPassword is turned on + if not is_length_in_range(password, 8, 41): + return False + # Ensure password doesn't contain "/", "\"", or "@" + restricted_chars = ["/", "\\", "@"] + for char in restricted_chars: + if char in password: + return False + # Optionally, check for only printable ASCII characters (32 to 126 ASCII range) + if not all(32 <= ord(c) <= 126 for c in password): + return False + return True + +def check_required_params(required_params, **kwargs): + for param in required_params: + if param not in kwargs.keys(): + return False + return True + +# import sqlite3 +# from sqlite3 import OperationalError +# import re +# import sys + +# def string_in_dict(string: str, values: dict) -> bool: +# """Check if the string is in dict.""" +# return string in values + +# def is_valid_length(string: str, min_length: int, max_length: int) -> bool: +# """Check if the string is valid based on the length.""" +# return min_length <= len(string) <= max_length + +# def is_valid_pattern(string: str, pattern: str) -> bool: +# """Check if the optionGroupName is valid based on the pattern.""" +# return bool(re.match(pattern, string)) + +# def exist_key_value_in_json_column(conn: sqlite3.Connection, table_name: str, column_name: str, key: str, value: str) -> bool: +# """Check if a specific key-value pair exists within a JSON column in the given table.""" +# try: +# c = conn.cursor() +# c.execute(f''' +# SELECT COUNT(*) FROM {table_name} +# WHERE {column_name} LIKE ? +# ''', (f'%"{key}": "{value}"%',)) +# return c.fetchone()[0] > 0 +# except OperationalError as e: +# print(f"Error: {e}") + +# def exist_value_in_column(conn: sqlite3.Connection, table_name: str, column_name: str, value: str) -> bool: +# """Check if a specific value exists within a column in the given table.""" +# try: +# c = conn.cursor() +# c.execute(f''' +# SELECT COUNT(*) FROM {table_name} +# WHERE {column_name} LIKE ? +# ''', (value,)) +# return c.fetchone()[0] > 0 +# except OperationalError as e: +# print(f"Error: {e}") + + + +# def is_valid_number(num: int, min: int = -sys.maxsize - 1, max: int = sys.maxsize) -> bool: +# return min <= num <= max + +# def check_required_params(required_params, **kwargs): +# for param in required_params: +# if param not in kwargs.keys(): +# return False +# return True \ No newline at end of file From ac36839912017dc6b7df483de92b45789de620aa Mon Sep 17 00:00:00 2001 From: Tem-M Date: Mon, 16 Sep 2024 15:02:39 +0300 Subject: [PATCH 07/16] commit --- DB/NEW_KT_DB/DataAccess/ObjectManager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DB/NEW_KT_DB/DataAccess/ObjectManager.py b/DB/NEW_KT_DB/DataAccess/ObjectManager.py index ec3b2ece..ce24835f 100644 --- a/DB/NEW_KT_DB/DataAccess/ObjectManager.py +++ b/DB/NEW_KT_DB/DataAccess/ObjectManager.py @@ -83,7 +83,7 @@ def _delete_object_from_management_table_by_id(self, pk_col, table_name, object_ # return self.db_manager.describe(self.table_name) - def _convert_object_name_to_management_table_name(object_name): + def _convert_object_name_to_management_table_name(self, object_name): return f'mng_{object_name}s' From aa3d696e8d01ecb15467edea3d154a5a26833ef2 Mon Sep 17 00:00:00 2001 From: sara-lea Date: Mon, 16 Sep 2024 16:15:44 +0300 Subject: [PATCH 08/16] try to run the file --- DB/NEW_KT_DB/Controller/DBClusterController.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/DB/NEW_KT_DB/Controller/DBClusterController.py b/DB/NEW_KT_DB/Controller/DBClusterController.py index b253975b..adc157b4 100644 --- a/DB/NEW_KT_DB/Controller/DBClusterController.py +++ b/DB/NEW_KT_DB/Controller/DBClusterController.py @@ -1,5 +1,5 @@ -from Service import DBClusterService - +# from Service.Classes import DBClusterService +from Service.Classes.DBClusterService import DBClusterService class DBClusterController: def __init__(self, service: DBClusterService): self.service = service @@ -15,4 +15,15 @@ def delete_db_cluster(self): def modify_db_cluster(self, updates): self.service.modify(updates) - \ No newline at end of file + + +if __name__=='__main__': + clusterController = DBClusterController() + cluster_data = { + 'db_cluster_identifier': 'my-cluster-1', + 'engine': 'aurora-mysql', + 'db_subnet_group_name': 'my-subnet-group' + } + + aaa = clusterController.create_db_cluster(**cluster_data) + print(aaa) \ No newline at end of file From c2eb55bc8c2b714f17970f70a3e901f894644bbb Mon Sep 17 00:00:00 2001 From: sara-lea Date: Tue, 17 Sep 2024 13:26:14 +0300 Subject: [PATCH 09/16] implementation on cluster crud without manager --- .../Controller/DBClusterController.py | 21 +- DB/NEW_KT_DB/DataAccess/DBClusterManager.py | 35 +- DB/NEW_KT_DB/DataAccess/DBManager.py | 765 ++++++++++++++---- DB/NEW_KT_DB/DataAccess/ObjectManager.py | 493 ++++++++--- DB/NEW_KT_DB/Models/DBClusterModel.py | 54 +- .../Service/Classes/DBClusterService.py | 191 +++-- .../Validation/DBClusterValiditions.py | 7 +- 7 files changed, 1232 insertions(+), 334 deletions(-) diff --git a/DB/NEW_KT_DB/Controller/DBClusterController.py b/DB/NEW_KT_DB/Controller/DBClusterController.py index adc157b4..198735fe 100644 --- a/DB/NEW_KT_DB/Controller/DBClusterController.py +++ b/DB/NEW_KT_DB/Controller/DBClusterController.py @@ -1,5 +1,12 @@ # from Service.Classes import DBClusterService +import sys +import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from KT_Cloud.Storage.NEW_KT_Storage.DataAccess import StorageManager from Service.Classes.DBClusterService import DBClusterService +from DataAccess import DBClusterManager class DBClusterController: def __init__(self, service: DBClusterService): self.service = service @@ -18,10 +25,18 @@ def modify_db_cluster(self, updates): if __name__=='__main__': - clusterController = DBClusterController() + + desktop_path = os.path.join(os.path.expanduser('~'), 'Desktop') + cluster_directory = os.path.join(desktop_path, f'Clusters/clusters.db') + base = os.path.join(desktop_path, f'Clusters') + storage_manager = StorageManager.StorageManager(base) + clusterManager = DBClusterManager.DBClusterManager(cluster_directory) + clusterService = DBClusterService(clusterManager,storage_manager, cluster_directory) + clusterController = DBClusterController(clusterService) cluster_data = { - 'db_cluster_identifier': 'my-cluster-1', - 'engine': 'aurora-mysql', + 'db_cluster_identifier': 'my-cluster-3', + 'engine': 'mysql', + 'allocated_storage':5, 'db_subnet_group_name': 'my-subnet-group' } diff --git a/DB/NEW_KT_DB/DataAccess/DBClusterManager.py b/DB/NEW_KT_DB/DataAccess/DBClusterManager.py index e35c36a9..9ee4ae83 100644 --- a/DB/NEW_KT_DB/DataAccess/DBClusterManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBClusterManager.py @@ -2,12 +2,14 @@ import json import sqlite3 from DataAccess import ObjectManager +from typing import Optional class DBClusterManager: def __init__(self, db_file: str): '''Initialize ObjectManager with the database connection.''' - self.object_manager = ObjectManager(db_file) - self.table_name ='cluster_managment' + self.object_manager = ObjectManager.ObjectManager(db_file) + self.object_name ='clusters' + self.pk_column = 'ClusterID' # self.create_table() @@ -15,13 +17,30 @@ def createInMemoryDBCluster(self, cluster_to_save): self.object_manager.save_in_memory(cluster_to_save) - def deleteInMemoryDBCluster(self): - self.object_manager.delete_from_memory() + def deleteInMemoryDBCluster(self,cluster_identifier): + self.object_manager.delete_from_memory(cluster_identifier) - def describeDBCluster(self): - self.object_manager.get_from_memory() + def describeDBCluster(self, cluster_id): + self.object_manager.get_from_memory(self.object_name, object_id = cluster_id) + def modifyDBCluster(self, cluster_id, updates): + self.object_manager.update_in_memory(self.object_name, updates, object_id = cluster_id) - def modifyDBCluster(self): - self.object_manager.update_in_memory() + def select(self, name:Optional[str] = None, columns = ["*"]): + data = self.object_manager.get_from_memory(self.object_name, columns = columns, object_id = name) + if data: + data_to_return = [{col:data[col] for col in columns}] + data_to_return[self.pk_column] = name + return data_to_return + + else: + raise ValueError(f"db cluster with name '{name}' not found") + + def is_exists(self, name): + """check if object exists in table""" + try: + self.select(name) + return True + except: + return False \ No newline at end of file diff --git a/DB/NEW_KT_DB/DataAccess/DBManager.py b/DB/NEW_KT_DB/DataAccess/DBManager.py index 52a3056b..c7ec64f6 100644 --- a/DB/NEW_KT_DB/DataAccess/DBManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBManager.py @@ -1,3 +1,575 @@ +# import sqlite3 +# from typing import Dict, Any, List, Optional, Tuple +# import json +# from sqlite3 import OperationalError + +# class DBManager: +# def __init__(self, db_file: str): +# '''Initialize the database connection and create tables if they do not exist.''' +# self.connection = sqlite3.connect(db_file) + + +# # rachel-8511, ShaniStrassProg +# def close(self): +# '''Close the database connection.''' +# self.connection.close() + + +# # # saraNoigershel +# # def execute_query_with_multiple_results(self, query: str) -> Optional[List[Tuple]]: +# # '''Execute a given query and return the results.''' +# # try: +# # c = self.connection.cursor() +# # c.execute(query) +# # results = c.fetchall() +# # # self.connection.commit() ??? + +# # saraNoigershel +# def execute_query_with_multiple_results(self, query: str, params:Tuple = ()) -> Optional[List[Tuple]]: +# '''Execute a given query and return the results.''' +# try: +# c = self.connection.cursor() +# c.execute(query, params) +# results = c.fetchall() +# self.connection.commit() +# return results if results else None +# except OperationalError as e: +# raise Exception(f'Error executing query {query}: {e}') + + +# # ShaniStrassProg +# def execute_query_with_single_result(self, query: str, params:Tuple = ()) -> Optional[Tuple]: +# '''Execute a given query and return a single result.''' +# try: +# c = self.connection.cursor() +# c.execute(query, params) +# result = c.fetchone() +# self.connection.commit() +# return result if result else None + +# except OperationalError as e: +# raise Exception(f'Error executing query {query}: {e}') + + +# # Riki7649255 +# def execute_query_without_results(self, query: str, params:Tuple = ()): +# '''Execute a given query without waiting for any result.''' +# try: +# c = self.connection.cursor() +# c.execute(query, params) +# self.connection.commit() +# except OperationalError as e: +# raise Exception(f'Error executing query {query}: {e}') + + +# # Yael, Riki7649255 +# def create_table(self, table_name, table_structure): +# '''create a table in a given db by given table_structure''' +# create_statement = f'''CREATE TABLE IF NOT EXISTS {table_name} ({table_structure})''' +# self.execute_query_without_results(create_statement) + +# # Riki7649255 based on rachel-8511, ShaniStrassProg +# def insert_data_into_table(self, table_name, columns, data): +# column_names = ', '.join(columns) +# placeholders = ', '.join(['?' for _ in range(len(columns))]) +# insert_query = f"INSERT INTO {table_name} ({column_names}) VALUES ({placeholders})" +# self.execute_query_without_results(insert_query, data) + +# # # Riki7649255 based on rachel-8511, Shani +# # def update_records_in_table(self, table_name: str, updates: Dict[str, Any], criteria: str) -> None: +# # '''Update records in the specified table based on criteria.''' + +# # # add documentation here +# # set_clause = ', '.join([f'{k} = ?' for k in updates.keys()]) +# # values = list(updates.values()) + +# # update_statement = f''' +# # UPDATE {table_name} +# # SET {set_clause} +# # WHERE {criteria} +# # ''' + +# # execute_query_without_results(update_statement) + + +# # def update_records_in_table(self, table_name: str, updates: Dict[str, Any], criteria: str) -> None: +# # '''Update records in the specified table based on criteria.''' + +# # set_clause = '(' + ', '.join([f'{k}' for k in updates.keys()]) + ') = (' + ', '.join([f'"{v}"' for v in updates.values()]) + ')' + +# # update_statement = f''' +# # UPDATE {table_name} +# # SET {set_clause} +# # WHERE {criteria} +# # ''' + +# # self.execute_query_without_results(update_statement) + +# # Riki7649255 based on rachel-8511, Shani +# def update_records_in_table(self, table_name: str, updates: Dict[str, Any], criteria: Optional[str]) -> None: +# '''Update records in the specified table based on criteria.''' + +# # add documentation here +# set_clause = ', '.join([f'{k} = ?' for k in updates.keys()]) +# values = tuple(updates.values()) + +# update_statement = f''' +# UPDATE {table_name} +# SET {set_clause} +# ''' +# if criteria: +# update_statement = update_statement + f'''WHERE {criteria}''' + +# self.execute_query_without_results(update_statement, values) + + +# # Riki7649255 based on rachel-8511 +# def delete_data_from_table(self, table_name: str, criteria: str) -> None: +# '''Delete a record from the specified table based on criteria.''' + +# delete_statement = f''' +# DELETE FROM {table_name} +# WHERE {criteria} +# ''' + +# execute_query_without_results(delete_statement) + + +# delete_statement = f''' +# DELETE FROM {table_name} +# WHERE {criteria} +# ''' + +# self.execute_query_without_results(delete_statement) + +# # Tem-M +# def get_columns_from_table(self, table_name): +# '''Get the columns from the specified table.''' +# try: +# get_columns_query = f"""PRAGMA table_info({table_name});""" +# cols = self.execute_query_with_multiple_results(get_columns_query) +# return [col[1] for col in cols] +# except Exception as e: +# print(f"Error occurred while fetching columns from table {table_name}: {e}") +# return [] + +# def get_all_data_from_table(self, table_name): +# try: +# get_all_data_query = f"""SELECT * FROM {table_name}""" +# return self.execute_query_with_multiple_results(get_all_data_query) +# except Exception as e: +# print(f"Error occurred while fetching data from table {table_name}: {e}") +# return [] + +# # rachel-8511, Riki7649255 +# def select_and_return_records_from_table(self, table_name: str, columns: List[str] = ['*'], criteria: Optional[str] = None) -> Dict[int, Dict[str, Any]]: +# '''Select records from the specified table based on criteria. +# Args: +# table_name (str): The name of the table. +# columns (List[str]): The columns to select. Default is all columns ('*'). +# criteria (str): SQL condition for filtering records. Default is no filter. +# Returns: +# Dict[int, Dict[str, Any]]: A dictionary where keys are object_ids and values are metadata. +# ''' +# cols = columns +# if cols == ['*']: +# cols = self.get_columns_from_table(table_name) + +# columns_clause = ', '.join(cols) +# query = f'SELECT {columns_clause} FROM {table_name}' +# if criteria: +# query += f' WHERE {criteria};' + +# try: +# results = self.execute_query_with_multiple_results(query) +# return {result[0]: dict(zip(cols if columns != ['*'] else cols[1:], result[1:])) for result in results} +# except OperationalError as e: +# raise Exception(f'Error selecting from {table_name}: {e}') +# except TypeError as e: +# raise Exception(f'Error selecting from {table_name}: {e}') + +# def is_exists_in_table(self, table_name:str, criteria:str): +# """check if rows exists in table""" +# return self.select_and_return_records_from_table(table_name, criteria=criteria) != {} + + +# # rachel-8511, ShaniStrassProg, Riki7649255 +# def describe_table(self, table_name: str) -> Dict[str, str]: +# '''Describe table structure.''' +# try: +# desc_statement = f'PRAGMA table_info({table_name})' +# columns = self.execute_query_with_multiple_results(desc_statement) +# return {col[1]: col[2] for col in columns} +# except OperationalError as e: +# raise Exception(f'Error describing table {table_name}: {e}') + +# # rachel-8511, ShaniStrassProg +# def close(self): +# '''Close the database connection.''' +# self.connection.close() + + +# # ShaniStrassProg +# # should be in ObjectManager and send the query to one of the execute_query functions +# # def is_json_column_contains_key_and_value(self, table_name: str, key: str, value: str) -> bool: +# # '''Check if a specific key-value pair exists within a JSON column in the given table.''' +# # try: +# # c = self.connection.cursor() +# # # Properly format the LIKE clause with escaped quotes for key and value +# # c.execute(f''' +# # SELECT COUNT(*) FROM {table_name} +# # WHERE metadata LIKE ? +# # LIMIT 1 +# # ''', (f'%"{key}": "{value}"%',)) +# # # Check if the count is greater than 0, indicating the key-value pair exists +# # return c.fetchone()[0] > 0 +# # except sqlite3.OperationalError as e: +# # print(f'Error: {e}') +# # return False + + +# # Yael, ShaniStrassProg +# # should be in ObjectManager and send the query to one of the execute_query functions +# # def is_identifier_exist(self, table_name: str, value: str) -> bool: +# # '''Check if a specific value exists within a column in the given table.''' +# # try: +# # c = self.connection.cursor() +# # c.execute(f''' +# # SELECT COUNT(*) FROM {table_name} +# # WHERE id LIKE ? +# # ''', (value,)) +# # return c.fetchone()[0] > 0 +# # except sqlite3.OperationalError as e: +# # print(f'Error: {e}') + + +# # sara-lea +# # should be in ObjectManager and send the query to one of the execute_query functions +# # def update_metadata(self, table_name: str, user_id: int, key: str, value: Any, action: str = 'set') -> None: +# # """ +# # Generic function to update a specific part of the metadata for a given user. + +# # Parameters: +# # - table_name: The name of the table containing the metadata. +# # - user_id: The ID of the user whose metadata needs to be updated. +# # - key: The specific part of the metadata to update (e.g., 'password', 'roles', 'policies', 'quotas'). +# # - value: The value to update, append, or delete (could be a new policy, role, password, etc.). +# # - action: The type of update to perform ('set' for replacing, 'append' for adding to lists, 'update' for dicts, 'delete' for removing). +# # """ + +# # # Step 1: Retrieve the current metadata for the user +# # query = f"SELECT metadata FROM {table_name} WHERE user_id = ?" +# # try: +# # c = self.connection.cursor() +# # c.execute(query, (user_id,)) +# # row = c.fetchone() +# # if row: +# # metadata = json.loads(row[0]) # Assuming metadata is stored as a JSON string +# # else: +# # raise Exception(f"User with id {user_id} not found.") +# # except OperationalError as e: +# # raise Exception(f'Error fetching metadata: {e}') + +# # # Step 2: Modify the relevant part of the metadata +# # if key not in metadata: +# # raise KeyError(f"Key '{key}' not found in metadata.") + +# # if action == 'set': +# # # Replace the value of the key directly +# # metadata[key] = value +# # elif action == 'append' and isinstance(metadata[key], list): +# # # Append to the list (for roles, policies, etc.) +# # metadata[key].append(value) +# # elif action == 'update' and isinstance(metadata[key], dict): +# # # Update a dictionary (for quotas or nested data) +# # metadata[key].update(value) +# # elif action == 'delete': +# # if isinstance(metadata[key], list): +# # # Remove an item from a list +# # if value in metadata[key]: +# # metadata[key].remove(value) +# # else: +# # raise ValueError(f"Value '{value}' not found in list '{key}'.") +# # elif isinstance(metadata[key], dict): +# # # Remove a key from a dictionary +# # if value in metadata[key]: +# # del metadata[key][value] +# # else: +# # raise ValueError(f"Key '{value}' not found in dictionary '{key}'.") +# # else: +# # raise ValueError(f"Action 'delete' is not supported for the data type of key '{key}'.") +# # else: +# # raise ValueError(f"Invalid action '{action}' or incompatible data type for key '{key}'.") + +# # # Step 3: Serialize metadata back to JSON string +# # updated_metadata = json.dumps(metadata) + +# # # Step 4: Use your update function to write the new metadata back to the database +# # updates = {"metadata": updated_metadata} +# # criteria = f"user_id = {user_id}" +# # self.update(table_name, updates, criteria) + + +# import sqlite3 +# from typing import Dict, Any, List, Optional, Tuple +# import json +# from sqlite3 import OperationalError + +# class DBManager: +# def __init__(self, db_file: str): +# '''Initialize the database connection and create tables if they do not exist.''' +# self.connection = sqlite3.connect(db_file) + + +# # saraNoigershel +# def execute_query_with_multiple_results(self, query: str, params:Tuple = ()) -> Optional[List[Tuple]]: +# '''Execute a given query and return the results.''' +# try: +# c = self.connection.cursor() +# c.execute(query, params) +# results = c.fetchall() +# self.connection.commit() +# return results if results else None +# except OperationalError as e: +# raise Exception(f'Error executing query {query}: {e}') + + +# # ShaniStrassProg +# def execute_query_with_single_result(self, query: str, params:Tuple = ()) -> Optional[Tuple]: +# '''Execute a given query and return a single result.''' +# try: +# c = self.connection.cursor() +# c.execute(query, params) +# result = c.fetchone() +# self.connection.commit() +# return result if result else None + +# except OperationalError as e: +# raise Exception(f'Error executing query {query}: {e}') + + +# # Riki7649255 +# def execute_query_without_results(self, query: str, params:Tuple = ()): +# '''Execute a given query without waiting for any result.''' +# try: +# c = self.connection.cursor() +# c.execute(query, params) +# self.connection.commit() +# except OperationalError as e: +# raise Exception(f'Error executing query {query}: {e}') + + +# # Yael, Riki7649255 +# def create_table(self, table_name, table_structure): +# '''create a table in a given db by given table_structure''' +# create_statement = f'''CREATE TABLE IF NOT EXISTS {table_name} ({table_structure})''' +# self.execute_query_without_results(create_statement) + +# # Riki7649255 based on rachel-8511, ShaniStrassProg +# def insert_data_into_table(self, table_name, columns, data): +# column_names = ', '.join(columns) +# placeholders = ', '.join(['?' for _ in range(len(columns))]) +# insert_query = f"INSERT INTO {table_name} ({column_names}) VALUES ({placeholders})" +# self.execute_query_without_results(insert_query, data) + + +# # Riki7649255 based on rachel-8511, Shani +# def update_records_in_table(self, table_name: str, updates: Dict[str, Any], criteria: Optional[str]) -> None: +# '''Update records in the specified table based on criteria.''' + +# # add documentation here +# set_clause = ', '.join([f'{k} = ?' for k in updates.keys()]) +# values = tuple(updates.values()) + +# update_statement = f''' +# UPDATE {table_name} +# SET {set_clause} +# ''' +# if criteria: +# update_statement = update_statement + f'''WHERE {criteria}''' + +# self.execute_query_without_results(update_statement, values) + + +# # Riki7649255 based on rachel-8511 +# def delete_data_from_table(self, table_name: str, criteria: str) -> None: +# '''Delete a record from the specified table based on criteria.''' + +# delete_statement = f''' +# DELETE FROM {table_name} +# WHERE {criteria} +# ''' + +# self.execute_query_without_results(delete_statement) + +# # Tem-M +# def get_columns_from_table(self, table_name): +# '''Get the columns from the specified table.''' +# try: +# get_columns_query = f"""PRAGMA table_info({table_name});""" +# cols = self.execute_query_with_multiple_results(get_columns_query) +# return [col[1] for col in cols] +# except Exception as e: +# print(f"Error occurred while fetching columns from table {table_name}: {e}") +# return [] + +# def get_all_data_from_table(self, table_name): +# try: +# get_all_data_query = f"""SELECT * FROM {table_name}""" +# return self.execute_query_with_multiple_results(get_all_data_query) +# except Exception as e: +# print(f"Error occurred while fetching data from table {table_name}: {e}") +# return [] + +# # rachel-8511, Riki7649255 +# def select_and_return_records_from_table(self, table_name: str, columns: List[str] = ['*'], criteria: Optional[str] = None) -> Dict[int, Dict[str, Any]]: +# '''Select records from the specified table based on criteria. +# Args: +# table_name (str): The name of the table. +# columns (List[str]): The columns to select. Default is all columns ('*'). +# criteria (str): SQL condition for filtering records. Default is no filter. +# Returns: +# Dict[int, Dict[str, Any]]: A dictionary where keys are object_ids and values are metadata. +# ''' +# cols = columns +# if cols == ['*']: +# cols = self.get_columns_from_table(table_name) + +# columns_clause = ', '.join(cols) +# query = f'SELECT {columns_clause} FROM {table_name}' +# if criteria: +# query += f' WHERE {criteria};' + +# try: +# results = self.execute_query_with_multiple_results(query) +# return {result[0]: dict(zip(cols if columns != ['*'] else cols[1:], result[1:])) for result in results} +# except OperationalError as e: +# raise Exception(f'Error selecting from {table_name}: {e}') +# except TypeError as e: +# raise Exception(f'Error selecting from {table_name}: {e}') + +# def is_exists_in_table(self, table_name:str, criteria:str): +# """check if rows exists in table""" +# return self.select_and_return_records_from_table(table_name, criteria=criteria) != {} + + +# # rachel-8511, ShaniStrassProg, Riki7649255 +# def describe_table(self, table_name: str) -> Dict[str, str]: +# '''Describe table structure.''' +# try: +# desc_statement = f'PRAGMA table_info({table_name})' +# columns = self.execute_query_with_multiple_results(desc_statement) +# return {col[1]: col[2] for col in columns} +# except OperationalError as e: +# raise Exception(f'Error describing table {table_name}: {e}') + +# # rachel-8511, ShaniStrassProg +# def close(self): +# '''Close the database connection.''' +# self.connection.close() + + +# # ShaniStrassProg +# # should be in ObjectManager and send the query to one of the execute_query functions +# # def is_json_column_contains_key_and_value(self, table_name: str, key: str, value: str) -> bool: +# # '''Check if a specific key-value pair exists within a JSON column in the given table.''' +# # try: +# # c = self.connection.cursor() +# # # Properly format the LIKE clause with escaped quotes for key and value +# # c.execute(f''' +# # SELECT COUNT(*) FROM {table_name} +# # WHERE metadata LIKE ? +# # LIMIT 1 +# # ''', (f'%"{key}": "{value}"%',)) +# # # Check if the count is greater than 0, indicating the key-value pair exists +# # return c.fetchone()[0] > 0 +# # except sqlite3.OperationalError as e: +# # print(f'Error: {e}') +# # return False + + +# # Yael, ShaniStrassProg +# # should be in ObjectManager and send the query to one of the execute_query functions +# # def is_identifier_exist(self, table_name: str, value: str) -> bool: +# # '''Check if a specific value exists within a column in the given table.''' +# # try: +# # c = self.connection.cursor() +# # c.execute(f''' +# # SELECT COUNT(*) FROM {table_name} +# # WHERE id LIKE ? +# # ''', (value,)) +# # return c.fetchone()[0] > 0 +# # except sqlite3.OperationalError as e: +# # print(f'Error: {e}') + + +# # sara-lea +# # should be in ObjectManager and send the query to one of the execute_query functions +# # def update_metadata(self, table_name: str, user_id: int, key: str, value: Any, action: str = 'set') -> None: +# # """ +# # Generic function to update a specific part of the metadata for a given user. + +# # Parameters: +# # - table_name: The name of the table containing the metadata. +# # - user_id: The ID of the user whose metadata needs to be updated. +# # - key: The specific part of the metadata to update (e.g., 'password', 'roles', 'policies', 'quotas'). +# # - value: The value to update, append, or delete (could be a new policy, role, password, etc.). +# # - action: The type of update to perform ('set' for replacing, 'append' for adding to lists, 'update' for dicts, 'delete' for removing). +# # """ + +# # # Step 1: Retrieve the current metadata for the user +# # query = f"SELECT metadata FROM {table_name} WHERE user_id = ?" +# # try: +# # c = self.connection.cursor() +# # c.execute(query, (user_id,)) +# # row = c.fetchone() +# # if row: +# # metadata = json.loads(row[0]) # Assuming metadata is stored as a JSON string +# # else: +# # raise Exception(f"User with id {user_id} not found.") +# # except OperationalError as e: +# # raise Exception(f'Error fetching metadata: {e}') + +# # # Step 2: Modify the relevant part of the metadata +# # if key not in metadata: +# # raise KeyError(f"Key '{key}' not found in metadata.") + +# # if action == 'set': +# # # Replace the value of the key directly +# # metadata[key] = value +# # elif action == 'append' and isinstance(metadata[key], list): +# # # Append to the list (for roles, policies, etc.) +# # metadata[key].append(value) +# # elif action == 'update' and isinstance(metadata[key], dict): +# # # Update a dictionary (for quotas or nested data) +# # metadata[key].update(value) +# # elif action == 'delete': +# # if isinstance(metadata[key], list): +# # # Remove an item from a list +# # if value in metadata[key]: +# # metadata[key].remove(value) +# # else: +# # raise ValueError(f"Value '{value}' not found in list '{key}'.") +# # elif isinstance(metadata[key], dict): +# # # Remove a key from a dictionary +# # if value in metadata[key]: +# # del metadata[key][value] +# # else: +# # raise ValueError(f"Key '{value}' not found in dictionary '{key}'.") +# # else: +# # raise ValueError(f"Action 'delete' is not supported for the data type of key '{key}'.") +# # else: +# # raise ValueError(f"Invalid action '{action}' or incompatible data type for key '{key}'.") + +# # # Step 3: Serialize metadata back to JSON string +# # updated_metadata = json.dumps(metadata) + +# # # Step 4: Use your update function to write the new metadata back to the database +# # updates = {"metadata": updated_metadata} +# # criteria = f"user_id = {user_id}" +# # self.update(table_name, updates, criteria) + + import sqlite3 from typing import Dict, Any, List, Optional, Tuple import json @@ -8,22 +580,6 @@ def __init__(self, db_file: str): '''Initialize the database connection and create tables if they do not exist.''' self.connection = sqlite3.connect(db_file) - - # rachel-8511, ShaniStrassProg - def close(self): - '''Close the database connection.''' - self.connection.close() - - - # # saraNoigershel - # def execute_query_with_multiple_results(self, query: str) -> Optional[List[Tuple]]: - # '''Execute a given query and return the results.''' - # try: - # c = self.connection.cursor() - # c.execute(query) - # results = c.fetchall() - # # self.connection.commit() ??? - # saraNoigershel def execute_query_with_multiple_results(self, query: str, params:Tuple = ()) -> Optional[List[Tuple]]: '''Execute a given query and return the results.''' @@ -31,22 +587,22 @@ def execute_query_with_multiple_results(self, query: str, params:Tuple = ()) -> c = self.connection.cursor() c.execute(query, params) results = c.fetchall() - self.connection.commit() + self.connection.commit() return results if results else None except OperationalError as e: raise Exception(f'Error executing query {query}: {e}') - # ShaniStrassProg + # ShaniStrassProg def execute_query_with_single_result(self, query: str, params:Tuple = ()) -> Optional[Tuple]: '''Execute a given query and return a single result.''' try: c = self.connection.cursor() c.execute(query, params) result = c.fetchone() - self.connection.commit() + self.connection.commit() return result if result else None - + except OperationalError as e: raise Exception(f'Error executing query {query}: {e}') @@ -60,7 +616,8 @@ def execute_query_without_results(self, query: str, params:Tuple = ()): self.connection.commit() except OperationalError as e: raise Exception(f'Error executing query {query}: {e}') - + + # Yael, Riki7649255 def create_table(self, table_name, table_structure): @@ -68,42 +625,11 @@ def create_table(self, table_name, table_structure): create_statement = f'''CREATE TABLE IF NOT EXISTS {table_name} ({table_structure})''' self.execute_query_without_results(create_statement) - # Riki7649255 based on rachel-8511, ShaniStrassProg - def insert_data_into_table(self, table_name, columns, data): - column_names = ', '.join(columns) - placeholders = ', '.join(['?' for _ in range(len(columns))]) - insert_query = f"INSERT INTO {table_name} ({column_names}) VALUES ({placeholders})" - self.execute_query_without_results(insert_query, data) - - # # Riki7649255 based on rachel-8511, Shani - # def update_records_in_table(self, table_name: str, updates: Dict[str, Any], criteria: str) -> None: - # '''Update records in the specified table based on criteria.''' - - # # add documentation here - # set_clause = ', '.join([f'{k} = ?' for k in updates.keys()]) - # values = list(updates.values()) - - # update_statement = f''' - # UPDATE {table_name} - # SET {set_clause} - # WHERE {criteria} - # ''' - - # execute_query_without_results(update_statement) - - # def update_records_in_table(self, table_name: str, updates: Dict[str, Any], criteria: str) -> None: - # '''Update records in the specified table based on criteria.''' + def insert_data_into_table(self, table_name, data): + insert_statement = f'''INSERT INTO {table_name} VALUES {data}''' + self.execute_query_without_results(insert_statement) - # set_clause = '(' + ', '.join([f'{k}' for k in updates.keys()]) + ') = (' + ', '.join([f'"{v}"' for v in updates.values()]) + ')' - - # update_statement = f''' - # UPDATE {table_name} - # SET {set_clause} - # WHERE {criteria} - # ''' - - # self.execute_query_without_results(update_statement) # Riki7649255 based on rachel-8511, Shani def update_records_in_table(self, table_name: str, updates: Dict[str, Any], criteria: Optional[str]) -> None: @@ -112,7 +638,7 @@ def update_records_in_table(self, table_name: str, updates: Dict[str, Any], crit # add documentation here set_clause = ', '.join([f'{k} = ?' for k in updates.keys()]) values = tuple(updates.values()) - + update_statement = f''' UPDATE {table_name} SET {set_clause} @@ -126,14 +652,6 @@ def update_records_in_table(self, table_name: str, updates: Dict[str, Any], crit # Riki7649255 based on rachel-8511 def delete_data_from_table(self, table_name: str, criteria: str) -> None: '''Delete a record from the specified table based on criteria.''' - - delete_statement = f''' - DELETE FROM {table_name} - WHERE {criteria} - ''' - - execute_query_without_results(delete_statement) - delete_statement = f''' DELETE FROM {table_name} @@ -152,7 +670,7 @@ def get_columns_from_table(self, table_name): except Exception as e: print(f"Error occurred while fetching columns from table {table_name}: {e}") return [] - + def get_all_data_from_table(self, table_name): try: get_all_data_query = f"""SELECT * FROM {table_name}""" @@ -160,7 +678,7 @@ def get_all_data_from_table(self, table_name): except Exception as e: print(f"Error occurred while fetching data from table {table_name}: {e}") return [] - + # rachel-8511, Riki7649255 def select_and_return_records_from_table(self, table_name: str, columns: List[str] = ['*'], criteria: Optional[str] = None) -> Dict[int, Dict[str, Any]]: '''Select records from the specified table based on criteria. @@ -174,24 +692,24 @@ def select_and_return_records_from_table(self, table_name: str, columns: List[st cols = columns if cols == ['*']: cols = self.get_columns_from_table(table_name) - + columns_clause = ', '.join(cols) query = f'SELECT {columns_clause} FROM {table_name}' if criteria: query += f' WHERE {criteria};' - + try: results = self.execute_query_with_multiple_results(query) return {result[0]: dict(zip(cols if columns != ['*'] else cols[1:], result[1:])) for result in results} except OperationalError as e: - raise Exception(f'Error selecting from {table_name}: {e}') + raise Exception(f'Error selecting from {table_name}: {e}') except TypeError as e: raise Exception(f'Error selecting from {table_name}: {e}') - + def is_exists_in_table(self, table_name:str, criteria:str): """check if rows exists in table""" return self.select_and_return_records_from_table(table_name, criteria=criteria) != {} - + # rachel-8511, ShaniStrassProg, Riki7649255 def describe_table(self, table_name: str) -> Dict[str, str]: @@ -202,109 +720,8 @@ def describe_table(self, table_name: str) -> Dict[str, str]: return {col[1]: col[2] for col in columns} except OperationalError as e: raise Exception(f'Error describing table {table_name}: {e}') - + # rachel-8511, ShaniStrassProg def close(self): '''Close the database connection.''' - self.connection.close() - - - # ShaniStrassProg - # should be in ObjectManager and send the query to one of the execute_query functions - # def is_json_column_contains_key_and_value(self, table_name: str, key: str, value: str) -> bool: - # '''Check if a specific key-value pair exists within a JSON column in the given table.''' - # try: - # c = self.connection.cursor() - # # Properly format the LIKE clause with escaped quotes for key and value - # c.execute(f''' - # SELECT COUNT(*) FROM {table_name} - # WHERE metadata LIKE ? - # LIMIT 1 - # ''', (f'%"{key}": "{value}"%',)) - # # Check if the count is greater than 0, indicating the key-value pair exists - # return c.fetchone()[0] > 0 - # except sqlite3.OperationalError as e: - # print(f'Error: {e}') - # return False - - - # Yael, ShaniStrassProg - # should be in ObjectManager and send the query to one of the execute_query functions - # def is_identifier_exist(self, table_name: str, value: str) -> bool: - # '''Check if a specific value exists within a column in the given table.''' - # try: - # c = self.connection.cursor() - # c.execute(f''' - # SELECT COUNT(*) FROM {table_name} - # WHERE id LIKE ? - # ''', (value,)) - # return c.fetchone()[0] > 0 - # except sqlite3.OperationalError as e: - # print(f'Error: {e}') - - - # sara-lea - # should be in ObjectManager and send the query to one of the execute_query functions - # def update_metadata(self, table_name: str, user_id: int, key: str, value: Any, action: str = 'set') -> None: - # """ - # Generic function to update a specific part of the metadata for a given user. - - # Parameters: - # - table_name: The name of the table containing the metadata. - # - user_id: The ID of the user whose metadata needs to be updated. - # - key: The specific part of the metadata to update (e.g., 'password', 'roles', 'policies', 'quotas'). - # - value: The value to update, append, or delete (could be a new policy, role, password, etc.). - # - action: The type of update to perform ('set' for replacing, 'append' for adding to lists, 'update' for dicts, 'delete' for removing). - # """ - - # # Step 1: Retrieve the current metadata for the user - # query = f"SELECT metadata FROM {table_name} WHERE user_id = ?" - # try: - # c = self.connection.cursor() - # c.execute(query, (user_id,)) - # row = c.fetchone() - # if row: - # metadata = json.loads(row[0]) # Assuming metadata is stored as a JSON string - # else: - # raise Exception(f"User with id {user_id} not found.") - # except OperationalError as e: - # raise Exception(f'Error fetching metadata: {e}') - - # # Step 2: Modify the relevant part of the metadata - # if key not in metadata: - # raise KeyError(f"Key '{key}' not found in metadata.") - - # if action == 'set': - # # Replace the value of the key directly - # metadata[key] = value - # elif action == 'append' and isinstance(metadata[key], list): - # # Append to the list (for roles, policies, etc.) - # metadata[key].append(value) - # elif action == 'update' and isinstance(metadata[key], dict): - # # Update a dictionary (for quotas or nested data) - # metadata[key].update(value) - # elif action == 'delete': - # if isinstance(metadata[key], list): - # # Remove an item from a list - # if value in metadata[key]: - # metadata[key].remove(value) - # else: - # raise ValueError(f"Value '{value}' not found in list '{key}'.") - # elif isinstance(metadata[key], dict): - # # Remove a key from a dictionary - # if value in metadata[key]: - # del metadata[key][value] - # else: - # raise ValueError(f"Key '{value}' not found in dictionary '{key}'.") - # else: - # raise ValueError(f"Action 'delete' is not supported for the data type of key '{key}'.") - # else: - # raise ValueError(f"Invalid action '{action}' or incompatible data type for key '{key}'.") - - # # Step 3: Serialize metadata back to JSON string - # updated_metadata = json.dumps(metadata) - - # # Step 4: Use your update function to write the new metadata back to the database - # updates = {"metadata": updated_metadata} - # criteria = f"user_id = {user_id}" - # self.update(table_name, updates, criteria) \ No newline at end of file + self.connection.close() \ No newline at end of file diff --git a/DB/NEW_KT_DB/DataAccess/ObjectManager.py b/DB/NEW_KT_DB/DataAccess/ObjectManager.py index 018ec64d..140fb971 100644 --- a/DB/NEW_KT_DB/DataAccess/ObjectManager.py +++ b/DB/NEW_KT_DB/DataAccess/ObjectManager.py @@ -1,7 +1,329 @@ +# from typing import Dict, Any, Optional +# import json +# import sqlite3 +# from .DBManager import DBManager + +# class ObjectManager: +# def __init__(self, db_file: str): +# '''Initialize ObjectManager with the database connection.''' +# self.db_manager = DBManager(db_file) + + +# # for internal use only: + +# # Riki7649255 based on rachel-8511 + +# # def create_management_table(self, table_name, table_structure='object_id INTEGER PRIMARY KEY AUTOINCREMENT,type_object TEXT NOT NULL,metadata TEXT NOT NULL'): + +# def _create_management_table(self, table_name, table_structure='object_id INTEGER PRIMARY KEY AUTOINCREMENT,type_object TEXT NOT NULL,metadata TEXT NOT NULL'): +# """ +# creates a management table with the name and the structure you specify +# make sure to keep track of the table name you send here - you will use it whenever you want to access the table +# you created - this function should only be called from within the specific manager you created (i.e. DBInstanceManager) +# """ +# self.db_manager.create_table(table_name, table_structure) + + +# # Riki7649255 based on saraNoigershel, Tem-M +# def _insert_object_to_management_table(self, table_name, object): +# """ +# inserts an object to the management table you specified, the object should be sent as is! not converted to a tuple or dictionary! +# if the table does not exist, the function will abort and raise an error +# the table should be created within the __init__ function of the manager you created (i.e. DBInstanceManager) +# """ +# columns = self.db_manager.get_columns_from_table(table_name) +# values = tuple([str(getattr(object, column)) for column in columns]) +# self.db_manager.insert_data_into_table(table_name, columns, values) + +# # Malki1844 +# def _get_all_data_from_table(self, table_name): +# self.db_manager.get_all_data_from_table(table_name) + +# # Riki7649255 based on rachel-8511 +# def _update_object_in_management_table_by_criteria(self, table_name, updates, criteria): +# updates = {k: str(v) for k, v in updates.items()} +# self.db_manager.update_records_in_table(table_name, updates, criteria) + + + +# # rachel-8511, Riki7649255 +# def _get_object_from_management_table(self, pk_col, table_name, object_id: int) -> Dict[str, Any]: +# '''Retrieve an object from the database.''' +# result = self.db_manager.select_and_return_records_from_table(table_name=table_name, criteria=f'{pk_col} = \'{object_id}\'') +# if result: +# return result +# else: +# raise FileNotFoundError(f'Object with ID {object_id} not found.') + +# def _get_objects_from_management_table_by_criteria(self, table_name, columns = ["*"], criteria:Optional[str] = None) -> Dict: +# '''Retrieve an object from the database.''' +# result = self.db_manager.select_and_return_records_from_table(table_name, columns, criteria) +# if result: +# return result +# else: +# raise FileNotFoundError(f'Objects with criteria {criteria} not found.') + + +# # rachel-8511, ShaniStrassProg, Riki7649255 +# def _delete_object_from_management_table(self, table_name, criteria) -> None: +# '''Delete an object from the database.''' +# self.db_manager.delete_data_from_table(table_name, criteria) + +# def _delete_object_from_management_table_by_id(self, pk_col, table_name, object_id) -> None: +# '''Delete an object from the database.''' +# self.db_manager.delete_data_from_table(table_name, criteria= f'{pk_col} = \'{object_id}\'') + + +# # rachel-8511, ShaniStrassProg is it needed? +# # def get_all_objects(self) -> Dict[int, Dict[str, Any]]: +# # '''Retrieve all objects from the database.''' +# # return self.db_manager.select(self.table_name, ['object_id', 'type_object', 'metadata']) + + +# # rachel-8511 is it needed? +# # def describe_table(self) -> Dict[str, str]: +# # '''Describe the schema of the table.''' +# # return self.db_manager.describe(self.table_name) + + +# def _convert_object_name_to_management_table_name(self, object_name): +# return f'mng_{object_name}s' + + + +# # def is_management_table_exist(table_name): +# # # check if table exists using single result query +# # return self.db_manager.execute_query_with_single_result(f'desc table {table_name}') + + +# # for outer use: +# # def save_in_memory(self, object): + +# def _is_management_table_exist(self, table_name): +# # Check if table exists by querying the sqlite_master table +# query = f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}'" +# return self.db_manager.execute_query_with_single_result(query) + + +# # for outer use: +# # def save_in_memory(self, table_name, object): + +# # # insert object info into management table mng_{object_name}s +# # # for exmple: object db_instance will be saved in table mng_db_instances +# # table_name = self.convert_object_name_to_management_table_name(self.object_name) + +# # if not self.is_management_table_exist(table_name): +# # self.create_management_table(table_name) + +# # self.insert_object_to_management_table(table_name, object) + +# def save_in_memory(self, object): +# # insert object info into management table mng_{object_name}s +# # for exmple: object db_instance will be saved in table mng_db_instances + +# table_name = str(object.__class__.__name__) +# if not self._is_management_table_exist(table_name): +# self._create_management_table(table_name) +# # self.insert_object_to_management_table(table_name, object) + +# self._insert_object_to_management_table(table_name, object) + + +# def delete_from_memory_by_id(self, pk_col, pk_val, table_name:str): +# # pk_val is the object id +# # if criteria not sent- use PK for deletion +# criteria = f'{pk_col} = \'{pk_val}\'' + +# table_name = self.convert_object_name_to_management_table_name(self.object_name) + +# self.delete_data_from_table(table_name, criteria) + + +# def update_in_memory(self, updates, criteria='default'): + +# # if criteria not sent- use PK for deletion +# if criteria == 'default': +# criteria = f'{self.pk_column} = {self.pk_value}' + +# table_name = self.convert_object_name_to_management_table_name(self.object_name) + +# self.update_object_in_management_table_by_criteria(table_name, updates, criteria) + + +# def get_from_memory(self): +# self.get_object_from_management_table(self.object_id) + +# self.db_manager.delete_data_from_table(table_name, criteria) + +# def update_in_memory_by_criteria(self,table_name:str, updates:Dict, criteria): +# self._update_object_in_management_table_by_criteria(table_name, updates, criteria) + +# def update_in_memory_by_id(self, pk_col, table_name, updates, object_id:Optional[str]): +# if not object_id: +# raise ValueError('must be or criteria or object id') +# criteria = f'{pk_col} = \'{object_id}\'' +# self.update_in_memory_by_criteria(table_name, updates, criteria) + + +# def get_from_memory_by_id(self, pk_col, table_name, object_id, columns = ["*"]): +# """get records from memory by criteria or id""" +# criteria = f'{pk_col} = \'{object_id}\'' +# return self._get_objects_from_management_table_by_criteria(table_name, columns, criteria) + + +# def convert_object_attributes_to_dictionary(**kwargs): + +# dict = {} + +# for key, value in kwargs.items(): +# dict[key] = value + +# return dict + + +# from typing import Dict, Any, Optional +# import json +# import sqlite3 +# from .DBManager import DBManager + +# class ObjectManager: +# def __init__(self, db_file: str): +# '''Initialize ObjectManager with the database connection.''' +# self.db_manager = DBManager(db_file) + + +# # for internal use only: + +# # Riki7649255 based on rachel-8511 +# def _create_management_table(self, table_name, table_structure='object_id INTEGER PRIMARY KEY AUTOINCREMENT,type_object TEXT NOT NULL,metadata TEXT NOT NULL'): +# """ +# creates a management table with the name and the structure you specify +# make sure to keep track of the table name you send here - you will use it whenever you want to access the table +# you created - this function should only be called from within the specific manager you created (i.e. DBInstanceManager) +# """ +# self.db_manager.create_table(table_name, table_structure) + + +# # Riki7649255 based on saraNoigershel, Tem-M +# def _insert_object_to_management_table(self, table_name, object): +# """ +# inserts an object to the management table you specified, the object should be sent as is! not converted to a tuple or dictionary! +# if the table does not exist, the function will abort and raise an error +# the table should be created within the __init__ function of the manager you created (i.e. DBInstanceManager) +# """ +# columns = self.db_manager.get_columns_from_table(table_name) +# values = tuple([str(getattr(object, column)) for column in columns]) +# self.db_manager.insert_data_into_table(table_name, columns, values) + +# # Malki1844 +# def _get_all_data_from_table(self, table_name): +# self.db_manager.get_all_data_from_table(table_name) + +# # Riki7649255 based on rachel-8511 +# def _update_object_in_management_table_by_criteria(self, table_name, updates, criteria): +# updates = {k: str(v) for k, v in updates.items()} +# self.db_manager.update_records_in_table(table_name, updates, criteria) + + + +# # rachel-8511, Riki7649255 +# def _get_object_from_management_table(self, pk_col, table_name, object_id: int) -> Dict[str, Any]: +# '''Retrieve an object from the database.''' +# result = self.db_manager.select_and_return_records_from_table(table_name=table_name, criteria=f'{pk_col} = \'{object_id}\'') +# if result: +# return result +# else: +# raise FileNotFoundError(f'Object with ID {object_id} not found.') + +# def _get_objects_from_management_table_by_criteria(self, table_name, columns = ["*"], criteria:Optional[str] = None) -> Dict: +# '''Retrieve an object from the database.''' +# result = self.db_manager.select_and_return_records_from_table(table_name, columns, criteria) +# if result: +# return result +# else: +# raise FileNotFoundError(f'Objects with criteria {criteria} not found.') + + +# # rachel-8511, ShaniStrassProg, Riki7649255 +# def _delete_object_from_management_table(self, table_name, criteria) -> None: +# '''Delete an object from the database.''' +# self.db_manager.delete_data_from_table(table_name, criteria) + +# def _delete_object_from_management_table_by_id(self, pk_col, table_name, object_id) -> None: +# '''Delete an object from the database.''' +# self.db_manager.delete_data_from_table(table_name, criteria= f'{pk_col} = \'{object_id}\'') + + +# # rachel-8511, ShaniStrassProg is it needed? +# # def get_all_objects(self) -> Dict[int, Dict[str, Any]]: +# # '''Retrieve all objects from the database.''' +# # return self.db_manager.select(self.table_name, ['object_id', 'type_object', 'metadata']) + + +# # rachel-8511 is it needed? +# # def describe_table(self) -> Dict[str, str]: +# # '''Describe the schema of the table.''' +# # return self.db_manager.describe(self.table_name) + + +# def _convert_object_name_to_management_table_name(self, object_name): +# return f'mng_{object_name}s' + + +# def _is_management_table_exist(self, table_name): +# # Check if table exists by querying the sqlite_master table +# query = f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}'" +# return self.db_manager.execute_query_with_single_result(query) + + +# # for outer use: +# def save_in_memory(self, table_name, object): + +# # insert object info into management table mng_{object_name}s +# # for exmple: object db_instance will be saved in table mng_db_instances + +# self._insert_object_to_management_table(table_name, object) + + +# def delete_from_memory_by_id(self, pk_col, pk_val, table_name:str): +# # pk_val is the object id +# # if criteria not sent- use PK for deletion +# criteria = f'{pk_col} = \'{pk_val}\'' + +# self.db_manager.delete_data_from_table(table_name, criteria) + +# def update_in_memory_by_criteria(self,table_name:str, updates:Dict, criteria): +# self._update_object_in_management_table_by_criteria(table_name, updates, criteria) + +# def update_in_memory_by_id(self, pk_col, table_name, updates, object_id:Optional[str]): +# if not object_id: +# raise ValueError('must be or criteria or object id') +# criteria = f'{pk_col} = \'{object_id}\'' +# self.update_in_memory_by_criteria(table_name, updates, criteria) + + +# def get_from_memory_by_id(self, pk_col, table_name, object_id, columns = ["*"]): +# """get records from memory by criteria or id""" +# criteria = f'{pk_col} = \'{object_id}\'' +# return self._get_objects_from_management_table_by_criteria(table_name, columns, criteria) + + +# def convert_object_attributes_to_dictionary(**kwargs): + +# dict = {} + +# for key, value in kwargs.items(): +# dict[key] = value + +# return dict + + + + from typing import Dict, Any, Optional import json import sqlite3 -from .DBManager import DBManager +from DB.NEW_KT_DB.DataAccess.DBManager import DBManager class ObjectManager: def __init__(self, db_file: str): @@ -12,52 +334,44 @@ def __init__(self, db_file: str): # for internal use only: # Riki7649255 based on rachel-8511 + def create_management_table(self, table_name, table_structure='object_id INTEGER PRIMARY KEY AUTOINCREMENT,type_object TEXT NOT NULL,metadata TEXT NOT NULL'): + self.db_manager.create_table(table_name, table_structure) - # def create_management_table(self, table_name, table_structure='object_id INTEGER PRIMARY KEY AUTOINCREMENT,type_object TEXT NOT NULL,metadata TEXT NOT NULL'): - - def _create_management_table(self, table_name, table_structure='object_id INTEGER PRIMARY KEY AUTOINCREMENT,type_object TEXT NOT NULL,metadata TEXT NOT NULL'): - """ - creates a management table with the name and the structure you specify - make sure to keep track of the table name you send here - you will use it whenever you want to access the table - you created - this function should only be called from within the specific manager you created (i.e. DBInstanceManager) - """ + def create_management_table_with_str_id(self, table_name, table_structure='object_id TEXT NOT NULL PRIMARY KEY ,metadata TEXT NOT NULL'): self.db_manager.create_table(table_name, table_structure) - - # Riki7649255 based on saraNoigershel, Tem-M - def _insert_object_to_management_table(self, table_name, object): - """ - inserts an object to the management table you specified, the object should be sent as is! not converted to a tuple or dictionary! - if the table does not exist, the function will abort and raise an error - the table should be created within the __init__ function of the manager you created (i.e. DBInstanceManager) - """ - columns = self.db_manager.get_columns_from_table(table_name) - values = tuple([str(getattr(object, column)) for column in columns]) - self.db_manager.insert_data_into_table(table_name, columns, values) - - # Malki1844 - def _get_all_data_from_table(self, table_name): - self.db_manager.get_all_data_from_table(table_name) + def insert_object_to_management_table(self, table_name, object): + self.db_manager.insert_data_into_table(table_name, object) + + # def insert_object_to_management_table_with_str_id(self, table_name, id, metadata): + # # self.db_manager.insert_data_into_table(table_name, ['object_id','metadata'],(id, metadata)) + + # def insert_object_to_management_table_with_str_id(self, table_name, id, metadata): + # self.db_manager.insert_data_into_table(table_name, ['object_id','metadata'],(id, metadata)) + + def update_object_in_management_table_by_id(self, table_name, object_id, updates): + self.db_manager.update_records_in_table(table_name, updates, f'object_id = {object_id}') # Riki7649255 based on rachel-8511 - def _update_object_in_management_table_by_criteria(self, table_name, updates, criteria): - updates = {k: str(v) for k, v in updates.items()} + def update_object_in_management_table_by_criteria(self, table_name, updates, criteria): self.db_manager.update_records_in_table(table_name, updates, criteria) - + + # def update_object_in_management_table_by_id(self, table_name, object_id, updates): + # self.db_manager.update_records_in_table(table_name, updates, f'object_id = {object_id}') # rachel-8511, Riki7649255 - def _get_object_from_management_table(self, pk_col, table_name, object_id: int) -> Dict[str, Any]: + def get_object_from_management_table(self, table_name, object_id: int, columns = ["*"]) -> Dict[str, Any]: '''Retrieve an object from the database.''' - result = self.db_manager.select_and_return_records_from_table(table_name=table_name, criteria=f'{pk_col} = \'{object_id}\'') + result = self.db_manager.select_and_return_records_from_table(table_name, columns, criteria= f'object_id = {object_id}') if result: - return result + return result[object_id] else: raise FileNotFoundError(f'Object with ID {object_id} not found.') - - def _get_objects_from_management_table_by_criteria(self, table_name, columns = ["*"], criteria:Optional[str] = None) -> Dict: + + def get_objects_from_management_table_by_criteria(self, object_id: int, columns = ["*"], criteria:Optional[str] = None) -> Dict: '''Retrieve an object from the database.''' - result = self.db_manager.select_and_return_records_from_table(table_name, columns, criteria) + result = self.db_manager.select_and_return_records_from_table(self.table_name, columns, criteria) if result: return result else: @@ -65,13 +379,13 @@ def _get_objects_from_management_table_by_criteria(self, table_name, columns = [ # rachel-8511, ShaniStrassProg, Riki7649255 - def _delete_object_from_management_table(self, table_name, criteria) -> None: + def delete_object_from_management_table(self, table_name, criteria) -> None: '''Delete an object from the database.''' self.db_manager.delete_data_from_table(table_name, criteria) - - def _delete_object_from_management_table_by_id(self, pk_col, table_name, object_id) -> None: + + def delete_object_from_management_table_by_id(self, table_name, object_id) -> None: '''Delete an object from the database.''' - self.db_manager.delete_data_from_table(table_name, criteria= f'{pk_col} = \'{object_id}\'') + self.db_manager.delete_data_from_table(table_name, criteria= f'object_id = {object_id}') # rachel-8511, ShaniStrassProg is it needed? @@ -86,96 +400,87 @@ def _delete_object_from_management_table_by_id(self, pk_col, table_name, object_ # return self.db_manager.describe(self.table_name) - def _convert_object_name_to_management_table_name(self, object_name): + def convert_object_name_to_management_table_name(self,object_name): return f'mng_{object_name}s' - - # def is_management_table_exist(table_name): - # # check if table exists using single result query - # return self.db_manager.execute_query_with_single_result(f'desc table {table_name}') - - - # for outer use: - # def save_in_memory(self, object): - - def _is_management_table_exist(self, table_name): + def is_management_table_exist(self, table_name): # Check if table exists by querying the sqlite_master table query = f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}'" return self.db_manager.execute_query_with_single_result(query) # for outer use: - # def save_in_memory(self, table_name, object): - + # def save_in_memory(self, object_name:str, metadata:Dict[str:Any], object_id:Optional[str] = None): + # # # insert object info into management table mng_{object_name}s # # for exmple: object db_instance will be saved in table mng_db_instances - # table_name = self.convert_object_name_to_management_table_name(self.object_name) - + # table_name = self.convert_object_name_to_management_table_name(object_name) + # # if not self.is_management_table_exist(table_name): # self.create_management_table(table_name) - - # self.insert_object_to_management_table(table_name, object) + # + # if object_id: + # self.insert_object_to_management_table_with_str_id(table_name, object_id, metadata) + # + # else: + # self.insert_object_to_management_table(table_name, metadata) + def save_in_memory(self, object): - # insert object info into management table mng_{object_name}s - # for exmple: object db_instance will be saved in table mng_db_instances + # insert object info into management table mng_{object_name}s + # for exmple: object db_instance will be saved in table mng_db_instances + table_name = str(object.__class__.__name__) + if not self.is_management_table_exist(table_name): + self.create_management_table(table_name) + self.insert_object_to_management_table(table_name, object) - table_name = str(object.__class__.__name__) - if not self._is_management_table_exist(table_name): - self._create_management_table(table_name) - # self.insert_object_to_management_table(table_name, object) - - self._insert_object_to_management_table(table_name, object) - - - def delete_from_memory_by_id(self, pk_col, pk_val, table_name:str): - # pk_val is the object id - # if criteria not sent- use PK for deletion - criteria = f'{pk_col} = \'{pk_val}\'' - - table_name = self.convert_object_name_to_management_table_name(self.object_name) - - self.delete_data_from_table(table_name, criteria) + def is_management_table_exist(self, table_name): + query = f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}'" + result = self.db_manager.execute_query_with_single_result(query) + return result is not None + def delete_from_memory(self,object_name:str, criteria='default', object_id:Optional[str] = None): - def update_in_memory(self, updates, criteria='default'): - # if criteria not sent- use PK for deletion if criteria == 'default': - criteria = f'{self.pk_column} = {self.pk_value}' + if not object_id: + raise ValueError('must be or criteria or object id') + criteria = f'object_id = {object_id}' - table_name = self.convert_object_name_to_management_table_name(self.object_name) + table_name = self.convert_object_name_to_management_table_name(object_name) - self.update_object_in_management_table_by_criteria(table_name, updates, criteria) + self.delete_object_from_management_table(table_name, criteria) - def get_from_memory(self): - self.get_object_from_management_table(self.object_id) + def update_in_memory(self, object_name, updates, criteria='default', object_id:Optional[str] = None): - self.db_manager.delete_data_from_table(table_name, criteria) + # if criteria not sent- use PK for deletion + if criteria == 'default': + if not object_id: + raise ValueError('must be or criteria or object id') + criteria = f'object_id = {object_id}' - def update_in_memory_by_criteria(self,table_name:str, updates:Dict, criteria): - self._update_object_in_management_table_by_criteria(table_name, updates, criteria) - - def update_in_memory_by_id(self, pk_col, table_name, updates, object_id:Optional[str]): - if not object_id: - raise ValueError('must be or criteria or object id') - criteria = f'{pk_col} = \'{object_id}\'' - self.update_in_memory_by_criteria(table_name, updates, criteria) + table_name = self.convert_object_name_to_management_table_name(object_name) + self.update_object_in_management_table_by_criteria(table_name, updates, criteria) - - def get_from_memory_by_id(self, pk_col, table_name, object_id, columns = ["*"]): + + def get_from_memory(self, object_name, columns = ["*"], object_id = None, criteria = None): """get records from memory by criteria or id""" - criteria = f'{pk_col} = \'{object_id}\'' - return self._get_objects_from_management_table_by_criteria(table_name, columns, criteria) + table_name = self.convert_object_name_to_management_table_name(object_name) + if object_id: + criteria = f'object_id = {object_id}' + self.get_objects_from_management_table_by_criteria(table_name, columns, criteria) - def convert_object_attributes_to_dictionary(**kwargs): + def convert_object_attributes_to_dictionary(self, **kwargs): dict = {} for key, value in kwargs.items(): dict[key] = value - + return dict + + def get_all_data_from_table(self, table_name): + self.db_manager.get_all_data_from_table(table_name) \ No newline at end of file diff --git a/DB/NEW_KT_DB/Models/DBClusterModel.py b/DB/NEW_KT_DB/Models/DBClusterModel.py index 846d9989..8bc688bc 100644 --- a/DB/NEW_KT_DB/Models/DBClusterModel.py +++ b/DB/NEW_KT_DB/Models/DBClusterModel.py @@ -1,7 +1,12 @@ from datetime import datetime from typing import Dict -from DataAccess import ObjectManager +import sys import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from DataAccess import ObjectManager +import json class Cluster: def __init__(self, **kwargs): @@ -39,7 +44,8 @@ def __init__(self, **kwargs): def to_dict(self) -> Dict: '''Retrieve the data of the DB cluster as a dictionary.''' - return ObjectManager.convert_object_attributes_to_dictionary( + return ObjectManager.ObjectManager.convert_object_attributes_to_dictionary( + self, db_cluster_identifier=self.db_cluster_identifier, engine=self.engine, allocated_storage=self.allocated_storage, @@ -59,7 +65,7 @@ def to_dict(self) -> Dict: storage_encrypted=self.storage_encrypted, storage_type=self.storage_type, tags=self.tags, - created_at=self.created_at, + # created_at=self.created_at, status=self.status, primary_writer_instance=self.primary_writer_instance, reader_instances=self.reader_instances, @@ -102,4 +108,44 @@ def to_dict(self) -> Dict: # endpoints=self.endpoints, # pk_column=self.pk_column, # pk_value=self.pk_value - # ) \ No newline at end of file + # ) + def cluster_to_dict(cluster): + # Create a dictionary with the key as the cluster identifier + cluster_dict = { + cluster.db_cluster_identifier: { + "engine": cluster.engine, + "allocated_storage": cluster.allocated_storage, + "copy_tags_to_snapshot": cluster.copy_tags_to_snapshot, + "db_cluster_instance_class": cluster.db_cluster_instance_class, + "database_name": cluster.database_name, + "db_cluster_parameter_group_name": cluster.db_cluster_parameter_group_name, + "db_subnet_group_name": cluster.db_subnet_group_name, + "deletion_protection": cluster.deletion_protection, + "engine_version": cluster.engine_version, + "master_username": cluster.master_username, + "master_user_password": cluster.master_user_password, + "manage_master_user_password": cluster.manage_master_user_password, + "option_group_name": cluster.option_group_name, + "port": cluster.port, + "replication_source_identifier": cluster.replication_source_identifier, + "storage_encrypted": cluster.storage_encrypted, + "storage_type": cluster.storage_type, + "tags": cluster.tags, + # "created_at": str(cluster.created_at), # Uncomment if needed + "status": cluster.status, + "primary_writer_instance": cluster.primary_writer_instance, + "reader_instances": cluster.reader_instances, + "cluster_endpoint": cluster.cluster_endpoint, + "instances_endpoints": cluster.instances_endpoints, + "pk_column": cluster.pk_column, + "pk_value": cluster.pk_value + } + } + return cluster_dict + + def to_sql(self): + # Convert the model instance to a dictionary + data_dict = self.to_dict() + values = '(' + ", ".join(f'\'{json.dumps(v)}\'' if isinstance(v, dict) or isinstance(v, list) else f'\'{v}\'' if isinstance(v, str) else f'\'{str(v)}\'' + for v in data_dict.values()) + ')' + return values \ No newline at end of file diff --git a/DB/NEW_KT_DB/Service/Classes/DBClusterService.py b/DB/NEW_KT_DB/Service/Classes/DBClusterService.py index b10f5ffe..560063ed 100644 --- a/DB/NEW_KT_DB/Service/Classes/DBClusterService.py +++ b/DB/NEW_KT_DB/Service/Classes/DBClusterService.py @@ -1,13 +1,20 @@ from typing import Dict, Optional -from DataAccess import ClusterManager + +import sys +import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +from DataAccess import DBClusterManager from Models import DBClusterModel from Abc import DBO from Validation import DBClusterValiditions from DataAccess import DBClusterManager -from DBInstanceService import DBInstanceService +# from DBInstanceService import DBInstanceService import os import json -from DBClusterValiditions import ( +from Validation.DBClusterValiditions import ( validate_db_cluster_identifier, validate_engine, validate_database_name, @@ -18,14 +25,29 @@ validate_master_user_password, validate_master_username ) - -class DBClusterService(DBO): - def __init__(self, dal: DBClusterManager): - self.dal = dal - - - # validations here +from Storage.NEW_KT_Storage.DataAccess.StorageManager import StorageManager + +class DBClusterService: + def __init__(self, dal: DBClusterManager, storage_manager: StorageManager, directory:str): + self.dal = dal, + self.directory = directory, + self.storage_manager = storage_manager + if not self.storage_manager.is_directory_exist(directory): + self.storage_manager.create_directory(directory) + def get_file_path(self, cluster_name: str): + return os.path.join(self.directory, cluster_name + '.json') + + def is_cluster_exist(self, cluster_identifier: str): + cluster_path = self.get_file_path(cluster_identifier) + cluster_configurations_path = self.get_file_path(cluster_identifier+"_configurations") + if not self.storage_manager.is_file_exist(cluster_configurations_path) or not self.storage_manager.is_directory_exist(cluster_path): + return False + + if not self.dal.is_exists(cluster_identifier): + return False + + return True def create(self, **kwargs): @@ -33,12 +55,15 @@ def create(self, **kwargs): '''Create a new DBCluster.''' # Validate required parameters - required_params = ['db_cluster_identifier', 'engine', 'db_subnet_group_name'] + required_params = ['db_cluster_identifier', 'engine', 'db_subnet_group_name', 'allocated_storage'] if not check_required_params(required_params, **kwargs): raise ValueError("Missing required parameters") # Perform validations - if not validate_db_cluster_identifier(kwargs.get('db_cluster_identifier', '')): + if self.dal.is_exists(kwargs.get('db_cluster_identifier')): + raise ValueError(f"Cluster {kwargs.get('db_cluster_identifier')} already exists") + + if not validate_db_cluster_identifier(kwargs.get('db_cluster_identifier')): raise ValueError(f"Invalid DBClusterIdentifier: {kwargs.get('db_cluster_identifier')}") if not validate_engine(kwargs.get('engine', '')): @@ -66,68 +91,134 @@ def create(self, **kwargs): cluster = DBClusterModel.Cluster(**kwargs) # Create physical folder structure - desktop_path = os.path.join(os.path.expanduser('~'), 'Desktop') - cluster_directory = os.path.join(desktop_path, f'Clusters/{cluster.db_cluster_identifier}') - os.makedirs(cluster_directory, exist_ok=True) + # desktop_path = os.path.join(os.path.expanduser('~'), 'Desktop') + # cluster_directory = os.path.join(desktop_path, f'Clusters/{cluster.db_cluster_identifier}') + cluster_directory = self.get_file_path(cluster.db_cluster_identifier) + self.storage_manager.create_directory(cluster_directory) + # os.makedirs(cluster_directory, exist_ok=True) # Set cluster endpoint cluster.cluster_endpoint = cluster_directory # Create the primary writer instance primary_instance_name = f'{cluster.db_cluster_identifier}-primary' - primary_instance = self.DBInstanceService( - instance_name=primary_instance_name, - cluster_identifier=cluster.db_cluster_identifier, - allocated_storage=cluster.allocated_storage, - master_username=cluster.master_username, - master_user_password=cluster.master_user_password - ) + # primary_instance = self.DBInstanceService( + # instance_name=primary_instance_name, + # cluster_identifier=cluster.db_cluster_identifier, + # allocated_storage=cluster.allocated_storage, + # master_username=cluster.master_username, + # master_user_password=cluster.master_user_password + # ) + primary_instance = { + "DBInstance": { + "db_instance_identifier": "my-db-instance-1", + "endpoint": { + "address": "my-db-instance-1.123456789012.us-west-2.rds.amazonaws.com", + "port": 3306, + "hosted_zone_id": "Z1PVIF0EXAMPLE" + }, + } + } # Retrieve primary instance details primary_instance_json_string = primary_instance.get("DBInstance") - primary_instance_json_data = json.loads(primary_instance_json_string) - cluster.instances_endpoints["primary_instance"] = primary_instance_json_data.get("endpoint") - cluster.primary_writer_instance = primary_instance_json_data.get('db_instance_identifier') - - # Create configuration file - cluster_config_path = os.path.join(cluster_directory, 'cluster_config.json') - cluster_dict = cluster.to_dict() - try: - with open(cluster_config_path, 'w') as file: - json.dump(cluster_dict, file, indent=4) - except IOError as e: - raise RuntimeError(f"Failed to write configuration file: {e}") - + # primary_instance_json_data = json.loads(primary_instance_json_string) + cluster.instances_endpoints["primary_instance"] = primary_instance_json_string.get("endpoint") + cluster.primary_writer_instance = primary_instance_json_string.get('db_instance_identifier') + + # # Create configuration file + # cluster_config_path = os.path.join(cluster_directory, 'cluster_config.json') + # cluster_dict = cluster.to_dict() + # try: + # with open(cluster_config_path, 'w') as file: + # json.dump(cluster_dict, file, indent=4) + # except IOError as e: + # raise RuntimeError(f"Failed to write configuration file: {e}") + + json_object = json.dumps(cluster.to_dict()) + file_path = self.get_file_path(cluster.db_cluster_identifier+"_configurations") + self.storage_manager.create_file( + file_path=file_path, content=json_object) + + # cluster_to_sql = cluster.to_sql() + ccc = cluster.cluster_to_dict() + bbb = json.dumps(ccc) # Store the cluster information in the database - self.dal.createInMemoryDBCluster(cluster) + self.dal.createInMemoryDBCluster(bbb) return {"DBCluster": cluster_dict} - def delete(self): + def delete(self, cluster_identifier:str): '''Delete an existing DBCluster.''' - # assign None to code object - # delete physical object - # delete from memory using DBClusterManager.deleteInMemoryDBCluster() function- send criteria using self attributes - pass + + if not self.is_cluster_exist(cluster_identifier): + raise ValueError("Cluster does not exist!!") + + file_path = self.get_file_path(cluster_identifier+"_configurations") + self.storage_manager.delete_file(file_path=file_path) + directory_path = self.get_file_path(cluster_identifier) + self.storage_manager.delete_directory(directory_path) - def describe(self): + self.dal.deleteInMemoryDBCluster(cluster_identifier) + + + def describe(self, cluster_id): '''Describe the details of DBCluster.''' - # use DBClusterManager.describeDBCluster() function - pass + if not self.is_cluster_exist(cluster_id): + raise ValueError("Cluster does not exist!!") + + return self.dal.describeDBCluster(cluster_id) - def modify(self, **updates): + def modify(self, cluster_id: str, **kwargs): '''Modify an existing DBCluster.''' # update object in code # modify physical object # update object in memory using DBClusterManager.modifyInMemoryDBCluster() function- send criteria using self attributes - pass + if not self.is_cluster_exist(cluster_id): + raise ValueError("Cluster does not exist!!") + + if 'db_cluster_identifier' in kwargs and not validate_db_cluster_identifier(kwargs.get('db_cluster_identifier')): + raise ValueError(f"Invalid DBClusterIdentifier: {kwargs.get('db_cluster_identifier')}") + + if 'engine' in kwargs and not validate_engine(kwargs.get('engine', '')): + raise ValueError(f"Invalid engine: {kwargs.get('engine')}") + + if 'database_name' in kwargs and kwargs['database_name'] and not validate_database_name(kwargs['database_name']): + raise ValueError(f"Invalid DatabaseName: {kwargs['database_name']}") + + if 'db_cluster_parameter_group_name' in kwargs and kwargs['db_cluster_parameter_group_name'] and not validate_db_cluster_parameter_group_name(kwargs['db_cluster_parameter_group_name']): + raise ValueError(f"Invalid DBClusterParameterGroupName: {kwargs['db_cluster_parameter_group_name']}") + + if kwargs.get('db_subnet_group_name') and not validate_db_subnet_group_name(kwargs.get('db_subnet_group_name')): + raise ValueError(f"Invalid DBSubnetGroupName: {kwargs['db_subnet_group_name']}") + + if 'port' in kwargs and kwargs['port'] and not validate_port(kwargs['port']): + raise ValueError(f"Invalid port: {kwargs['port']}. Valid range is 1150-65535.") + + if 'master_username' in kwargs and not validate_master_username(kwargs['master_username']): + raise ValueError("Invalid master username") + + if 'master_user_password' in kwargs and not validate_master_user_password(kwargs['master_user_password'], kwargs.get('manage_master_user_password', False)): + raise ValueError("Invalid master user password") + + current_cluster = self.describe(cluster_id) + + # Update the cluster object + for key, value in kwargs.items(): + setattr(current_cluster, key, value) + #update in memory + self.dal.modifyDBCluster(cluster_id,current_cluster) + + #update configurations + file_path = self.get_file_path(cluster_id+'_configurations') + self.storage_manager.delete_file(file_path) + self.storage_manager.create_file(file_path, current_cluster) + # self.storage_manager.create_file(file_path, json.dumps(current_cluster)) + - def get(self): - '''get code object.''' - # return real time object - pass + diff --git a/DB/NEW_KT_DB/Validation/DBClusterValiditions.py b/DB/NEW_KT_DB/Validation/DBClusterValiditions.py index 3ffcbf6f..84bd1319 100644 --- a/DB/NEW_KT_DB/Validation/DBClusterValiditions.py +++ b/DB/NEW_KT_DB/Validation/DBClusterValiditions.py @@ -1,5 +1,10 @@ import re -import GeneralValidations +import sys +import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from Validation import GeneralValidations # def is_db_cluster_name_valid(cluster_name): # return is_length_in_range(cluster_name, 5, 20) From b4848b3df56ff68bd9163dc17280b7b1442c8c8b Mon Sep 17 00:00:00 2001 From: sara-lea Date: Tue, 17 Sep 2024 15:47:28 +0300 Subject: [PATCH 10/16] update and pull main --- .../Untitled-checkpoint.ipynb | 6 ++ ...lusters',)my-cluster-3_configurations.json | 1 + .../Clustersmy-cluster-3_configurations.json | 1 + .../Controller/DBClusterController.py | 6 +- DB/NEW_KT_DB/Controller/Untitled.ipynb | 79 +++++++++++++++++++ DB/NEW_KT_DB/DataAccess/DBClusterManager.py | 11 ++- DB/NEW_KT_DB/DataAccess/ObjectManager.py | 13 ++- .../Service/Classes/DBClusterService.py | 31 ++++---- 8 files changed, 120 insertions(+), 28 deletions(-) create mode 100644 DB/NEW_KT_DB/Controller/.ipynb_checkpoints/Untitled-checkpoint.ipynb create mode 100644 DB/NEW_KT_DB/Controller/Clusters/('Clusters',)my-cluster-3_configurations.json create mode 100644 DB/NEW_KT_DB/Controller/Clusters/Clustersmy-cluster-3_configurations.json create mode 100644 DB/NEW_KT_DB/Controller/Untitled.ipynb diff --git a/DB/NEW_KT_DB/Controller/.ipynb_checkpoints/Untitled-checkpoint.ipynb b/DB/NEW_KT_DB/Controller/.ipynb_checkpoints/Untitled-checkpoint.ipynb new file mode 100644 index 00000000..363fcab7 --- /dev/null +++ b/DB/NEW_KT_DB/Controller/.ipynb_checkpoints/Untitled-checkpoint.ipynb @@ -0,0 +1,6 @@ +{ + "cells": [], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/DB/NEW_KT_DB/Controller/Clusters/('Clusters',)my-cluster-3_configurations.json b/DB/NEW_KT_DB/Controller/Clusters/('Clusters',)my-cluster-3_configurations.json new file mode 100644 index 00000000..f16056df --- /dev/null +++ b/DB/NEW_KT_DB/Controller/Clusters/('Clusters',)my-cluster-3_configurations.json @@ -0,0 +1 @@ +{"db_cluster_identifier": "my-cluster-3", "engine": "mysql", "allocated_storage": 5, "copy_tags_to_snapshot": false, "db_cluster_instance_class": false, "database_name": null, "db_cluster_parameter_group_name": null, "db_subnet_group_name": "my-subnet-group", "deletion_protection": false, "engine_version": null, "master_username": null, "master_user_password": null, "manage_master_user_password": false, "option_group_name": null, "port": null, "replication_source_identifier": null, "storage_encrypted": null, "storage_type": "aurora", "tags": null, "status": "available", "primary_writer_instance": "my-db-instance-1", "reader_instances": [], "cluster_endpoint": "('Clusters',)my-cluster-3.json", "instances_endpoints": {"primary_instance": {"address": "my-db-instance-1.123456789012.us-west-2.rds.amazonaws.com", "port": 3306, "hosted_zone_id": "Z1PVIF0EXAMPLE"}}, "pk_column": "ClusterID", "pk_value": "my-cluster-3"} \ No newline at end of file diff --git a/DB/NEW_KT_DB/Controller/Clusters/Clustersmy-cluster-3_configurations.json b/DB/NEW_KT_DB/Controller/Clusters/Clustersmy-cluster-3_configurations.json new file mode 100644 index 00000000..43a5a6ba --- /dev/null +++ b/DB/NEW_KT_DB/Controller/Clusters/Clustersmy-cluster-3_configurations.json @@ -0,0 +1 @@ +{"db_cluster_identifier": "my-cluster-3", "engine": "mysql", "allocated_storage": 5, "copy_tags_to_snapshot": false, "db_cluster_instance_class": false, "database_name": null, "db_cluster_parameter_group_name": null, "db_subnet_group_name": "my-subnet-group", "deletion_protection": false, "engine_version": null, "master_username": null, "master_user_password": null, "manage_master_user_password": false, "option_group_name": null, "port": null, "replication_source_identifier": null, "storage_encrypted": null, "storage_type": "aurora", "tags": null, "status": "available", "primary_writer_instance": "my-db-instance-1", "reader_instances": [], "cluster_endpoint": "Clustersmy-cluster-3.json", "instances_endpoints": {"primary_instance": {"address": "my-db-instance-1.123456789012.us-west-2.rds.amazonaws.com", "port": 3306, "hosted_zone_id": "Z1PVIF0EXAMPLE"}}, "pk_column": "ClusterID", "pk_value": "my-cluster-3"} \ No newline at end of file diff --git a/DB/NEW_KT_DB/Controller/DBClusterController.py b/DB/NEW_KT_DB/Controller/DBClusterController.py index 198735fe..089e936f 100644 --- a/DB/NEW_KT_DB/Controller/DBClusterController.py +++ b/DB/NEW_KT_DB/Controller/DBClusterController.py @@ -29,9 +29,9 @@ def modify_db_cluster(self, updates): desktop_path = os.path.join(os.path.expanduser('~'), 'Desktop') cluster_directory = os.path.join(desktop_path, f'Clusters/clusters.db') base = os.path.join(desktop_path, f'Clusters') - storage_manager = StorageManager.StorageManager(base) - clusterManager = DBClusterManager.DBClusterManager(cluster_directory) - clusterService = DBClusterService(clusterManager,storage_manager, cluster_directory) + storage_manager = StorageManager.StorageManager('Clusters') + clusterManager = DBClusterManager.DBClusterManager('Clusters/clusters.db') + clusterService = DBClusterService(clusterManager,storage_manager, 'Clusters') clusterController = DBClusterController(clusterService) cluster_data = { 'db_cluster_identifier': 'my-cluster-3', diff --git a/DB/NEW_KT_DB/Controller/Untitled.ipynb b/DB/NEW_KT_DB/Controller/Untitled.ipynb new file mode 100644 index 00000000..3d9d557f --- /dev/null +++ b/DB/NEW_KT_DB/Controller/Untitled.ipynb @@ -0,0 +1,79 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 4, + "id": "d0900ce9", + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name '__file__' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[1;32mIn[4], line 3\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01msys\u001b[39;00m\n\u001b[0;32m 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mos\u001b[39;00m\n\u001b[1;32m----> 3\u001b[0m sys\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mappend(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mabspath(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mdirname(\u001b[38;5;18m__file__\u001b[39m), \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m../../../..\u001b[39m\u001b[38;5;124m'\u001b[39m)))\n\u001b[0;32m 4\u001b[0m sys\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mappend(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mabspath(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mdirname(\u001b[38;5;18m__file__\u001b[39m), \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m../..\u001b[39m\u001b[38;5;124m'\u001b[39m)))\n\u001b[0;32m 5\u001b[0m sys\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mappend(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mabspath(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mdirname(\u001b[38;5;18m__file__\u001b[39m), \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m..\u001b[39m\u001b[38;5;124m'\u001b[39m)))\n", + "\u001b[1;31mNameError\u001b[0m: name '__file__' is not defined" + ] + } + ], + "source": [ + "import sys\n", + "import os\n", + "sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..')))\n", + "sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))\n", + "sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))\n", + "from KT_Cloud.Storage.NEW_KT_Storage.DataAccess import StorageManager\n", + "from Service.Classes.DBClusterService import DBClusterService\n", + "from DataAccess import DBClusterManager\n", + "\n", + "desktop_path = os.path.join(os.path.expanduser('~'), 'Desktop')\n", + "cluster_directory = os.path.join(desktop_path, f'Clusters/clusters.db')\n", + "base = os.path.join(desktop_path, f'Clusters')\n", + "storage_manager = StorageManager.StorageManager('Clusters')\n", + "clusterManager = DBClusterManager.DBClusterManager('Clusters/clusters.db')\n", + "clusterService = DBClusterService(clusterManager,storage_manager, 'Clusters')\n", + "clusterController = DBClusterController(clusterService)\n", + "cluster_data = {\n", + "'db_cluster_identifier': 'my-cluster-3',\n", + "'engine': 'mysql',\n", + "'allocated_storage':5,\n", + "'db_subnet_group_name': 'my-subnet-group'\n", + "}\n", + "\n", + "aaa = clusterController.create_db_cluster(**cluster_data)\n", + "print(aaa)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5ea3eb25", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/DB/NEW_KT_DB/DataAccess/DBClusterManager.py b/DB/NEW_KT_DB/DataAccess/DBClusterManager.py index 9ee4ae83..05b36e09 100644 --- a/DB/NEW_KT_DB/DataAccess/DBClusterManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBClusterManager.py @@ -14,21 +14,20 @@ def __init__(self, db_file: str): def createInMemoryDBCluster(self, cluster_to_save): - self.object_manager.save_in_memory(cluster_to_save) + self.object_manager.save_in_memory(self.object_name, cluster_to_save) def deleteInMemoryDBCluster(self,cluster_identifier): - self.object_manager.delete_from_memory(cluster_identifier) - + self.object_manager.delete_from_memory_by_pk(self.object_name, self.pk_column, cluster_identifier) def describeDBCluster(self, cluster_id): - self.object_manager.get_from_memory(self.object_name, object_id = cluster_id) + self.object_manager.get_from_memory(self.object_name, criteria=f" {self.pk_column} = {cluster_id}") def modifyDBCluster(self, cluster_id, updates): - self.object_manager.update_in_memory(self.object_name, updates, object_id = cluster_id) + self.object_manager.update_in_memory(self.object_name, updates, criteria=f" {self.pk_column} = {cluster_id}") def select(self, name:Optional[str] = None, columns = ["*"]): - data = self.object_manager.get_from_memory(self.object_name, columns = columns, object_id = name) + data = self.object_manager.get_from_memory(self.object_name,criteria=f" {self.pk_column} = {name}") if data: data_to_return = [{col:data[col] for col in columns}] data_to_return[self.pk_column] = name diff --git a/DB/NEW_KT_DB/DataAccess/ObjectManager.py b/DB/NEW_KT_DB/DataAccess/ObjectManager.py index 8b91fc46..b57e51ae 100644 --- a/DB/NEW_KT_DB/DataAccess/ObjectManager.py +++ b/DB/NEW_KT_DB/DataAccess/ObjectManager.py @@ -321,12 +321,17 @@ from typing import Dict, Any, Optional import json import sqlite3 -from DBManager import DBManager +import sys +import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from DataAccess import DBManager class ObjectManager: def __init__(self, db_file: str): '''Initialize ObjectManager with the database connection.''' - self.db_manager = DBManager(db_file) + self.db_manager = DBManager.DBManager(db_file) def create_management_table(self, object_name, table_structure='default', pk_column_data_type='INTEGER'): @@ -364,7 +369,7 @@ def save_in_memory(self, object_name, object_info, columns=None): # for exmple: object db_instance will be saved in table mng_db_instances table_name = self._convert_object_name_to_management_table_name(object_name) - if not self._is_management_table_exist(table_name): + if not self._is_management_table_exist(object_name): self.create_management_table(object_name) if columns is None: @@ -426,4 +431,4 @@ def convert_object_attributes_to_dictionary(self, **kwargs): for key, value in kwargs.items(): dict[key] = value - return dict \ No newline at end of file + return dict diff --git a/DB/NEW_KT_DB/Service/Classes/DBClusterService.py b/DB/NEW_KT_DB/Service/Classes/DBClusterService.py index 560063ed..3230bbfe 100644 --- a/DB/NEW_KT_DB/Service/Classes/DBClusterService.py +++ b/DB/NEW_KT_DB/Service/Classes/DBClusterService.py @@ -29,14 +29,15 @@ class DBClusterService: def __init__(self, dal: DBClusterManager, storage_manager: StorageManager, directory:str): - self.dal = dal, - self.directory = directory, + self.dal = dal + self.directory = directory self.storage_manager = storage_manager if not self.storage_manager.is_directory_exist(directory): self.storage_manager.create_directory(directory) def get_file_path(self, cluster_name: str): - return os.path.join(self.directory, cluster_name + '.json') + # return os.path.join(self.directory, cluster_name + '.json') + return str(self.directory)+str(cluster_name)+'.json' def is_cluster_exist(self, cluster_identifier: str): cluster_path = self.get_file_path(cluster_identifier) @@ -60,8 +61,8 @@ def create(self, **kwargs): raise ValueError("Missing required parameters") # Perform validations - if self.dal.is_exists(kwargs.get('db_cluster_identifier')): - raise ValueError(f"Cluster {kwargs.get('db_cluster_identifier')} already exists") + # if self.dal.is_exists(kwargs.get('db_cluster_identifier')): + # raise ValueError(f"Cluster {kwargs.get('db_cluster_identifier')} already exists") if not validate_db_cluster_identifier(kwargs.get('db_cluster_identifier')): raise ValueError(f"Invalid DBClusterIdentifier: {kwargs.get('db_cluster_identifier')}") @@ -141,20 +142,20 @@ def create(self, **kwargs): file_path=file_path, content=json_object) # cluster_to_sql = cluster.to_sql() - ccc = cluster.cluster_to_dict() - bbb = json.dumps(ccc) + ccc = cluster.to_dict() + # bbb = json.dumps(ccc) # Store the cluster information in the database - self.dal.createInMemoryDBCluster(bbb) + self.dal.createInMemoryDBCluster(ccc) - return {"DBCluster": cluster_dict} + return {"DBCluster": ccc} def delete(self, cluster_identifier:str): '''Delete an existing DBCluster.''' - if not self.is_cluster_exist(cluster_identifier): - raise ValueError("Cluster does not exist!!") + # if not self.is_cluster_exist(cluster_identifier): + # raise ValueError("Cluster does not exist!!") file_path = self.get_file_path(cluster_identifier+"_configurations") self.storage_manager.delete_file(file_path=file_path) @@ -167,8 +168,8 @@ def delete(self, cluster_identifier:str): def describe(self, cluster_id): '''Describe the details of DBCluster.''' - if not self.is_cluster_exist(cluster_id): - raise ValueError("Cluster does not exist!!") + # if not self.is_cluster_exist(cluster_id): + # raise ValueError("Cluster does not exist!!") return self.dal.describeDBCluster(cluster_id) @@ -178,8 +179,8 @@ def modify(self, cluster_id: str, **kwargs): # update object in code # modify physical object # update object in memory using DBClusterManager.modifyInMemoryDBCluster() function- send criteria using self attributes - if not self.is_cluster_exist(cluster_id): - raise ValueError("Cluster does not exist!!") + # if not self.is_cluster_exist(cluster_id): + # raise ValueError("Cluster does not exist!!") if 'db_cluster_identifier' in kwargs and not validate_db_cluster_identifier(kwargs.get('db_cluster_identifier')): raise ValueError(f"Invalid DBClusterIdentifier: {kwargs.get('db_cluster_identifier')}") From c26034e1005a8cd97dc604928ba1b71acefaf547 Mon Sep 17 00:00:00 2001 From: sara-lea Date: Wed, 18 Sep 2024 14:00:56 +0300 Subject: [PATCH 11/16] pull changes from main --- .../Untitled-checkpoint.ipynb | 6 -- ...lusters',)my-cluster-3_configurations.json | 1 - .../myCluster6/myCluster6_configurations.json | 1 + .../Clustersmy-cluster-3_configurations.json | 1 - .../ClustersmyCluster1_configurations.json | 29 +++++++ .../ClustersmyCluster2_configurations.json | 1 + .../ClustersmyCluster3_configurations.json | 1 + .../Controller/DBClusterController.py | 50 +++++++++--- .../Controller/DBInstanceController.py | 21 +++++ DB/NEW_KT_DB/Controller/Untitled.ipynb | 79 ------------------- 10 files changed, 90 insertions(+), 100 deletions(-) delete mode 100644 DB/NEW_KT_DB/Controller/.ipynb_checkpoints/Untitled-checkpoint.ipynb delete mode 100644 DB/NEW_KT_DB/Controller/Clusters/('Clusters',)my-cluster-3_configurations.json create mode 100644 DB/NEW_KT_DB/Controller/Clusters/Clusters/myCluster6/myCluster6_configurations.json delete mode 100644 DB/NEW_KT_DB/Controller/Clusters/Clustersmy-cluster-3_configurations.json create mode 100644 DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster1_configurations.json create mode 100644 DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster2_configurations.json create mode 100644 DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster3_configurations.json create mode 100644 DB/NEW_KT_DB/Controller/DBInstanceController.py delete mode 100644 DB/NEW_KT_DB/Controller/Untitled.ipynb diff --git a/DB/NEW_KT_DB/Controller/.ipynb_checkpoints/Untitled-checkpoint.ipynb b/DB/NEW_KT_DB/Controller/.ipynb_checkpoints/Untitled-checkpoint.ipynb deleted file mode 100644 index 363fcab7..00000000 --- a/DB/NEW_KT_DB/Controller/.ipynb_checkpoints/Untitled-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/DB/NEW_KT_DB/Controller/Clusters/('Clusters',)my-cluster-3_configurations.json b/DB/NEW_KT_DB/Controller/Clusters/('Clusters',)my-cluster-3_configurations.json deleted file mode 100644 index f16056df..00000000 --- a/DB/NEW_KT_DB/Controller/Clusters/('Clusters',)my-cluster-3_configurations.json +++ /dev/null @@ -1 +0,0 @@ -{"db_cluster_identifier": "my-cluster-3", "engine": "mysql", "allocated_storage": 5, "copy_tags_to_snapshot": false, "db_cluster_instance_class": false, "database_name": null, "db_cluster_parameter_group_name": null, "db_subnet_group_name": "my-subnet-group", "deletion_protection": false, "engine_version": null, "master_username": null, "master_user_password": null, "manage_master_user_password": false, "option_group_name": null, "port": null, "replication_source_identifier": null, "storage_encrypted": null, "storage_type": "aurora", "tags": null, "status": "available", "primary_writer_instance": "my-db-instance-1", "reader_instances": [], "cluster_endpoint": "('Clusters',)my-cluster-3.json", "instances_endpoints": {"primary_instance": {"address": "my-db-instance-1.123456789012.us-west-2.rds.amazonaws.com", "port": 3306, "hosted_zone_id": "Z1PVIF0EXAMPLE"}}, "pk_column": "ClusterID", "pk_value": "my-cluster-3"} \ No newline at end of file diff --git a/DB/NEW_KT_DB/Controller/Clusters/Clusters/myCluster6/myCluster6_configurations.json b/DB/NEW_KT_DB/Controller/Clusters/Clusters/myCluster6/myCluster6_configurations.json new file mode 100644 index 00000000..06b8a5a2 --- /dev/null +++ b/DB/NEW_KT_DB/Controller/Clusters/Clusters/myCluster6/myCluster6_configurations.json @@ -0,0 +1 @@ +{"db_cluster_identifier": "myCluster6", "engine": "mysql", "allocated_storage": 5, "copy_tags_to_snapshot": false, "db_cluster_instance_class": false, "database_name": null, "db_cluster_parameter_group_name": null, "db_subnet_group_name": "my-subnet-group", "deletion_protection": false, "engine_version": null, "master_username": null, "master_user_password": null, "manage_master_user_password": false, "option_group_name": null, "port": null, "replication_source_identifier": null, "storage_encrypted": null, "storage_type": "aurora", "tags": null, "created_at": "2024-09-18T11:03:52.483405", "status": "available", "primary_writer_instance": "myCluster6-primary", "reader_instances": [], "cluster_endpoint": "Clusters\\myCluster6", "instances_endpoints": {"primary_instance": "myCluster6-primary"}, "pk_column": "db_cluster_identifier", "pk_value": "myCluster6"} \ No newline at end of file diff --git a/DB/NEW_KT_DB/Controller/Clusters/Clustersmy-cluster-3_configurations.json b/DB/NEW_KT_DB/Controller/Clusters/Clustersmy-cluster-3_configurations.json deleted file mode 100644 index 43a5a6ba..00000000 --- a/DB/NEW_KT_DB/Controller/Clusters/Clustersmy-cluster-3_configurations.json +++ /dev/null @@ -1 +0,0 @@ -{"db_cluster_identifier": "my-cluster-3", "engine": "mysql", "allocated_storage": 5, "copy_tags_to_snapshot": false, "db_cluster_instance_class": false, "database_name": null, "db_cluster_parameter_group_name": null, "db_subnet_group_name": "my-subnet-group", "deletion_protection": false, "engine_version": null, "master_username": null, "master_user_password": null, "manage_master_user_password": false, "option_group_name": null, "port": null, "replication_source_identifier": null, "storage_encrypted": null, "storage_type": "aurora", "tags": null, "status": "available", "primary_writer_instance": "my-db-instance-1", "reader_instances": [], "cluster_endpoint": "Clustersmy-cluster-3.json", "instances_endpoints": {"primary_instance": {"address": "my-db-instance-1.123456789012.us-west-2.rds.amazonaws.com", "port": 3306, "hosted_zone_id": "Z1PVIF0EXAMPLE"}}, "pk_column": "ClusterID", "pk_value": "my-cluster-3"} \ No newline at end of file diff --git a/DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster1_configurations.json b/DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster1_configurations.json new file mode 100644 index 00000000..067f158f --- /dev/null +++ b/DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster1_configurations.json @@ -0,0 +1,29 @@ +{ + "db_cluster_identifier": "myCluster1", + "engine": "postgres", + "allocated_storage": 3, + "copy_tags_to_snapshot": "False", + "db_cluster_instance_class": "False", + "database_name": "None", + "db_cluster_parameter_group_name": "None", + "db_subnet_group_name": "my-subnet-group", + "deletion_protection": "False", + "engine_version": "None", + "master_username": "None", + "master_user_password": "None", + "manage_master_user_password": "False", + "option_group_name": "None", + "port": "None", + "replication_source_identifier": "None", + "storage_encrypted": "None", + "storage_type": "aurora", + "tags": "None", + "created_at": "2024-09-17T21:08:23.715538", + "status": "available", + "primary_writer_instance": "my-db-instance-1", + "reader_instances": "[]", + "cluster_endpoint": "ClustersmyCluster1", + "instances_endpoints": "{\"primary_instance\": {\"address\": \"my-db-instance-1.123456789012.us-west-2.rds.amazonaws.com\", \"port\": 3306, \"hosted_zone_id\": \"Z1PVIF0EXAMPLE\"}}", + "pk_column": "db_cluster_identifier", + "pk_value": "myCluster1" +} \ No newline at end of file diff --git a/DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster2_configurations.json b/DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster2_configurations.json new file mode 100644 index 00000000..2b5e47d5 --- /dev/null +++ b/DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster2_configurations.json @@ -0,0 +1 @@ +{"db_cluster_identifier": "myCluster2", "engine": "mysql", "allocated_storage": 5, "copy_tags_to_snapshot": false, "db_cluster_instance_class": false, "database_name": null, "db_cluster_parameter_group_name": null, "db_subnet_group_name": "my-subnet-group", "deletion_protection": false, "engine_version": null, "master_username": null, "master_user_password": null, "manage_master_user_password": false, "option_group_name": null, "port": null, "replication_source_identifier": null, "storage_encrypted": null, "storage_type": "aurora", "tags": null, "created_at": "2024-09-17T21:09:36.710599", "status": "available", "primary_writer_instance": "my-db-instance-1", "reader_instances": [], "cluster_endpoint": "ClustersmyCluster2", "instances_endpoints": {"primary_instance": {"address": "my-db-instance-1.123456789012.us-west-2.rds.amazonaws.com", "port": 3306, "hosted_zone_id": "Z1PVIF0EXAMPLE"}}, "pk_column": "db_cluster_identifier", "pk_value": "myCluster2"} \ No newline at end of file diff --git a/DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster3_configurations.json b/DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster3_configurations.json new file mode 100644 index 00000000..ba8c9b5e --- /dev/null +++ b/DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster3_configurations.json @@ -0,0 +1 @@ +{"db_cluster_identifier": "myCluster3", "engine": "mysql", "allocated_storage": 5, "copy_tags_to_snapshot": false, "db_cluster_instance_class": false, "database_name": null, "db_cluster_parameter_group_name": null, "db_subnet_group_name": "my-subnet-group", "deletion_protection": false, "engine_version": null, "master_username": null, "master_user_password": null, "manage_master_user_password": false, "option_group_name": null, "port": null, "replication_source_identifier": null, "storage_encrypted": null, "storage_type": "aurora", "tags": null, "created_at": "2024-09-17T21:11:52.012571", "status": "available", "primary_writer_instance": "my-db-instance-1", "reader_instances": [], "cluster_endpoint": "ClustersmyCluster3", "instances_endpoints": {"primary_instance": {"address": "my-db-instance-1.123456789012.us-west-2.rds.amazonaws.com", "port": 3306, "hosted_zone_id": "Z1PVIF0EXAMPLE"}}, "pk_column": "db_cluster_identifier", "pk_value": "myCluster3"} \ No newline at end of file diff --git a/DB/NEW_KT_DB/Controller/DBClusterController.py b/DB/NEW_KT_DB/Controller/DBClusterController.py index 089e936f..d1c44810 100644 --- a/DB/NEW_KT_DB/Controller/DBClusterController.py +++ b/DB/NEW_KT_DB/Controller/DBClusterController.py @@ -7,38 +7,62 @@ from KT_Cloud.Storage.NEW_KT_Storage.DataAccess import StorageManager from Service.Classes.DBClusterService import DBClusterService from DataAccess import DBClusterManager +from Controller import DBInstanceController +from Service.Classes.DBInstanceService import DBInstanceService +from DataAccess import DBInstanceManager +from DataAccess import ObjectManager class DBClusterController: - def __init__(self, service: DBClusterService): + def __init__(self, service: DBClusterService, instance_controller:DBInstanceController): self.service = service + self.instance_controller = instance_controller def create_db_cluster(self, **kwargs): - self.service.create(**kwargs) + self.service.create(self.instance_controller,**kwargs) - def delete_db_cluster(self): - self.service.delete() + def delete_db_cluster(self , cluster_identifier): + self.service.delete(cluster_identifier) - def modify_db_cluster(self, updates): - self.service.modify(updates) + def modify_db_cluster(self, cluster_identifier, **updates): + self.service.modify(cluster_identifier,**updates) + def describe_db_cluster(self, cluster_id): + return self.service.describe(cluster_id) if __name__=='__main__': - desktop_path = os.path.join(os.path.expanduser('~'), 'Desktop') - cluster_directory = os.path.join(desktop_path, f'Clusters/clusters.db') - base = os.path.join(desktop_path, f'Clusters') + # desktop_path = os.path.join(os.path.expanduser('~'), 'Desktop') + # cluster_directory = os.path.join(desktop_path, f'Clusters/clusters.db') + # base = os.path.join(desktop_path, f'Clusters') + storage_manager = StorageManager.StorageManager('Instances') + db_file = ObjectManager.ObjectManager('Clusters/instances.db') + instance_manager = DBInstanceManager.DBInstanceManager(db_file) + instanceService = DBInstanceService(instance_manager) + instanceController = DBInstanceController.DBInstanceController(instanceService) + storage_manager = StorageManager.StorageManager('Clusters') clusterManager = DBClusterManager.DBClusterManager('Clusters/clusters.db') clusterService = DBClusterService(clusterManager,storage_manager, 'Clusters') - clusterController = DBClusterController(clusterService) + clusterController = DBClusterController(clusterService,instanceController) cluster_data = { - 'db_cluster_identifier': 'my-cluster-3', + 'db_cluster_identifier': 'myCluster6', 'engine': 'mysql', 'allocated_storage':5, 'db_subnet_group_name': 'my-subnet-group' } - aaa = clusterController.create_db_cluster(**cluster_data) - print(aaa) \ No newline at end of file + clusterController.create_db_cluster(**cluster_data) + # clusterController.delete_db_cluster('myCluster5') + update_data = { + 'engine': 'postgres', + 'allocated_storage':3, + } + # clusterController.modify_db_cluster('myCluster1', **update_data) + + # dfgh = clusterController.describe_db_cluster('myCluster3') + # print(dfgh) + + + diff --git a/DB/NEW_KT_DB/Controller/DBInstanceController.py b/DB/NEW_KT_DB/Controller/DBInstanceController.py new file mode 100644 index 00000000..ffa916cd --- /dev/null +++ b/DB/NEW_KT_DB/Controller/DBInstanceController.py @@ -0,0 +1,21 @@ +import datetime +from typing import Optional, Dict +from Service.Classes.DBInstanceService import DBInstanceService + + +class DBInstanceController: + def __init__(self, service: DBInstanceService): + self.service = service + + def create_db_instance(self,**kwargs): + return self.service.create(**kwargs) + + def delete_db_instance(self, db_instance_identifier: str): + self.service.delete(db_instance_identifier) + + def modify_db_instance(self, **kwargs): + return self.service.modify(**kwargs) + + def describe_db_instance(self, db_instance_identifier: str): + return self.service.describe(db_instance_identifier) + diff --git a/DB/NEW_KT_DB/Controller/Untitled.ipynb b/DB/NEW_KT_DB/Controller/Untitled.ipynb deleted file mode 100644 index 3d9d557f..00000000 --- a/DB/NEW_KT_DB/Controller/Untitled.ipynb +++ /dev/null @@ -1,79 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 4, - "id": "d0900ce9", - "metadata": {}, - "outputs": [ - { - "ename": "NameError", - "evalue": "name '__file__' is not defined", - "output_type": "error", - "traceback": [ - "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[1;31mNameError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[1;32mIn[4], line 3\u001b[0m\n\u001b[0;32m 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01msys\u001b[39;00m\n\u001b[0;32m 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mos\u001b[39;00m\n\u001b[1;32m----> 3\u001b[0m sys\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mappend(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mabspath(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mdirname(\u001b[38;5;18m__file__\u001b[39m), \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m../../../..\u001b[39m\u001b[38;5;124m'\u001b[39m)))\n\u001b[0;32m 4\u001b[0m sys\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mappend(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mabspath(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mdirname(\u001b[38;5;18m__file__\u001b[39m), \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m../..\u001b[39m\u001b[38;5;124m'\u001b[39m)))\n\u001b[0;32m 5\u001b[0m sys\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mappend(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mabspath(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mdirname(\u001b[38;5;18m__file__\u001b[39m), \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m..\u001b[39m\u001b[38;5;124m'\u001b[39m)))\n", - "\u001b[1;31mNameError\u001b[0m: name '__file__' is not defined" - ] - } - ], - "source": [ - "import sys\n", - "import os\n", - "sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..')))\n", - "sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..')))\n", - "sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))\n", - "from KT_Cloud.Storage.NEW_KT_Storage.DataAccess import StorageManager\n", - "from Service.Classes.DBClusterService import DBClusterService\n", - "from DataAccess import DBClusterManager\n", - "\n", - "desktop_path = os.path.join(os.path.expanduser('~'), 'Desktop')\n", - "cluster_directory = os.path.join(desktop_path, f'Clusters/clusters.db')\n", - "base = os.path.join(desktop_path, f'Clusters')\n", - "storage_manager = StorageManager.StorageManager('Clusters')\n", - "clusterManager = DBClusterManager.DBClusterManager('Clusters/clusters.db')\n", - "clusterService = DBClusterService(clusterManager,storage_manager, 'Clusters')\n", - "clusterController = DBClusterController(clusterService)\n", - "cluster_data = {\n", - "'db_cluster_identifier': 'my-cluster-3',\n", - "'engine': 'mysql',\n", - "'allocated_storage':5,\n", - "'db_subnet_group_name': 'my-subnet-group'\n", - "}\n", - "\n", - "aaa = clusterController.create_db_cluster(**cluster_data)\n", - "print(aaa)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5ea3eb25", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From d416a7328f7570cbe09857e73ce4c3df83afe134 Mon Sep 17 00:00:00 2001 From: sara-lea Date: Wed, 18 Sep 2024 14:01:18 +0300 Subject: [PATCH 12/16] pull changes from main --- DB/NEW_KT_DB/DataAccess/DBClusterManager.py | 81 ++++++++++---- DB/NEW_KT_DB/DataAccess/DBInstanceManager.py | 33 ++++++ DB/NEW_KT_DB/Exception/exception.py | 59 ++++++++++ DB/NEW_KT_DB/Models/DBClusterModel.py | 103 ++++++------------ DB/NEW_KT_DB/Models/DBInstanceModel.py | 59 ++++++++++ .../Service/Classes/DBClusterService.py | 90 ++++++++------- .../Service/Classes/DBInstanceService.py | 101 +++++++++++++++++ DB/NEW_KT_DB/Test/DBClusterTests.py | 47 ++++++-- .../Validation/DBInstanceValidition.py | 22 ++++ 9 files changed, 457 insertions(+), 138 deletions(-) create mode 100644 DB/NEW_KT_DB/DataAccess/DBInstanceManager.py create mode 100644 DB/NEW_KT_DB/Exception/exception.py create mode 100644 DB/NEW_KT_DB/Models/DBInstanceModel.py create mode 100644 DB/NEW_KT_DB/Service/Classes/DBInstanceService.py create mode 100644 DB/NEW_KT_DB/Validation/DBInstanceValidition.py diff --git a/DB/NEW_KT_DB/DataAccess/DBClusterManager.py b/DB/NEW_KT_DB/DataAccess/DBClusterManager.py index 05b36e09..8a00099f 100644 --- a/DB/NEW_KT_DB/DataAccess/DBClusterManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBClusterManager.py @@ -2,16 +2,44 @@ import json import sqlite3 from DataAccess import ObjectManager +from Models.DBClusterModel import Cluster from typing import Optional class DBClusterManager: def __init__(self, db_file: str): '''Initialize ObjectManager with the database connection.''' self.object_manager = ObjectManager.ObjectManager(db_file) - self.object_name ='clusters' - self.pk_column = 'ClusterID' - # self.create_table() - + self.object_name ='cluster' + self.pk_column = 'db_cluster_identifier' + self.table_schema = """ db_cluster_identifier TEXT PRIMARY KEY, + engine TEXT, + allocated_storage INTEGER, + copy_tags_to_snapshot BOOLEAN, + db_cluster_instance_class TEXT, + database_name TEXT, + db_cluster_parameter_group_name TEXT, + db_subnet_group_name TEXT, + deletion_protection BOOLEAN, + engine_version TEXT, + master_username TEXT, + master_user_password TEXT, + manage_master_user_password BOOLEAN, + option_group_name TEXT, + port INTEGER, + replication_source_identifier TEXT, + storage_encrypted BOOLEAN, + storage_type TEXT, + tags TEXT, + created_at TEXT, + status TEXT, + primary_writer_instance TEXT, + reader_instances TEXT, + cluster_endpoint TEXT, + instances_endpoints TEXT, + pk_column TEXT, + pk_value TEXT + """ + self.object_manager.create_management_table(self.object_name, table_structure = self.table_schema) def createInMemoryDBCluster(self, cluster_to_save): self.object_manager.save_in_memory(self.object_name, cluster_to_save) @@ -21,25 +49,36 @@ def deleteInMemoryDBCluster(self,cluster_identifier): self.object_manager.delete_from_memory_by_pk(self.object_name, self.pk_column, cluster_identifier) def describeDBCluster(self, cluster_id): - self.object_manager.get_from_memory(self.object_name, criteria=f" {self.pk_column} = {cluster_id}") + return self.object_manager.get_from_memory(self.object_name, criteria=f" {self.pk_column} = '{cluster_id}'") def modifyDBCluster(self, cluster_id, updates): - self.object_manager.update_in_memory(self.object_name, updates, criteria=f" {self.pk_column} = {cluster_id}") + self.object_manager.update_in_memory(self.object_name, updates, criteria=f" {self.pk_column} = '{cluster_id}'") - def select(self, name:Optional[str] = None, columns = ["*"]): - data = self.object_manager.get_from_memory(self.object_name,criteria=f" {self.pk_column} = {name}") - if data: - data_to_return = [{col:data[col] for col in columns}] - data_to_return[self.pk_column] = name - return data_to_return + # def select(self, name:Optional[str] = None, columns = ["*"]): + # data = self.object_manager.get_from_memory(self.object_name,criteria=f" {self.pk_column} = {name}") + # if data: + # data_to_return = [{col:data[col] for col in columns}] + # data_to_return[self.pk_column] = name + # return data_to_return + # else: + # raise ValueError(f"db cluster with name '{name}' not found") + + # def is_exists(self, name): + # """check if object exists in table""" + # try: + # self.select(name) + # return True + # except: + # return False + + def get(self, cluster_id: str): + data = self.object_manager.get_from_memory(self.object_name, criteria=f" {self.pk_column} = '{cluster_id}'") + # get_from_memory(self, object_name, columns=None, criteria=None) + if data: + data_mapping = {'db_cluster_identifier':cluster_id} + for key, value in data[cluster_id].items(): + data_mapping[key] = value + return Cluster(**data_mapping) else: - raise ValueError(f"db cluster with name '{name}' not found") - - def is_exists(self, name): - """check if object exists in table""" - try: - self.select(name) - return True - except: - return False \ No newline at end of file + raise ValueError(f"subnet group with name '{cluster_id}' not found") \ No newline at end of file diff --git a/DB/NEW_KT_DB/DataAccess/DBInstanceManager.py b/DB/NEW_KT_DB/DataAccess/DBInstanceManager.py new file mode 100644 index 00000000..3e6e1b8b --- /dev/null +++ b/DB/NEW_KT_DB/DataAccess/DBInstanceManager.py @@ -0,0 +1,33 @@ +import os +import sys +from typing import Dict, Any, List +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +from DataAccess.ObjectManager import ObjectManager +from Models.DBInstanceModel import DBInstance + +class DBInstanceManager: + def __init__(self, object_manager:ObjectManager): + self.object_manager = object_manager + self.object_manager.create_management_table( DBInstance.table_name, DBInstance.table_structure) + + def createInMemoryDBInstance(self,db_instance:DBInstance): + columns=self.object_manager.db_manager.get_column_names_of_table(self.object_manager._convert_object_name_to_management_table_name(DBInstance.table_name)) + columns_str = ', '.join(columns) + self.object_manager.save_in_memory(DBInstance.table_name,db_instance.to_sql(),columns=columns_str) + + + def deleteInMemoryDBInstance(self,db_instance_identifier:str): + self.object_manager.delete_from_memory_by_pk(pk_column=DBInstance.pk_column ,pk_value= db_instance_identifier,object_name=DBInstance.table_name) + + + def describeDBInstance(self,db_instance_identifier:str): + return self.object_manager.get_from_memory(criteria=f"{DBInstance.pk_column} = '{db_instance_identifier}'",object_name=DBInstance.table_name,columns='*')#[db_instance_identifier] + + + def modifyDBInstance(self,db_instance_identifier,updates): + self.object_manager.update_in_memory(criteria=f"{DBInstance.pk_column} = '{db_instance_identifier}'",object_name=DBInstance.table_name,updates=updates) + + def is_db_instance_identifier_exist(self, db_instance_identifier: int) -> bool: + '''Check if an db_instance with the given ID exists in the database.''' + return self.object_manager.db_manager.is_object_exist(self.object_manager._convert_object_name_to_management_table_name(DBInstance.table_name),criteria=f"{DBInstance.pk_column} = '{db_instance_identifier}'") diff --git a/DB/NEW_KT_DB/Exception/exception.py b/DB/NEW_KT_DB/Exception/exception.py new file mode 100644 index 00000000..baa9404c --- /dev/null +++ b/DB/NEW_KT_DB/Exception/exception.py @@ -0,0 +1,59 @@ +class AlreadyExistsError(Exception): + """Raised when an object already exists.""" + pass + + +class DatabaseCreationError(Exception): + """Raised when there is an error creating the database.""" + pass + + +class ConnectionError(Exception): + """Raised when a connection to the database fails.""" + pass + + +class DatabaseNotFoundError(Exception): + """Raised when a database is not found.""" + pass + + +class DBInstanceNotFoundError(Exception): + """Raised when a database instance is not found.""" + pass + + +class MissingRequireParamError(Exception): + """Raised when a required parameter in a function is missing.""" + pass + + +class InvalidDBInstanceStateError(Exception): + """Raised when trying to perform an operation on DBInstaace when it is not in the appropriate status for this operation""" + pass + + +class ParamValidationError(Exception): + pass + + +class StartNonStoppedDBInstance(Exception): + """Raised when trying to start non stopped db instance""" + pass + + +class DatabaseCloneError(Exception): + """Custom exception for database cloning errors.""" + pass + +class DbSnapshotIdentifierNotFoundError(Exception): + """Raised when a db snapshot identifier is not found.""" + pass + +class InvalidQueryError(Exception): + """Raised when a query is not properly constructed or contains syntax errors.""" + pass + +class InvalidQueryError(Exception): + """Raised when a query is not properly constructed or contains syntax errors.""" + pass diff --git a/DB/NEW_KT_DB/Models/DBClusterModel.py b/DB/NEW_KT_DB/Models/DBClusterModel.py index 8bc688bc..6f78d8e1 100644 --- a/DB/NEW_KT_DB/Models/DBClusterModel.py +++ b/DB/NEW_KT_DB/Models/DBClusterModel.py @@ -30,16 +30,43 @@ def __init__(self, **kwargs): self.storage_encrypted = kwargs.get('storage_encrypted', None) self.storage_type = kwargs.get('storage_type', 'aurora') self.tags = kwargs.get('tags', None) - self.created_at = datetime.now() + self.created_at = kwargs.get('created_at', datetime.now()) self.status = 'available' self.primary_writer_instance = None self.reader_instances = [] self.cluster_endpoint = None self.instances_endpoints = {} # Added attribute to store endpoints - self.pk_column = kwargs.get('pk_column', 'ClusterID') + self.pk_column = kwargs.get('pk_column', 'db_cluster_identifier') self.pk_value = kwargs.get('pk_value', self.db_cluster_identifier) - + self.table_schema = """ db_cluster_identifier TEXT PRIMARY KEY, + engine TEXT, + allocated_storage INTEGER, + copy_tags_to_snapshot BOOLEAN, + db_cluster_instance_class TEXT, + database_name TEXT, + db_cluster_parameter_group_name TEXT, + db_subnet_group_name TEXT, + deletion_protection BOOLEAN, + engine_version TEXT, + master_username TEXT, + master_user_password TEXT, + manage_master_user_password BOOLEAN, + option_group_name TEXT, + port INTEGER, + replication_source_identifier TEXT, + storage_encrypted BOOLEAN, + storage_type TEXT, + tags TEXT, + created_at TEXT, + status TEXT, + primary_writer_instance TEXT, + reader_instances TEXT, + cluster_endpoint TEXT, + instances_endpoints TEXT, + pk_column TEXT, + pk_value TEXT + """ def to_dict(self) -> Dict: '''Retrieve the data of the DB cluster as a dictionary.''' @@ -65,7 +92,7 @@ def to_dict(self) -> Dict: storage_encrypted=self.storage_encrypted, storage_type=self.storage_type, tags=self.tags, - # created_at=self.created_at, + created_at=self.created_at.isoformat(), status=self.status, primary_writer_instance=self.primary_writer_instance, reader_instances=self.reader_instances, @@ -75,74 +102,6 @@ def to_dict(self) -> Dict: pk_value=self.pk_value ) - # def to_dict(self) -> Dict: - # '''Retrieve the data of the DB cluster as a dictionary.''' - - # return ObjectManager.convert_object_attributes_to_dictionary( - # db_cluster_identifier=self.db_cluster_identifier, - # engine=self.engine, - # availability_zones=self.availability_zones, - # copy_tags_to_snapshot=self.copy_tags_to_snapshot, - # database_name=self.database_name, - # db_cluster_parameter_group_name=self.db_cluster_parameter_group_name, - # db_subnet_group_name=self.db_subnet_group_name, - # deletion_protection=self.deletion_protection, - # enable_cloudwatch_logs_exports=self.enable_cloudwatch_logs_exports, - # enable_global_write_forwarding=self.enable_global_write_forwarding, - # enable_http_endpoint=self.enable_http_endpoint, - # enable_limitless_database=self.enable_limitless_database, - # enable_local_write_forwarding=self.enable_local_write_forwarding, - # engine_version=self.engine_version, - # global_cluster_identifier=self.global_cluster_identifier, - # option_group_name=self.option_group_name, - # port=self.port, - # replication_source_identifier=self.replication_source_identifier, - # scaling_configuration=self.scaling_configuration, - # storage_encrypted=self.storage_encrypted, - # storage_type=self.storage_type, - # tags=self.tags, - # created_at=self.created_at, - # status=self.status, - # primary_writer_instance = self.primary_writer_instance, - # reader_instances = self.reader_instances, - # endpoints=self.endpoints, - # pk_column=self.pk_column, - # pk_value=self.pk_value - # ) - def cluster_to_dict(cluster): - # Create a dictionary with the key as the cluster identifier - cluster_dict = { - cluster.db_cluster_identifier: { - "engine": cluster.engine, - "allocated_storage": cluster.allocated_storage, - "copy_tags_to_snapshot": cluster.copy_tags_to_snapshot, - "db_cluster_instance_class": cluster.db_cluster_instance_class, - "database_name": cluster.database_name, - "db_cluster_parameter_group_name": cluster.db_cluster_parameter_group_name, - "db_subnet_group_name": cluster.db_subnet_group_name, - "deletion_protection": cluster.deletion_protection, - "engine_version": cluster.engine_version, - "master_username": cluster.master_username, - "master_user_password": cluster.master_user_password, - "manage_master_user_password": cluster.manage_master_user_password, - "option_group_name": cluster.option_group_name, - "port": cluster.port, - "replication_source_identifier": cluster.replication_source_identifier, - "storage_encrypted": cluster.storage_encrypted, - "storage_type": cluster.storage_type, - "tags": cluster.tags, - # "created_at": str(cluster.created_at), # Uncomment if needed - "status": cluster.status, - "primary_writer_instance": cluster.primary_writer_instance, - "reader_instances": cluster.reader_instances, - "cluster_endpoint": cluster.cluster_endpoint, - "instances_endpoints": cluster.instances_endpoints, - "pk_column": cluster.pk_column, - "pk_value": cluster.pk_value - } - } - return cluster_dict - def to_sql(self): # Convert the model instance to a dictionary data_dict = self.to_dict() diff --git a/DB/NEW_KT_DB/Models/DBInstanceModel.py b/DB/NEW_KT_DB/Models/DBInstanceModel.py new file mode 100644 index 00000000..5adc79b8 --- /dev/null +++ b/DB/NEW_KT_DB/Models/DBInstanceModel.py @@ -0,0 +1,59 @@ +import json +from datetime import datetime +import os +import sys +from DataAccess.ObjectManager import ObjectManager +sys.path.insert(0, os.path.abspath( + os.path.join(os.path.dirname(__file__), '../../../'))) +from Storage.NEW_KT_Storage.DataAccess.StorageManager import StorageManager + + +class DBInstance: + BASE_PATH = "db_instances" + table_name = 'db_instance' + pk_column = 'db_instance_id' + pk_column_data_type = 'TEXT' + table_structure = 'db_instance_id VARCHAR(255) PRIMARY KEY NOT NULL, allocated_storage INT NOT NULL, master_username VARCHAR(255) NOT NULL, master_user_password VARCHAR(255) NOT NULL, db_name VARCHAR(255) NOT NULL, port INT NOT NULL, status VARCHAR(50) NOT NULL, created_time DATETIME NOT NULL, endpoint VARCHAR(255) NOT NULL, databases TEXT NOT NULL, pk_value VARCHAR(255) NOT NULL' + + def __init__(self, **kwargs): + """ + Initialize a new DBInstance with the given parameters. + """ + self.db_instance_identifier = kwargs['db_instance_identifier'] + self.allocated_storage = kwargs.get('allocated_storage', 20) + self.master_username = kwargs['master_username'] + self.master_user_password = kwargs['master_user_password'] + self.db_name = kwargs.get('db_name', None) + self.port = kwargs.get('port', 3306) + self.status = 'available' + self.created_time = datetime.now() + self.endpoint = self.db_instance_identifier + storageManager = StorageManager(DBInstance.BASE_PATH) + storageManager.create_directory(self.endpoint) + self.databases = kwargs.get('databases', {}) + self.pk_value = kwargs.get('pk_value', self.db_instance_identifier) + + def to_dict(self): + """Retrieve the metadata of the DB instance as a dictionary.""" + return ObjectManager.convert_object_attributes_to_dictionary( + db_instance_identifier=self.db_instance_identifier, + allocated_storage=self.allocated_storage, + master_username=self.master_username, + master_user_password=self.master_user_password, + db_name=self.db_name, + port=self.port, + status=self.status, + created_time=str(self.created_time), + endpoint=self.endpoint, + databases=self.databases, + pk_value=self.pk_value + + ) + + def to_sql(self): + # Convert the model instance to a dictionary + data_dict = self.to_dict() + values = '(' + ", ".join(f'\'{json.dumps(v)}\'' if isinstance(v, dict) or isinstance(v, list) else f'\'{v}\'' if isinstance(v, str) else f'\'{str(v)}\'' + for v in data_dict.values()) + ')' + # values='(\''+self.db_instance_identifier+'\',\''+json.dumps(self.to_dict())+'\')' + return values diff --git a/DB/NEW_KT_DB/Service/Classes/DBClusterService.py b/DB/NEW_KT_DB/Service/Classes/DBClusterService.py index 3230bbfe..7d324428 100644 --- a/DB/NEW_KT_DB/Service/Classes/DBClusterService.py +++ b/DB/NEW_KT_DB/Service/Classes/DBClusterService.py @@ -11,6 +11,8 @@ from Abc import DBO from Validation import DBClusterValiditions from DataAccess import DBClusterManager + + # from DBInstanceService import DBInstanceService import os import json @@ -37,7 +39,7 @@ def __init__(self, dal: DBClusterManager, storage_manager: StorageManager, direc def get_file_path(self, cluster_name: str): # return os.path.join(self.directory, cluster_name + '.json') - return str(self.directory)+str(cluster_name)+'.json' + return str(self.directory)+'\\'+str(cluster_name)+'.json' def is_cluster_exist(self, cluster_identifier: str): cluster_path = self.get_file_path(cluster_identifier) @@ -51,7 +53,7 @@ def is_cluster_exist(self, cluster_identifier: str): return True - def create(self, **kwargs): + def create(self, instance_controller, **kwargs): '''Create a new DBCluster.''' @@ -94,32 +96,36 @@ def create(self, **kwargs): # Create physical folder structure # desktop_path = os.path.join(os.path.expanduser('~'), 'Desktop') # cluster_directory = os.path.join(desktop_path, f'Clusters/{cluster.db_cluster_identifier}') - cluster_directory = self.get_file_path(cluster.db_cluster_identifier) + cluster_directory = str(self.directory)+'\\'+str(cluster.db_cluster_identifier) self.storage_manager.create_directory(cluster_directory) # os.makedirs(cluster_directory, exist_ok=True) # Set cluster endpoint cluster.cluster_endpoint = cluster_directory + # storage_manager = StorageManager('Instances') + # instance_manager = DBInstanceManager.D('Clusters/instances.db') + # instanceService = DBInstanceService(instance_manager,storage_manager, 'Instances') + # instanceController = DBInstanceController.DBInstanceController(instanceService) # Create the primary writer instance primary_instance_name = f'{cluster.db_cluster_identifier}-primary' - # primary_instance = self.DBInstanceService( - # instance_name=primary_instance_name, - # cluster_identifier=cluster.db_cluster_identifier, - # allocated_storage=cluster.allocated_storage, - # master_username=cluster.master_username, - # master_user_password=cluster.master_user_password - # ) - primary_instance = { - "DBInstance": { - "db_instance_identifier": "my-db-instance-1", - "endpoint": { - "address": "my-db-instance-1.123456789012.us-west-2.rds.amazonaws.com", - "port": 3306, - "hosted_zone_id": "Z1PVIF0EXAMPLE" - }, - } - } + primary_instance = instance_controller.create_db_instance( + db_instance_identifier=primary_instance_name, + # cluster_identifier=cluster.db_cluster_identifier, + allocated_storage=cluster.allocated_storage, + master_username=cluster.master_username, + master_user_password=cluster.master_user_password + ) + # primary_instance = { + # "DBInstance": { + # "db_instance_identifier": "my-db-instance-1", + # "endpoint": { + # "address": "my-db-instance-1.123456789012.us-west-2.rds.amazonaws.com", + # "port": 3306, + # "hosted_zone_id": "Z1PVIF0EXAMPLE" + # }, + # } + # } # Retrieve primary instance details primary_instance_json_string = primary_instance.get("DBInstance") @@ -135,19 +141,19 @@ def create(self, **kwargs): # json.dump(cluster_dict, file, indent=4) # except IOError as e: # raise RuntimeError(f"Failed to write configuration file: {e}") - + configuration_file_path = cluster_directory+'\\'+cluster.db_cluster_identifier + "_configurations.json" json_object = json.dumps(cluster.to_dict()) - file_path = self.get_file_path(cluster.db_cluster_identifier+"_configurations") + # file_path = self.get_file_path(cluster.db_cluster_identifier+"_configurations") self.storage_manager.create_file( - file_path=file_path, content=json_object) + file_path=configuration_file_path, content=json_object) - # cluster_to_sql = cluster.to_sql() - ccc = cluster.to_dict() + cluster_to_sql = cluster.to_sql() + # ccc = cluster.to_dict() # bbb = json.dumps(ccc) # Store the cluster information in the database - self.dal.createInMemoryDBCluster(ccc) + return self.dal.createInMemoryDBCluster(cluster_to_sql) - return {"DBCluster": ccc} + # return {"DBCluster": ccc} @@ -160,8 +166,9 @@ def delete(self, cluster_identifier:str): file_path = self.get_file_path(cluster_identifier+"_configurations") self.storage_manager.delete_file(file_path=file_path) - directory_path = self.get_file_path(cluster_identifier) + directory_path = str(self.directory)+'\\'+str(cluster_identifier) self.storage_manager.delete_directory(directory_path) + self.dal.deleteInMemoryDBCluster(cluster_identifier) @@ -176,9 +183,7 @@ def describe(self, cluster_id): def modify(self, cluster_id: str, **kwargs): '''Modify an existing DBCluster.''' - # update object in code - # modify physical object - # update object in memory using DBClusterManager.modifyInMemoryDBCluster() function- send criteria using self attributes + # if not self.is_cluster_exist(cluster_id): # raise ValueError("Cluster does not exist!!") @@ -206,18 +211,27 @@ def modify(self, cluster_id: str, **kwargs): if 'master_user_password' in kwargs and not validate_master_user_password(kwargs['master_user_password'], kwargs.get('manage_master_user_password', False)): raise ValueError("Invalid master user password") - current_cluster = self.describe(cluster_id) + str_parts = ', '.join(f"{key} = '{value}'" for key, value in kwargs.items()) - # Update the cluster object - for key, value in kwargs.items(): - setattr(current_cluster, key, value) #update in memory - self.dal.modifyDBCluster(cluster_id,current_cluster) - + self.dal.modifyDBCluster(cluster_id,str_parts) + + columns = ['db_cluster_identifier', 'engine', 'allocated_storage', 'copy_tags_to_snapshot', + 'db_cluster_instance_class', 'database_name', 'db_cluster_parameter_group_name', + 'db_subnet_group_name', 'deletion_protection', 'engine_version', 'master_username', + 'master_user_password', 'manage_master_user_password', 'option_group_name', 'port', + 'replication_source_identifier', 'storage_encrypted', 'storage_type', 'tags', + 'created_at', 'status', 'primary_writer_instance', 'reader_instances', 'cluster_endpoint', + 'instances_endpoints', 'pk_column', 'pk_value'] + #update configurations + current_cluster = self.describe(cluster_id) + cluster_dict = dict(zip(columns, current_cluster[0])) + cluster_string = json.dumps(cluster_dict, indent=4) + file_path = self.get_file_path(cluster_id+'_configurations') self.storage_manager.delete_file(file_path) - self.storage_manager.create_file(file_path, current_cluster) + self.storage_manager.create_file(file_path, cluster_string) # self.storage_manager.create_file(file_path, json.dumps(current_cluster)) diff --git a/DB/NEW_KT_DB/Service/Classes/DBInstanceService.py b/DB/NEW_KT_DB/Service/Classes/DBInstanceService.py new file mode 100644 index 00000000..df436f91 --- /dev/null +++ b/DB/NEW_KT_DB/Service/Classes/DBInstanceService.py @@ -0,0 +1,101 @@ +import os +import shutil +import sys +from typing import Dict, Optional +from Exception.exception import DBInstanceNotFoundError, ParamValidationError +from Validation.DBInstanceValidition import check_extra_params, check_required_params,is_valid_db_instance_identifier +from Models.DBInstanceModel import DBInstance +from Service.Abc.DBO import DBO +from DataAccess.DBInstanceManager import DBInstanceManager +from Exception.exception import AlreadyExistsError +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../'))) +from Storage.NEW_KT_Storage.DataAccess.StorageManager import StorageManager + + +class DBInstanceService(DBO): + def __init__(self, dal:DBInstanceManager ): + self.dal = dal + + def create(self,**attributes): + '''Create a new DBCluster.''' + required_params = ['db_instance_identifier', 'master_username', 'master_user_password'] + all_params = ['db_name', 'port','allocated_storage'] + all_params.extend(required_params) + check_required_params(required_params, attributes) # check if there are all required parameters + check_extra_params(all_params, attributes) #check if there are not extra params that the function can't get + db_instance_identifier=attributes['db_instance_identifier'] + if not is_valid_db_instance_identifier(db_instance_identifier,63): + raise ValueError('db_instance_identifier is invalid') + if self.dal.is_db_instance_identifier_exist(db_instance_identifier): + raise AlreadyExistsError(f"the id {db_instance_identifier} is already exist") + db_instance = DBInstance(**attributes) + self.dal.createInMemoryDBInstance(db_instance) + return {'DBInstance': db_instance.to_dict()} + + def delete(self,kwargs): + """Delete a DB instance.""" + required_params = ['db_instance_identifier'] + all_params = ['skip_final_snapshot', 'final_db_snapshot_identifier', 'delete_automated_backups'] + all_params.extend(required_params) + check_required_params(required_params, kwargs) # check if there are all required parameters + check_extra_params(all_params, kwargs) #check if there are not extra params that the function can't get + db_instance_identifier = kwargs['db_instance_identifier'] + if not self.dal.is_db_instance_identifier_exist(db_instance_identifier): # check if db to delete exists + raise DBInstanceNotFoundError('This DB instance identifier does not exist') + db_instance=self.get(db_instance_identifier) + if 'skip_final_snapshot' not in kwargs or kwargs['skip_final_snapshot'] == False: #if need to do final snapshot + if 'final_db_snapshot_identifier' not in kwargs: #raise when snapshot id was not given + raise ParamValidationError('If you do not enable skip_final_snapshot parameter, you must specify the FinalDBSnapshotIdentifier parameter') + create_db_snapshot(db_instance_identifier=kwargs['db_instance_identifier'], #create final snapshot + db_snapshot_identifier=kwargs['final_db_snapshot_identifier']) + self.dal.deleteInMemoryDBInstance(db_instance_identifier) + endpoint = db_instance.endpoint + storageManager=StorageManager(DBInstance.BASE_PATH) + storageManager.delete_directory(endpoint) + del db_instance + + def describe(self,db_instance_identifier): + '''Describe the details of DBCluster.''' + if not self.dal.is_db_instance_identifier_exist(db_instance_identifier): # check if db to delete exists + raise DBInstanceNotFoundError('This DB instance identifier does not exist') + describe_db_instance=self.dal.describeDBInstance(db_instance_identifier)[0] + describe_db_instance_dict = { + 'db_instance_identifier': describe_db_instance[0], + 'allocated_storage': describe_db_instance[1], + 'master_username': describe_db_instance[2], + 'master_user_password': describe_db_instance[3], + 'db_name': describe_db_instance[4], + 'port': describe_db_instance[5], + 'status': describe_db_instance[6], + 'created_time': str(describe_db_instance[7]), + 'endpoint': describe_db_instance[8], + 'databases': describe_db_instance[9], + 'pk_value': describe_db_instance[10] + } + + return {'DBInstance':describe_db_instance_dict} + + def modify(self, **updates): + '''Modify an existing DBCluster.''' + required_params = ['db_instance_identifier'] + all_params = ['port','allocated_storage', 'master_user_password'] + all_params.extend(required_params) + check_required_params(required_params, updates) # check if there are all required parameters + check_extra_params(all_params, updates) #check if there are not extra params that the function can't get + db_instance_identifier=updates['db_instance_identifier'] + filtered_updates = {key: value for key, value in updates.items() if key != 'db_instance_identifier'} + set_clause = ', '.join([f"{key} = '{value}'" for key, value in filtered_updates.items()]) + self.dal.modifyDBInstance(db_instance_identifier,set_clause) + update_db_instance=self.get(db_instance_identifier) + return {'DBInstance':update_db_instance} + + def get(self,db_instance_identifier): + '''get code object.''' + describe_result=self.describe(db_instance_identifier) + if describe_result : + describe_result=describe_result['DBInstance'] + return DBInstance( + **describe_result + ) + return None + diff --git a/DB/NEW_KT_DB/Test/DBClusterTests.py b/DB/NEW_KT_DB/Test/DBClusterTests.py index a34dc75a..5619ccc8 100644 --- a/DB/NEW_KT_DB/Test/DBClusterTests.py +++ b/DB/NEW_KT_DB/Test/DBClusterTests.py @@ -1,10 +1,43 @@ import pytest -from GeneralTests import test_file_exists -from Service import DBClusterService +import sqlite3 +import sys +import os +# Add paths to sys.path +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '.'))) -def test_create(option_group, capsys): - """Test the create method.""" - DBClusterService.create("name"="DBClusterTest") - # ... - assert test_file_exists("DBClusterTest") \ No newline at end of file +from Service.Classes.DBClusterService import DBClusterService +from DataAccess.DBClusterManager import DBClusterManager +from Controller.DBClusterController import DBClusterController +from Storage.KT_Storage.DataAccess.StorageManager import StorageManager +from DataAccess.ObjectManager import ObjectManager +from Models.DBClusterModel import Cluster + +@pytest.fixture(scope = 'module',autouse=True) +def setup_services(): + + # Initialize the managers, services, controllers, and storage managers + manager = DBClusterManager('Clusters/clusters.db') + service = DBClusterService(manager,storage_manager, 'Clusters') + controller = DBClusterController(service) + storage_manager = StorageManager('Clusters') + + # Provide the variables as a dictionary or as individual items + return { + 'manager': manager, + 'service': service, + 'controller': controller, + 'storage_manager': storage_manager + } + +# def test_create_DBCluster_function(): +# cluster_data = { +# 'db_cluster_identifier': 'myCluster5', +# 'engine': 'mysql', +# 'allocated_storage':5, +# 'db_subnet_group_name': 'my-subnet-group' +# } + + # assert setup_services['controller'].create_db_cluster(**cluster_data) \ No newline at end of file diff --git a/DB/NEW_KT_DB/Validation/DBInstanceValidition.py b/DB/NEW_KT_DB/Validation/DBInstanceValidition.py new file mode 100644 index 00000000..ff0d4393 --- /dev/null +++ b/DB/NEW_KT_DB/Validation/DBInstanceValidition.py @@ -0,0 +1,22 @@ +import re + +from Exception.exception import ParamValidationError, MissingRequireParamError + + +def check_required_params(required_params, kwargs): + """Check if all required parameters are present in kwargs.""" + for param in required_params: + if param not in kwargs: + raise MissingRequireParamError(f"Missing required parameter in input: {param}") + + +def check_extra_params(all_params, kwargs): + string_all_params = ", ".join(all_params) + for param in kwargs: + if param not in all_params: + raise ParamValidationError(f"Unknown parameter in input: {param}, must be one of: {string_all_params}") + + +def is_valid_db_instance_identifier(identifier, length): + pattern = r'^[a-zA-Z][a-zA-Z0-9-]*[a-zA-Z0-9]$' + return ((1 <= len(identifier) <= length) and re.match(pattern, identifier) and '--' not in identifier) \ No newline at end of file From 2e5cd1fef1a5604b1b8d673cd355a34b28d876ec Mon Sep 17 00:00:00 2001 From: sara-lea Date: Wed, 18 Sep 2024 16:22:25 +0300 Subject: [PATCH 13/16] finish implementation of function including instance functions --- .../myCluster6/myCluster6_configurations.json | 1 - .../ClustersmyCluster1_configurations.json | 29 - .../ClustersmyCluster2_configurations.json | 1 - .../ClustersmyCluster3_configurations.json | 1 - .../Controller/DBClusterController.py | 36 +- .../Controller/DBInstanceController.py | 35 +- DB/NEW_KT_DB/DataAccess/DBClusterManager.py | 19 - DB/NEW_KT_DB/DataAccess/DBInstanceManager.py | 73 ++- DB/NEW_KT_DB/DataAccess/DBManager.py | 570 ------------------ DB/NEW_KT_DB/DataAccess/ObjectManager.py | 323 ---------- DB/NEW_KT_DB/Models/DBClusterModel.py | 1 - DB/NEW_KT_DB/Models/DBInstanceModel.py | 2 +- .../Service/Classes/DBClusterService.py | 53 +- .../Service/Classes/DBInstanceService.py | 188 ++++-- .../Validation/DBClusterValiditions.py | 52 -- 15 files changed, 235 insertions(+), 1149 deletions(-) delete mode 100644 DB/NEW_KT_DB/Controller/Clusters/Clusters/myCluster6/myCluster6_configurations.json delete mode 100644 DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster1_configurations.json delete mode 100644 DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster2_configurations.json delete mode 100644 DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster3_configurations.json diff --git a/DB/NEW_KT_DB/Controller/Clusters/Clusters/myCluster6/myCluster6_configurations.json b/DB/NEW_KT_DB/Controller/Clusters/Clusters/myCluster6/myCluster6_configurations.json deleted file mode 100644 index 06b8a5a2..00000000 --- a/DB/NEW_KT_DB/Controller/Clusters/Clusters/myCluster6/myCluster6_configurations.json +++ /dev/null @@ -1 +0,0 @@ -{"db_cluster_identifier": "myCluster6", "engine": "mysql", "allocated_storage": 5, "copy_tags_to_snapshot": false, "db_cluster_instance_class": false, "database_name": null, "db_cluster_parameter_group_name": null, "db_subnet_group_name": "my-subnet-group", "deletion_protection": false, "engine_version": null, "master_username": null, "master_user_password": null, "manage_master_user_password": false, "option_group_name": null, "port": null, "replication_source_identifier": null, "storage_encrypted": null, "storage_type": "aurora", "tags": null, "created_at": "2024-09-18T11:03:52.483405", "status": "available", "primary_writer_instance": "myCluster6-primary", "reader_instances": [], "cluster_endpoint": "Clusters\\myCluster6", "instances_endpoints": {"primary_instance": "myCluster6-primary"}, "pk_column": "db_cluster_identifier", "pk_value": "myCluster6"} \ No newline at end of file diff --git a/DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster1_configurations.json b/DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster1_configurations.json deleted file mode 100644 index 067f158f..00000000 --- a/DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster1_configurations.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_cluster_identifier": "myCluster1", - "engine": "postgres", - "allocated_storage": 3, - "copy_tags_to_snapshot": "False", - "db_cluster_instance_class": "False", - "database_name": "None", - "db_cluster_parameter_group_name": "None", - "db_subnet_group_name": "my-subnet-group", - "deletion_protection": "False", - "engine_version": "None", - "master_username": "None", - "master_user_password": "None", - "manage_master_user_password": "False", - "option_group_name": "None", - "port": "None", - "replication_source_identifier": "None", - "storage_encrypted": "None", - "storage_type": "aurora", - "tags": "None", - "created_at": "2024-09-17T21:08:23.715538", - "status": "available", - "primary_writer_instance": "my-db-instance-1", - "reader_instances": "[]", - "cluster_endpoint": "ClustersmyCluster1", - "instances_endpoints": "{\"primary_instance\": {\"address\": \"my-db-instance-1.123456789012.us-west-2.rds.amazonaws.com\", \"port\": 3306, \"hosted_zone_id\": \"Z1PVIF0EXAMPLE\"}}", - "pk_column": "db_cluster_identifier", - "pk_value": "myCluster1" -} \ No newline at end of file diff --git a/DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster2_configurations.json b/DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster2_configurations.json deleted file mode 100644 index 2b5e47d5..00000000 --- a/DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster2_configurations.json +++ /dev/null @@ -1 +0,0 @@ -{"db_cluster_identifier": "myCluster2", "engine": "mysql", "allocated_storage": 5, "copy_tags_to_snapshot": false, "db_cluster_instance_class": false, "database_name": null, "db_cluster_parameter_group_name": null, "db_subnet_group_name": "my-subnet-group", "deletion_protection": false, "engine_version": null, "master_username": null, "master_user_password": null, "manage_master_user_password": false, "option_group_name": null, "port": null, "replication_source_identifier": null, "storage_encrypted": null, "storage_type": "aurora", "tags": null, "created_at": "2024-09-17T21:09:36.710599", "status": "available", "primary_writer_instance": "my-db-instance-1", "reader_instances": [], "cluster_endpoint": "ClustersmyCluster2", "instances_endpoints": {"primary_instance": {"address": "my-db-instance-1.123456789012.us-west-2.rds.amazonaws.com", "port": 3306, "hosted_zone_id": "Z1PVIF0EXAMPLE"}}, "pk_column": "db_cluster_identifier", "pk_value": "myCluster2"} \ No newline at end of file diff --git a/DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster3_configurations.json b/DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster3_configurations.json deleted file mode 100644 index ba8c9b5e..00000000 --- a/DB/NEW_KT_DB/Controller/Clusters/ClustersmyCluster3_configurations.json +++ /dev/null @@ -1 +0,0 @@ -{"db_cluster_identifier": "myCluster3", "engine": "mysql", "allocated_storage": 5, "copy_tags_to_snapshot": false, "db_cluster_instance_class": false, "database_name": null, "db_cluster_parameter_group_name": null, "db_subnet_group_name": "my-subnet-group", "deletion_protection": false, "engine_version": null, "master_username": null, "master_user_password": null, "manage_master_user_password": false, "option_group_name": null, "port": null, "replication_source_identifier": null, "storage_encrypted": null, "storage_type": "aurora", "tags": null, "created_at": "2024-09-17T21:11:52.012571", "status": "available", "primary_writer_instance": "my-db-instance-1", "reader_instances": [], "cluster_endpoint": "ClustersmyCluster3", "instances_endpoints": {"primary_instance": {"address": "my-db-instance-1.123456789012.us-west-2.rds.amazonaws.com", "port": 3306, "hosted_zone_id": "Z1PVIF0EXAMPLE"}}, "pk_column": "db_cluster_identifier", "pk_value": "myCluster3"} \ No newline at end of file diff --git a/DB/NEW_KT_DB/Controller/DBClusterController.py b/DB/NEW_KT_DB/Controller/DBClusterController.py index d1c44810..5a4c0cbd 100644 --- a/DB/NEW_KT_DB/Controller/DBClusterController.py +++ b/DB/NEW_KT_DB/Controller/DBClusterController.py @@ -1,4 +1,3 @@ -# from Service.Classes import DBClusterService import sys import os sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..'))) @@ -22,7 +21,7 @@ def create_db_cluster(self, **kwargs): def delete_db_cluster(self , cluster_identifier): - self.service.delete(cluster_identifier) + self.service.delete(self.instance_controller, cluster_identifier) def modify_db_cluster(self, cluster_identifier, **updates): @@ -31,38 +30,5 @@ def modify_db_cluster(self, cluster_identifier, **updates): def describe_db_cluster(self, cluster_id): return self.service.describe(cluster_id) -if __name__=='__main__': - - # desktop_path = os.path.join(os.path.expanduser('~'), 'Desktop') - # cluster_directory = os.path.join(desktop_path, f'Clusters/clusters.db') - # base = os.path.join(desktop_path, f'Clusters') - storage_manager = StorageManager.StorageManager('Instances') - db_file = ObjectManager.ObjectManager('Clusters/instances.db') - instance_manager = DBInstanceManager.DBInstanceManager(db_file) - instanceService = DBInstanceService(instance_manager) - instanceController = DBInstanceController.DBInstanceController(instanceService) - - storage_manager = StorageManager.StorageManager('Clusters') - clusterManager = DBClusterManager.DBClusterManager('Clusters/clusters.db') - clusterService = DBClusterService(clusterManager,storage_manager, 'Clusters') - clusterController = DBClusterController(clusterService,instanceController) - cluster_data = { - 'db_cluster_identifier': 'myCluster6', - 'engine': 'mysql', - 'allocated_storage':5, - 'db_subnet_group_name': 'my-subnet-group' - } - - clusterController.create_db_cluster(**cluster_data) - # clusterController.delete_db_cluster('myCluster5') - update_data = { - 'engine': 'postgres', - 'allocated_storage':3, - } - # clusterController.modify_db_cluster('myCluster1', **update_data) - - # dfgh = clusterController.describe_db_cluster('myCluster3') - # print(dfgh) - diff --git a/DB/NEW_KT_DB/Controller/DBInstanceController.py b/DB/NEW_KT_DB/Controller/DBInstanceController.py index ffa916cd..a285d6fa 100644 --- a/DB/NEW_KT_DB/Controller/DBInstanceController.py +++ b/DB/NEW_KT_DB/Controller/DBInstanceController.py @@ -2,20 +2,45 @@ from typing import Optional, Dict from Service.Classes.DBInstanceService import DBInstanceService - class DBInstanceController: + def __init__(self, service: DBInstanceService): self.service = service - def create_db_instance(self,**kwargs): + def create_db_instance(self, **kwargs): + """ + Create a new DBInstance by passing the necessary attributes to the service. + + Params: kwargs: The required and optional attributes for creating a DBInstance. + Return: A dictionary containing the newly created DBInstance. + """ return self.service.create(**kwargs) - def delete_db_instance(self, db_instance_identifier: str): - self.service.delete(db_instance_identifier) + def delete_db_instance(self,db_instance_identifier,skip_final_snapshot=False,final_db_snapshot_identifier=None,delete_automated_backups=False): + """ + Delete a DBInstance by its identifier with options to handle final snapshots and automated backups. + + Params: db_instance_identifier: The primary key (ID) of the DBInstance to delete. + skip_final_snapshot: If True, skip creating a final snapshot before deletion. Defaults to False. + final_db_snapshot_identifier: If skip_final_snapshot is False, specify an identifier for the final DB snapshot. + delete_automated_backups: If True, delete automated backups along with the DBInstance. Defaults to False. + """ + self.service.delete(db_instance_identifier,skip_final_snapshot,final_db_snapshot_identifier,delete_automated_backups) def modify_db_instance(self, **kwargs): + """ + Modify an existing DBInstance by passing updated attributes. + + Params: kwargs: The attributes to modify for the DBInstance. + Return: A dictionary containing the modified DBInstance. + """ return self.service.modify(**kwargs) def describe_db_instance(self, db_instance_identifier: str): - return self.service.describe(db_instance_identifier) + """ + Retrieve details of a DBInstance by its identifier. + Params: db_instance_identifier: The primary key (ID) of the DBInstance to describe. + Return: A dictionary containing the details of the DBInstance. + """ + return self.service.describe(db_instance_identifier) diff --git a/DB/NEW_KT_DB/DataAccess/DBClusterManager.py b/DB/NEW_KT_DB/DataAccess/DBClusterManager.py index 8a00099f..7af84f8b 100644 --- a/DB/NEW_KT_DB/DataAccess/DBClusterManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBClusterManager.py @@ -54,27 +54,8 @@ def describeDBCluster(self, cluster_id): def modifyDBCluster(self, cluster_id, updates): self.object_manager.update_in_memory(self.object_name, updates, criteria=f" {self.pk_column} = '{cluster_id}'") - # def select(self, name:Optional[str] = None, columns = ["*"]): - # data = self.object_manager.get_from_memory(self.object_name,criteria=f" {self.pk_column} = {name}") - # if data: - # data_to_return = [{col:data[col] for col in columns}] - # data_to_return[self.pk_column] = name - # return data_to_return - - # else: - # raise ValueError(f"db cluster with name '{name}' not found") - - # def is_exists(self, name): - # """check if object exists in table""" - # try: - # self.select(name) - # return True - # except: - # return False - def get(self, cluster_id: str): data = self.object_manager.get_from_memory(self.object_name, criteria=f" {self.pk_column} = '{cluster_id}'") - # get_from_memory(self, object_name, columns=None, criteria=None) if data: data_mapping = {'db_cluster_identifier':cluster_id} for key, value in data[cluster_id].items(): diff --git a/DB/NEW_KT_DB/DataAccess/DBInstanceManager.py b/DB/NEW_KT_DB/DataAccess/DBInstanceManager.py index 3e6e1b8b..42b0d7c9 100644 --- a/DB/NEW_KT_DB/DataAccess/DBInstanceManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBInstanceManager.py @@ -7,27 +7,70 @@ from Models.DBInstanceModel import DBInstance class DBInstanceManager: - def __init__(self, object_manager:ObjectManager): + def __init__(self, object_manager: ObjectManager): self.object_manager = object_manager - self.object_manager.create_management_table( DBInstance.table_name, DBInstance.table_structure) + # Create the management table for DBInstance using its object name and table structure + self.object_manager.create_management_table(DBInstance.object_name, DBInstance.table_structure) - def createInMemoryDBInstance(self,db_instance:DBInstance): - columns=self.object_manager.db_manager.get_column_names_of_table(self.object_manager._convert_object_name_to_management_table_name(DBInstance.table_name)) - columns_str = ', '.join(columns) - self.object_manager.save_in_memory(DBInstance.table_name,db_instance.to_sql(),columns=columns_str) + def createInMemoryDBInstance(self, db_instance: DBInstance): + """ + Create a new DBInstance in memory. + Params db_instance: DBInstance object to be saved in memory. + """ + self.object_manager.save_in_memory(DBInstance.object_name, db_instance.to_sql()) - def deleteInMemoryDBInstance(self,db_instance_identifier:str): - self.object_manager.delete_from_memory_by_pk(pk_column=DBInstance.pk_column ,pk_value= db_instance_identifier,object_name=DBInstance.table_name) + def deleteInMemoryDBInstance(self, db_instance_identifier: str): + """ + Delete a DBInstance from memory by its identifier. + Params: db_instance_identifier: The primary key (ID) of the DBInstance to delete. + """ + self.object_manager.delete_from_memory_by_pk( + pk_column=DBInstance.pk_column, + pk_value=db_instance_identifier, + object_name=DBInstance.object_name + ) - def describeDBInstance(self,db_instance_identifier:str): - return self.object_manager.get_from_memory(criteria=f"{DBInstance.pk_column} = '{db_instance_identifier}'",object_name=DBInstance.table_name,columns='*')#[db_instance_identifier] + def describeDBInstance(self, db_instance_identifier: str): + """ + Retrieve the details of a DBInstance based on its identifier. + Params: db_instance_identifier: The primary key (ID) of the DBInstance to describe. + + Return: List of DBInstance attributes matching the criteria. + """ + # Fetch DBInstance from memory using its primary key + return self.object_manager.get_from_memory( + criteria=f"{DBInstance.pk_column} = '{db_instance_identifier}'", + object_name=DBInstance.object_name, + columns='*' + ) - def modifyDBInstance(self,db_instance_identifier,updates): - self.object_manager.update_in_memory(criteria=f"{DBInstance.pk_column} = '{db_instance_identifier}'",object_name=DBInstance.table_name,updates=updates) + def modifyDBInstance(self, db_instance_identifier: str, updates: str): + """ + Modify the attributes of an existing DBInstance in memory. + + Params: db_instance_identifier: The primary key (ID) of the DBInstance to modify. + updates: SQL-like string containing the updates to be applied (e.g., "port = '3306'"). + """ + # Update the instance's attributes based on the provided update string + self.object_manager.update_in_memory( + criteria=f"{DBInstance.pk_column} = '{db_instance_identifier}'", + object_name=DBInstance.object_name, + updates=updates + ) - def is_db_instance_identifier_exist(self, db_instance_identifier: int) -> bool: - '''Check if an db_instance with the given ID exists in the database.''' - return self.object_manager.db_manager.is_object_exist(self.object_manager._convert_object_name_to_management_table_name(DBInstance.table_name),criteria=f"{DBInstance.pk_column} = '{db_instance_identifier}'") + def is_db_instance_exist(self, db_instance_identifier: int) -> bool: + """ + Check if a DBInstance with the given identifier exists in memory. + + Params: db_instance_identifier: The primary key (ID) of the DBInstance to check. + + Return: True if the DBInstance exists, otherwise False. + """ + # Check if the object exists by its primary key in the management table + return bool(self.object_manager.db_manager.is_object_exist( + self.object_manager._convert_object_name_to_management_table_name(DBInstance.object_name), + criteria=f"{DBInstance.pk_column} = '{db_instance_identifier}'" + )) diff --git a/DB/NEW_KT_DB/DataAccess/DBManager.py b/DB/NEW_KT_DB/DataAccess/DBManager.py index 8d576271..06314d32 100644 --- a/DB/NEW_KT_DB/DataAccess/DBManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBManager.py @@ -1,573 +1,3 @@ -# import sqlite3 -# from typing import Dict, Any, List, Optional, Tuple -# import json -# from sqlite3 import OperationalError - -# class DBManager: -# def __init__(self, db_file: str): -# '''Initialize the database connection and create tables if they do not exist.''' -# self.connection = sqlite3.connect(db_file) - - -# # rachel-8511, ShaniStrassProg -# def close(self): -# '''Close the database connection.''' -# self.connection.close() - - -# # # saraNoigershel -# # def execute_query_with_multiple_results(self, query: str) -> Optional[List[Tuple]]: -# # '''Execute a given query and return the results.''' -# # try: -# # c = self.connection.cursor() -# # c.execute(query) -# # results = c.fetchall() -# # # self.connection.commit() ??? - -# # saraNoigershel -# def execute_query_with_multiple_results(self, query: str, params:Tuple = ()) -> Optional[List[Tuple]]: -# '''Execute a given query and return the results.''' -# try: -# c = self.connection.cursor() -# c.execute(query, params) -# results = c.fetchall() -# self.connection.commit() -# return results if results else None -# except OperationalError as e: -# raise Exception(f'Error executing query {query}: {e}') - - -# # ShaniStrassProg -# def execute_query_with_single_result(self, query: str, params:Tuple = ()) -> Optional[Tuple]: -# '''Execute a given query and return a single result.''' -# try: -# c = self.connection.cursor() -# c.execute(query, params) -# result = c.fetchone() -# self.connection.commit() -# return result if result else None - -# except OperationalError as e: -# raise Exception(f'Error executing query {query}: {e}') - - -# # Riki7649255 -# def execute_query_without_results(self, query: str, params:Tuple = ()): -# '''Execute a given query without waiting for any result.''' -# try: -# c = self.connection.cursor() -# c.execute(query, params) -# self.connection.commit() -# except OperationalError as e: -# raise Exception(f'Error executing query {query}: {e}') - - -# # Yael, Riki7649255 -# def create_table(self, table_name, table_structure): -# '''create a table in a given db by given table_structure''' -# create_statement = f'''CREATE TABLE IF NOT EXISTS {table_name} ({table_structure})''' -# self.execute_query_without_results(create_statement) - -# # Riki7649255 based on rachel-8511, ShaniStrassProg -# def insert_data_into_table(self, table_name, columns, data): -# column_names = ', '.join(columns) -# placeholders = ', '.join(['?' for _ in range(len(columns))]) -# insert_query = f"INSERT INTO {table_name} ({column_names}) VALUES ({placeholders})" -# self.execute_query_without_results(insert_query, data) - -# # # Riki7649255 based on rachel-8511, Shani -# # def update_records_in_table(self, table_name: str, updates: Dict[str, Any], criteria: str) -> None: -# # '''Update records in the specified table based on criteria.''' - -# # # add documentation here -# # set_clause = ', '.join([f'{k} = ?' for k in updates.keys()]) -# # values = list(updates.values()) - -# # update_statement = f''' -# # UPDATE {table_name} -# # SET {set_clause} -# # WHERE {criteria} -# # ''' - -# # execute_query_without_results(update_statement) - - -# # def update_records_in_table(self, table_name: str, updates: Dict[str, Any], criteria: str) -> None: -# # '''Update records in the specified table based on criteria.''' - -# # set_clause = '(' + ', '.join([f'{k}' for k in updates.keys()]) + ') = (' + ', '.join([f'"{v}"' for v in updates.values()]) + ')' - -# # update_statement = f''' -# # UPDATE {table_name} -# # SET {set_clause} -# # WHERE {criteria} -# # ''' - -# # self.execute_query_without_results(update_statement) - -# # Riki7649255 based on rachel-8511, Shani -# def update_records_in_table(self, table_name: str, updates: Dict[str, Any], criteria: Optional[str]) -> None: -# '''Update records in the specified table based on criteria.''' - -# # add documentation here -# set_clause = ', '.join([f'{k} = ?' for k in updates.keys()]) -# values = tuple(updates.values()) - -# update_statement = f''' -# UPDATE {table_name} -# SET {set_clause} -# ''' -# if criteria: -# update_statement = update_statement + f'''WHERE {criteria}''' - -# self.execute_query_without_results(update_statement, values) - - -# # Riki7649255 based on rachel-8511 -# def delete_data_from_table(self, table_name: str, criteria: str) -> None: -# '''Delete a record from the specified table based on criteria.''' - -# delete_statement = f''' -# DELETE FROM {table_name} -# WHERE {criteria} -# ''' - -# execute_query_without_results(delete_statement) - - -# delete_statement = f''' -# DELETE FROM {table_name} -# WHERE {criteria} -# ''' - -# self.execute_query_without_results(delete_statement) - -# # Tem-M -# def get_columns_from_table(self, table_name): -# '''Get the columns from the specified table.''' -# try: -# get_columns_query = f"""PRAGMA table_info({table_name});""" -# cols = self.execute_query_with_multiple_results(get_columns_query) -# return [col[1] for col in cols] -# except Exception as e: -# print(f"Error occurred while fetching columns from table {table_name}: {e}") -# return [] - -# def get_all_data_from_table(self, table_name): -# try: -# get_all_data_query = f"""SELECT * FROM {table_name}""" -# return self.execute_query_with_multiple_results(get_all_data_query) -# except Exception as e: -# print(f"Error occurred while fetching data from table {table_name}: {e}") -# return [] - -# # rachel-8511, Riki7649255 -# def select_and_return_records_from_table(self, table_name: str, columns: List[str] = ['*'], criteria: Optional[str] = None) -> Dict[int, Dict[str, Any]]: -# '''Select records from the specified table based on criteria. -# Args: -# table_name (str): The name of the table. -# columns (List[str]): The columns to select. Default is all columns ('*'). -# criteria (str): SQL condition for filtering records. Default is no filter. -# Returns: -# Dict[int, Dict[str, Any]]: A dictionary where keys are object_ids and values are metadata. -# ''' -# cols = columns -# if cols == ['*']: -# cols = self.get_columns_from_table(table_name) - -# columns_clause = ', '.join(cols) -# query = f'SELECT {columns_clause} FROM {table_name}' -# if criteria: -# query += f' WHERE {criteria};' - -# try: -# results = self.execute_query_with_multiple_results(query) -# return {result[0]: dict(zip(cols if columns != ['*'] else cols[1:], result[1:])) for result in results} -# except OperationalError as e: -# raise Exception(f'Error selecting from {table_name}: {e}') -# except TypeError as e: -# raise Exception(f'Error selecting from {table_name}: {e}') - -# def is_exists_in_table(self, table_name:str, criteria:str): -# """check if rows exists in table""" -# return self.select_and_return_records_from_table(table_name, criteria=criteria) != {} - - -# # rachel-8511, ShaniStrassProg, Riki7649255 -# def describe_table(self, table_name: str) -> Dict[str, str]: -# '''Describe table structure.''' -# try: -# desc_statement = f'PRAGMA table_info({table_name})' -# columns = self.execute_query_with_multiple_results(desc_statement) -# return {col[1]: col[2] for col in columns} -# except OperationalError as e: -# raise Exception(f'Error describing table {table_name}: {e}') - -# # rachel-8511, ShaniStrassProg -# def close(self): -# '''Close the database connection.''' -# self.connection.close() - - -# # ShaniStrassProg -# # should be in ObjectManager and send the query to one of the execute_query functions -# # def is_json_column_contains_key_and_value(self, table_name: str, key: str, value: str) -> bool: -# # '''Check if a specific key-value pair exists within a JSON column in the given table.''' -# # try: -# # c = self.connection.cursor() -# # # Properly format the LIKE clause with escaped quotes for key and value -# # c.execute(f''' -# # SELECT COUNT(*) FROM {table_name} -# # WHERE metadata LIKE ? -# # LIMIT 1 -# # ''', (f'%"{key}": "{value}"%',)) -# # # Check if the count is greater than 0, indicating the key-value pair exists -# # return c.fetchone()[0] > 0 -# # except sqlite3.OperationalError as e: -# # print(f'Error: {e}') -# # return False - - -# # Yael, ShaniStrassProg -# # should be in ObjectManager and send the query to one of the execute_query functions -# # def is_identifier_exist(self, table_name: str, value: str) -> bool: -# # '''Check if a specific value exists within a column in the given table.''' -# # try: -# # c = self.connection.cursor() -# # c.execute(f''' -# # SELECT COUNT(*) FROM {table_name} -# # WHERE id LIKE ? -# # ''', (value,)) -# # return c.fetchone()[0] > 0 -# # except sqlite3.OperationalError as e: -# # print(f'Error: {e}') - - -# # sara-lea -# # should be in ObjectManager and send the query to one of the execute_query functions -# # def update_metadata(self, table_name: str, user_id: int, key: str, value: Any, action: str = 'set') -> None: -# # """ -# # Generic function to update a specific part of the metadata for a given user. - -# # Parameters: -# # - table_name: The name of the table containing the metadata. -# # - user_id: The ID of the user whose metadata needs to be updated. -# # - key: The specific part of the metadata to update (e.g., 'password', 'roles', 'policies', 'quotas'). -# # - value: The value to update, append, or delete (could be a new policy, role, password, etc.). -# # - action: The type of update to perform ('set' for replacing, 'append' for adding to lists, 'update' for dicts, 'delete' for removing). -# # """ - -# # # Step 1: Retrieve the current metadata for the user -# # query = f"SELECT metadata FROM {table_name} WHERE user_id = ?" -# # try: -# # c = self.connection.cursor() -# # c.execute(query, (user_id,)) -# # row = c.fetchone() -# # if row: -# # metadata = json.loads(row[0]) # Assuming metadata is stored as a JSON string -# # else: -# # raise Exception(f"User with id {user_id} not found.") -# # except OperationalError as e: -# # raise Exception(f'Error fetching metadata: {e}') - -# # # Step 2: Modify the relevant part of the metadata -# # if key not in metadata: -# # raise KeyError(f"Key '{key}' not found in metadata.") - -# # if action == 'set': -# # # Replace the value of the key directly -# # metadata[key] = value -# # elif action == 'append' and isinstance(metadata[key], list): -# # # Append to the list (for roles, policies, etc.) -# # metadata[key].append(value) -# # elif action == 'update' and isinstance(metadata[key], dict): -# # # Update a dictionary (for quotas or nested data) -# # metadata[key].update(value) -# # elif action == 'delete': -# # if isinstance(metadata[key], list): -# # # Remove an item from a list -# # if value in metadata[key]: -# # metadata[key].remove(value) -# # else: -# # raise ValueError(f"Value '{value}' not found in list '{key}'.") -# # elif isinstance(metadata[key], dict): -# # # Remove a key from a dictionary -# # if value in metadata[key]: -# # del metadata[key][value] -# # else: -# # raise ValueError(f"Key '{value}' not found in dictionary '{key}'.") -# # else: -# # raise ValueError(f"Action 'delete' is not supported for the data type of key '{key}'.") -# # else: -# # raise ValueError(f"Invalid action '{action}' or incompatible data type for key '{key}'.") - -# # # Step 3: Serialize metadata back to JSON string -# # updated_metadata = json.dumps(metadata) - -# # # Step 4: Use your update function to write the new metadata back to the database -# # updates = {"metadata": updated_metadata} -# # criteria = f"user_id = {user_id}" -# # self.update(table_name, updates, criteria) - - -# import sqlite3 -# from typing import Dict, Any, List, Optional, Tuple -# import json -# from sqlite3 import OperationalError - -# class DBManager: -# def __init__(self, db_file: str): -# '''Initialize the database connection and create tables if they do not exist.''' -# self.connection = sqlite3.connect(db_file) - - -# # saraNoigershel -# def execute_query_with_multiple_results(self, query: str, params:Tuple = ()) -> Optional[List[Tuple]]: -# '''Execute a given query and return the results.''' -# try: -# c = self.connection.cursor() -# c.execute(query, params) -# results = c.fetchall() -# self.connection.commit() -# return results if results else None -# except OperationalError as e: -# raise Exception(f'Error executing query {query}: {e}') - - -# # ShaniStrassProg -# def execute_query_with_single_result(self, query: str, params:Tuple = ()) -> Optional[Tuple]: -# '''Execute a given query and return a single result.''' -# try: -# c = self.connection.cursor() -# c.execute(query, params) -# result = c.fetchone() -# self.connection.commit() -# return result if result else None - -# except OperationalError as e: -# raise Exception(f'Error executing query {query}: {e}') - - -# # Riki7649255 -# def execute_query_without_results(self, query: str, params:Tuple = ()): -# '''Execute a given query without waiting for any result.''' -# try: -# c = self.connection.cursor() -# c.execute(query, params) -# self.connection.commit() -# except OperationalError as e: -# raise Exception(f'Error executing query {query}: {e}') - - -# # Yael, Riki7649255 -# def create_table(self, table_name, table_structure): -# '''create a table in a given db by given table_structure''' -# create_statement = f'''CREATE TABLE IF NOT EXISTS {table_name} ({table_structure})''' -# self.execute_query_without_results(create_statement) - -# # Riki7649255 based on rachel-8511, ShaniStrassProg -# def insert_data_into_table(self, table_name, columns, data): -# column_names = ', '.join(columns) -# placeholders = ', '.join(['?' for _ in range(len(columns))]) -# insert_query = f"INSERT INTO {table_name} ({column_names}) VALUES ({placeholders})" -# self.execute_query_without_results(insert_query, data) - - -# # Riki7649255 based on rachel-8511, Shani -# def update_records_in_table(self, table_name: str, updates: Dict[str, Any], criteria: Optional[str]) -> None: -# '''Update records in the specified table based on criteria.''' - -# # add documentation here -# set_clause = ', '.join([f'{k} = ?' for k in updates.keys()]) -# values = tuple(updates.values()) - -# update_statement = f''' -# UPDATE {table_name} -# SET {set_clause} -# ''' -# if criteria: -# update_statement = update_statement + f'''WHERE {criteria}''' - -# self.execute_query_without_results(update_statement, values) - - -# # Riki7649255 based on rachel-8511 -# def delete_data_from_table(self, table_name: str, criteria: str) -> None: -# '''Delete a record from the specified table based on criteria.''' - -# delete_statement = f''' -# DELETE FROM {table_name} -# WHERE {criteria} -# ''' - -# self.execute_query_without_results(delete_statement) - -# # Tem-M -# def get_columns_from_table(self, table_name): -# '''Get the columns from the specified table.''' -# try: -# get_columns_query = f"""PRAGMA table_info({table_name});""" -# cols = self.execute_query_with_multiple_results(get_columns_query) -# return [col[1] for col in cols] -# except Exception as e: -# print(f"Error occurred while fetching columns from table {table_name}: {e}") -# return [] - -# def get_all_data_from_table(self, table_name): -# try: -# get_all_data_query = f"""SELECT * FROM {table_name}""" -# return self.execute_query_with_multiple_results(get_all_data_query) -# except Exception as e: -# print(f"Error occurred while fetching data from table {table_name}: {e}") -# return [] - -# # rachel-8511, Riki7649255 -# def select_and_return_records_from_table(self, table_name: str, columns: List[str] = ['*'], criteria: Optional[str] = None) -> Dict[int, Dict[str, Any]]: -# '''Select records from the specified table based on criteria. -# Args: -# table_name (str): The name of the table. -# columns (List[str]): The columns to select. Default is all columns ('*'). -# criteria (str): SQL condition for filtering records. Default is no filter. -# Returns: -# Dict[int, Dict[str, Any]]: A dictionary where keys are object_ids and values are metadata. -# ''' -# cols = columns -# if cols == ['*']: -# cols = self.get_columns_from_table(table_name) - -# columns_clause = ', '.join(cols) -# query = f'SELECT {columns_clause} FROM {table_name}' -# if criteria: -# query += f' WHERE {criteria};' - -# try: -# results = self.execute_query_with_multiple_results(query) -# return {result[0]: dict(zip(cols if columns != ['*'] else cols[1:], result[1:])) for result in results} -# except OperationalError as e: -# raise Exception(f'Error selecting from {table_name}: {e}') -# except TypeError as e: -# raise Exception(f'Error selecting from {table_name}: {e}') - -# def is_exists_in_table(self, table_name:str, criteria:str): -# """check if rows exists in table""" -# return self.select_and_return_records_from_table(table_name, criteria=criteria) != {} - - -# # rachel-8511, ShaniStrassProg, Riki7649255 -# def describe_table(self, table_name: str) -> Dict[str, str]: -# '''Describe table structure.''' -# try: -# desc_statement = f'PRAGMA table_info({table_name})' -# columns = self.execute_query_with_multiple_results(desc_statement) -# return {col[1]: col[2] for col in columns} -# except OperationalError as e: -# raise Exception(f'Error describing table {table_name}: {e}') - -# # rachel-8511, ShaniStrassProg -# def close(self): -# '''Close the database connection.''' -# self.connection.close() - - -# # ShaniStrassProg -# # should be in ObjectManager and send the query to one of the execute_query functions -# # def is_json_column_contains_key_and_value(self, table_name: str, key: str, value: str) -> bool: -# # '''Check if a specific key-value pair exists within a JSON column in the given table.''' -# # try: -# # c = self.connection.cursor() -# # # Properly format the LIKE clause with escaped quotes for key and value -# # c.execute(f''' -# # SELECT COUNT(*) FROM {table_name} -# # WHERE metadata LIKE ? -# # LIMIT 1 -# # ''', (f'%"{key}": "{value}"%',)) -# # # Check if the count is greater than 0, indicating the key-value pair exists -# # return c.fetchone()[0] > 0 -# # except sqlite3.OperationalError as e: -# # print(f'Error: {e}') -# # return False - - -# # Yael, ShaniStrassProg -# # should be in ObjectManager and send the query to one of the execute_query functions -# # def is_identifier_exist(self, table_name: str, value: str) -> bool: -# # '''Check if a specific value exists within a column in the given table.''' -# # try: -# # c = self.connection.cursor() -# # c.execute(f''' -# # SELECT COUNT(*) FROM {table_name} -# # WHERE id LIKE ? -# # ''', (value,)) -# # return c.fetchone()[0] > 0 -# # except sqlite3.OperationalError as e: -# # print(f'Error: {e}') - - -# # sara-lea -# # should be in ObjectManager and send the query to one of the execute_query functions -# # def update_metadata(self, table_name: str, user_id: int, key: str, value: Any, action: str = 'set') -> None: -# # """ -# # Generic function to update a specific part of the metadata for a given user. - -# # Parameters: -# # - table_name: The name of the table containing the metadata. -# # - user_id: The ID of the user whose metadata needs to be updated. -# # - key: The specific part of the metadata to update (e.g., 'password', 'roles', 'policies', 'quotas'). -# # - value: The value to update, append, or delete (could be a new policy, role, password, etc.). -# # - action: The type of update to perform ('set' for replacing, 'append' for adding to lists, 'update' for dicts, 'delete' for removing). -# # """ - -# # # Step 1: Retrieve the current metadata for the user -# # query = f"SELECT metadata FROM {table_name} WHERE user_id = ?" -# # try: -# # c = self.connection.cursor() -# # c.execute(query, (user_id,)) -# # row = c.fetchone() -# # if row: -# # metadata = json.loads(row[0]) # Assuming metadata is stored as a JSON string -# # else: -# # raise Exception(f"User with id {user_id} not found.") -# # except OperationalError as e: -# # raise Exception(f'Error fetching metadata: {e}') - -# # # Step 2: Modify the relevant part of the metadata -# # if key not in metadata: -# # raise KeyError(f"Key '{key}' not found in metadata.") - -# # if action == 'set': -# # # Replace the value of the key directly -# # metadata[key] = value -# # elif action == 'append' and isinstance(metadata[key], list): -# # # Append to the list (for roles, policies, etc.) -# # metadata[key].append(value) -# # elif action == 'update' and isinstance(metadata[key], dict): -# # # Update a dictionary (for quotas or nested data) -# # metadata[key].update(value) -# # elif action == 'delete': -# # if isinstance(metadata[key], list): -# # # Remove an item from a list -# # if value in metadata[key]: -# # metadata[key].remove(value) -# # else: -# # raise ValueError(f"Value '{value}' not found in list '{key}'.") -# # elif isinstance(metadata[key], dict): -# # # Remove a key from a dictionary -# # if value in metadata[key]: -# # del metadata[key][value] -# # else: -# # raise ValueError(f"Key '{value}' not found in dictionary '{key}'.") -# # else: -# # raise ValueError(f"Action 'delete' is not supported for the data type of key '{key}'.") -# # else: -# # raise ValueError(f"Invalid action '{action}' or incompatible data type for key '{key}'.") - -# # # Step 3: Serialize metadata back to JSON string -# # updated_metadata = json.dumps(metadata) - -# # # Step 4: Use your update function to write the new metadata back to the database -# # updates = {"metadata": updated_metadata} -# # criteria = f"user_id = {user_id}" -# # self.update(table_name, updates, criteria) import sqlite3 from typing import Dict, Any, List, Optional, Tuple import json diff --git a/DB/NEW_KT_DB/DataAccess/ObjectManager.py b/DB/NEW_KT_DB/DataAccess/ObjectManager.py index e5bde115..d8f6e244 100644 --- a/DB/NEW_KT_DB/DataAccess/ObjectManager.py +++ b/DB/NEW_KT_DB/DataAccess/ObjectManager.py @@ -1,331 +1,8 @@ -# from typing import Dict, Any, Optional -# import json -# import sqlite3 -# from .DBManager import DBManager - -# class ObjectManager: -# def __init__(self, db_file: str): -# '''Initialize ObjectManager with the database connection.''' -# self.db_manager = DBManager(db_file) - - -# # for internal use only: - -# # Riki7649255 based on rachel-8511 - -# # def create_management_table(self, table_name, table_structure='object_id INTEGER PRIMARY KEY AUTOINCREMENT,type_object TEXT NOT NULL,metadata TEXT NOT NULL'): - -# def _create_management_table(self, table_name, table_structure='object_id INTEGER PRIMARY KEY AUTOINCREMENT,type_object TEXT NOT NULL,metadata TEXT NOT NULL'): -# """ -# creates a management table with the name and the structure you specify -# make sure to keep track of the table name you send here - you will use it whenever you want to access the table -# you created - this function should only be called from within the specific manager you created (i.e. DBInstanceManager) -# """ -# self.db_manager.create_table(table_name, table_structure) - - -# # Riki7649255 based on saraNoigershel, Tem-M -# def _insert_object_to_management_table(self, table_name, object): -# """ -# inserts an object to the management table you specified, the object should be sent as is! not converted to a tuple or dictionary! -# if the table does not exist, the function will abort and raise an error -# the table should be created within the __init__ function of the manager you created (i.e. DBInstanceManager) -# """ -# columns = self.db_manager.get_columns_from_table(table_name) -# values = tuple([str(getattr(object, column)) for column in columns]) -# self.db_manager.insert_data_into_table(table_name, columns, values) - -# # Malki1844 -# def _get_all_data_from_table(self, table_name): -# self.db_manager.get_all_data_from_table(table_name) - -# # Riki7649255 based on rachel-8511 -# def _update_object_in_management_table_by_criteria(self, table_name, updates, criteria): -# updates = {k: str(v) for k, v in updates.items()} -# self.db_manager.update_records_in_table(table_name, updates, criteria) - - - -# # rachel-8511, Riki7649255 -# def _get_object_from_management_table(self, pk_col, table_name, object_id: int) -> Dict[str, Any]: -# '''Retrieve an object from the database.''' -# result = self.db_manager.select_and_return_records_from_table(table_name=table_name, criteria=f'{pk_col} = \'{object_id}\'') -# if result: -# return result -# else: -# raise FileNotFoundError(f'Object with ID {object_id} not found.') - -# def _get_objects_from_management_table_by_criteria(self, table_name, columns = ["*"], criteria:Optional[str] = None) -> Dict: -# '''Retrieve an object from the database.''' -# result = self.db_manager.select_and_return_records_from_table(table_name, columns, criteria) -# if result: -# return result -# else: -# raise FileNotFoundError(f'Objects with criteria {criteria} not found.') - - -# # rachel-8511, ShaniStrassProg, Riki7649255 -# def _delete_object_from_management_table(self, table_name, criteria) -> None: -# '''Delete an object from the database.''' -# self.db_manager.delete_data_from_table(table_name, criteria) - -# def _delete_object_from_management_table_by_id(self, pk_col, table_name, object_id) -> None: -# '''Delete an object from the database.''' -# self.db_manager.delete_data_from_table(table_name, criteria= f'{pk_col} = \'{object_id}\'') - - -# # rachel-8511, ShaniStrassProg is it needed? -# # def get_all_objects(self) -> Dict[int, Dict[str, Any]]: -# # '''Retrieve all objects from the database.''' -# # return self.db_manager.select(self.table_name, ['object_id', 'type_object', 'metadata']) - - -# # rachel-8511 is it needed? -# # def describe_table(self) -> Dict[str, str]: -# # '''Describe the schema of the table.''' -# # return self.db_manager.describe(self.table_name) - - -# def _convert_object_name_to_management_table_name(self, object_name): -# return f'mng_{object_name}s' - - - -# # def is_management_table_exist(table_name): -# # # check if table exists using single result query -# # return self.db_manager.execute_query_with_single_result(f'desc table {table_name}') - - -# # for outer use: -# # def save_in_memory(self, object): - -# def _is_management_table_exist(self, table_name): -# # Check if table exists by querying the sqlite_master table -# query = f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}'" -# return self.db_manager.execute_query_with_single_result(query) - - -# # for outer use: -# # def save_in_memory(self, table_name, object): - -# # # insert object info into management table mng_{object_name}s -# # # for exmple: object db_instance will be saved in table mng_db_instances -# # table_name = self.convert_object_name_to_management_table_name(self.object_name) - -# # if not self.is_management_table_exist(table_name): -# # self.create_management_table(table_name) - -# # self.insert_object_to_management_table(table_name, object) - -# def save_in_memory(self, object): -# # insert object info into management table mng_{object_name}s -# # for exmple: object db_instance will be saved in table mng_db_instances - -# table_name = str(object.__class__.__name__) -# if not self._is_management_table_exist(table_name): -# self._create_management_table(table_name) -# # self.insert_object_to_management_table(table_name, object) - -# self._insert_object_to_management_table(table_name, object) - - -# def delete_from_memory_by_id(self, pk_col, pk_val, table_name:str): -# # pk_val is the object id -# # if criteria not sent- use PK for deletion -# criteria = f'{pk_col} = \'{pk_val}\'' - -# table_name = self.convert_object_name_to_management_table_name(self.object_name) - -# self.delete_data_from_table(table_name, criteria) - - -# def update_in_memory(self, updates, criteria='default'): - -# # if criteria not sent- use PK for deletion -# if criteria == 'default': -# criteria = f'{self.pk_column} = {self.pk_value}' - -# table_name = self.convert_object_name_to_management_table_name(self.object_name) - -# self.update_object_in_management_table_by_criteria(table_name, updates, criteria) - - -# def get_from_memory(self): -# self.get_object_from_management_table(self.object_id) - -# self.db_manager.delete_data_from_table(table_name, criteria) - -# def update_in_memory_by_criteria(self,table_name:str, updates:Dict, criteria): -# self._update_object_in_management_table_by_criteria(table_name, updates, criteria) - -# def update_in_memory_by_id(self, pk_col, table_name, updates, object_id:Optional[str]): -# if not object_id: -# raise ValueError('must be or criteria or object id') -# criteria = f'{pk_col} = \'{object_id}\'' -# self.update_in_memory_by_criteria(table_name, updates, criteria) - - -# def get_from_memory_by_id(self, pk_col, table_name, object_id, columns = ["*"]): -# """get records from memory by criteria or id""" -# criteria = f'{pk_col} = \'{object_id}\'' -# return self._get_objects_from_management_table_by_criteria(table_name, columns, criteria) - - -# def convert_object_attributes_to_dictionary(**kwargs): - -# dict = {} - -# for key, value in kwargs.items(): -# dict[key] = value - -# return dict - - -# from typing import Dict, Any, Optional -# import json -# import sqlite3 -# from .DBManager import DBManager - -# class ObjectManager: -# def __init__(self, db_file: str): -# '''Initialize ObjectManager with the database connection.''' -# self.db_manager = DBManager(db_file) - - -# # for internal use only: - -# # Riki7649255 based on rachel-8511 -# def _create_management_table(self, table_name, table_structure='object_id INTEGER PRIMARY KEY AUTOINCREMENT,type_object TEXT NOT NULL,metadata TEXT NOT NULL'): -# """ -# creates a management table with the name and the structure you specify -# make sure to keep track of the table name you send here - you will use it whenever you want to access the table -# you created - this function should only be called from within the specific manager you created (i.e. DBInstanceManager) -# """ -# self.db_manager.create_table(table_name, table_structure) - - -# # Riki7649255 based on saraNoigershel, Tem-M -# def _insert_object_to_management_table(self, table_name, object): -# """ -# inserts an object to the management table you specified, the object should be sent as is! not converted to a tuple or dictionary! -# if the table does not exist, the function will abort and raise an error -# the table should be created within the __init__ function of the manager you created (i.e. DBInstanceManager) -# """ -# columns = self.db_manager.get_columns_from_table(table_name) -# values = tuple([str(getattr(object, column)) for column in columns]) -# self.db_manager.insert_data_into_table(table_name, columns, values) - -# # Malki1844 -# def _get_all_data_from_table(self, table_name): -# self.db_manager.get_all_data_from_table(table_name) - -# # Riki7649255 based on rachel-8511 -# def _update_object_in_management_table_by_criteria(self, table_name, updates, criteria): -# updates = {k: str(v) for k, v in updates.items()} -# self.db_manager.update_records_in_table(table_name, updates, criteria) - - - -# # rachel-8511, Riki7649255 -# def _get_object_from_management_table(self, pk_col, table_name, object_id: int) -> Dict[str, Any]: -# '''Retrieve an object from the database.''' -# result = self.db_manager.select_and_return_records_from_table(table_name=table_name, criteria=f'{pk_col} = \'{object_id}\'') -# if result: -# return result -# else: -# raise FileNotFoundError(f'Object with ID {object_id} not found.') - -# def _get_objects_from_management_table_by_criteria(self, table_name, columns = ["*"], criteria:Optional[str] = None) -> Dict: -# '''Retrieve an object from the database.''' -# result = self.db_manager.select_and_return_records_from_table(table_name, columns, criteria) -# if result: -# return result -# else: -# raise FileNotFoundError(f'Objects with criteria {criteria} not found.') - - -# # rachel-8511, ShaniStrassProg, Riki7649255 -# def _delete_object_from_management_table(self, table_name, criteria) -> None: -# '''Delete an object from the database.''' -# self.db_manager.delete_data_from_table(table_name, criteria) - -# def _delete_object_from_management_table_by_id(self, pk_col, table_name, object_id) -> None: -# '''Delete an object from the database.''' -# self.db_manager.delete_data_from_table(table_name, criteria= f'{pk_col} = \'{object_id}\'') - - -# # rachel-8511, ShaniStrassProg is it needed? -# # def get_all_objects(self) -> Dict[int, Dict[str, Any]]: -# # '''Retrieve all objects from the database.''' -# # return self.db_manager.select(self.table_name, ['object_id', 'type_object', 'metadata']) - - -# # rachel-8511 is it needed? -# # def describe_table(self) -> Dict[str, str]: -# # '''Describe the schema of the table.''' -# # return self.db_manager.describe(self.table_name) - - -# def _convert_object_name_to_management_table_name(self, object_name): -# return f'mng_{object_name}s' - - -# def _is_management_table_exist(self, table_name): -# # Check if table exists by querying the sqlite_master table -# query = f"SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}'" -# return self.db_manager.execute_query_with_single_result(query) - - -# # for outer use: -# def save_in_memory(self, table_name, object): - -# # insert object info into management table mng_{object_name}s -# # for exmple: object db_instance will be saved in table mng_db_instances - -# self._insert_object_to_management_table(table_name, object) - - -# def delete_from_memory_by_id(self, pk_col, pk_val, table_name:str): -# # pk_val is the object id -# # if criteria not sent- use PK for deletion -# criteria = f'{pk_col} = \'{pk_val}\'' - -# self.db_manager.delete_data_from_table(table_name, criteria) - -# def update_in_memory_by_criteria(self,table_name:str, updates:Dict, criteria): -# self._update_object_in_management_table_by_criteria(table_name, updates, criteria) - -# def update_in_memory_by_id(self, pk_col, table_name, updates, object_id:Optional[str]): -# if not object_id: -# raise ValueError('must be or criteria or object id') -# criteria = f'{pk_col} = \'{object_id}\'' -# self.update_in_memory_by_criteria(table_name, updates, criteria) - - -# def get_from_memory_by_id(self, pk_col, table_name, object_id, columns = ["*"]): -# """get records from memory by criteria or id""" -# criteria = f'{pk_col} = \'{object_id}\'' -# return self._get_objects_from_management_table_by_criteria(table_name, columns, criteria) - - -# def convert_object_attributes_to_dictionary(**kwargs): - -# dict = {} - -# for key, value in kwargs.items(): -# dict[key] = value - -# return dict - - from typing import Dict, Any, Optional import json import sqlite3 import sys import os -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..'))) -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from DataAccess import DBManager class ObjectManager: diff --git a/DB/NEW_KT_DB/Models/DBClusterModel.py b/DB/NEW_KT_DB/Models/DBClusterModel.py index 6f78d8e1..850c5439 100644 --- a/DB/NEW_KT_DB/Models/DBClusterModel.py +++ b/DB/NEW_KT_DB/Models/DBClusterModel.py @@ -72,7 +72,6 @@ def to_dict(self) -> Dict: '''Retrieve the data of the DB cluster as a dictionary.''' return ObjectManager.ObjectManager.convert_object_attributes_to_dictionary( - self, db_cluster_identifier=self.db_cluster_identifier, engine=self.engine, allocated_storage=self.allocated_storage, diff --git a/DB/NEW_KT_DB/Models/DBInstanceModel.py b/DB/NEW_KT_DB/Models/DBInstanceModel.py index 5adc79b8..75a155ed 100644 --- a/DB/NEW_KT_DB/Models/DBInstanceModel.py +++ b/DB/NEW_KT_DB/Models/DBInstanceModel.py @@ -10,7 +10,7 @@ class DBInstance: BASE_PATH = "db_instances" - table_name = 'db_instance' + object_name = 'db_instance' pk_column = 'db_instance_id' pk_column_data_type = 'TEXT' table_structure = 'db_instance_id VARCHAR(255) PRIMARY KEY NOT NULL, allocated_storage INT NOT NULL, master_username VARCHAR(255) NOT NULL, master_user_password VARCHAR(255) NOT NULL, db_name VARCHAR(255) NOT NULL, port INT NOT NULL, status VARCHAR(50) NOT NULL, created_time DATETIME NOT NULL, endpoint VARCHAR(255) NOT NULL, databases TEXT NOT NULL, pk_value VARCHAR(255) NOT NULL' diff --git a/DB/NEW_KT_DB/Service/Classes/DBClusterService.py b/DB/NEW_KT_DB/Service/Classes/DBClusterService.py index 7d324428..9086ab25 100644 --- a/DB/NEW_KT_DB/Service/Classes/DBClusterService.py +++ b/DB/NEW_KT_DB/Service/Classes/DBClusterService.py @@ -38,7 +38,6 @@ def __init__(self, dal: DBClusterManager, storage_manager: StorageManager, direc self.storage_manager.create_directory(directory) def get_file_path(self, cluster_name: str): - # return os.path.join(self.directory, cluster_name + '.json') return str(self.directory)+'\\'+str(cluster_name)+'.json' def is_cluster_exist(self, cluster_identifier: str): @@ -94,20 +93,12 @@ def create(self, instance_controller, **kwargs): cluster = DBClusterModel.Cluster(**kwargs) # Create physical folder structure - # desktop_path = os.path.join(os.path.expanduser('~'), 'Desktop') - # cluster_directory = os.path.join(desktop_path, f'Clusters/{cluster.db_cluster_identifier}') cluster_directory = str(self.directory)+'\\'+str(cluster.db_cluster_identifier) self.storage_manager.create_directory(cluster_directory) - # os.makedirs(cluster_directory, exist_ok=True) # Set cluster endpoint cluster.cluster_endpoint = cluster_directory - # storage_manager = StorageManager('Instances') - # instance_manager = DBInstanceManager.D('Clusters/instances.db') - # instanceService = DBInstanceService(instance_manager,storage_manager, 'Instances') - # instanceController = DBInstanceController.DBInstanceController(instanceService) - # Create the primary writer instance primary_instance_name = f'{cluster.db_cluster_identifier}-primary' primary_instance = instance_controller.create_db_instance( db_instance_identifier=primary_instance_name, @@ -116,48 +107,25 @@ def create(self, instance_controller, **kwargs): master_username=cluster.master_username, master_user_password=cluster.master_user_password ) - # primary_instance = { - # "DBInstance": { - # "db_instance_identifier": "my-db-instance-1", - # "endpoint": { - # "address": "my-db-instance-1.123456789012.us-west-2.rds.amazonaws.com", - # "port": 3306, - # "hosted_zone_id": "Z1PVIF0EXAMPLE" - # }, - # } - # } # Retrieve primary instance details primary_instance_json_string = primary_instance.get("DBInstance") - # primary_instance_json_data = json.loads(primary_instance_json_string) cluster.instances_endpoints["primary_instance"] = primary_instance_json_string.get("endpoint") cluster.primary_writer_instance = primary_instance_json_string.get('db_instance_identifier') # # Create configuration file - # cluster_config_path = os.path.join(cluster_directory, 'cluster_config.json') - # cluster_dict = cluster.to_dict() - # try: - # with open(cluster_config_path, 'w') as file: - # json.dump(cluster_dict, file, indent=4) - # except IOError as e: - # raise RuntimeError(f"Failed to write configuration file: {e}") configuration_file_path = cluster_directory+'\\'+cluster.db_cluster_identifier + "_configurations.json" json_object = json.dumps(cluster.to_dict()) - # file_path = self.get_file_path(cluster.db_cluster_identifier+"_configurations") self.storage_manager.create_file( file_path=configuration_file_path, content=json_object) cluster_to_sql = cluster.to_sql() - # ccc = cluster.to_dict() - # bbb = json.dumps(ccc) - # Store the cluster information in the database return self.dal.createInMemoryDBCluster(cluster_to_sql) - # return {"DBCluster": ccc} - def delete(self, cluster_identifier:str): + def delete(self,instance_controller, cluster_identifier:str): '''Delete an existing DBCluster.''' # if not self.is_cluster_exist(cluster_identifier): @@ -168,7 +136,23 @@ def delete(self, cluster_identifier:str): directory_path = str(self.directory)+'\\'+str(cluster_identifier) self.storage_manager.delete_directory(directory_path) - + + columns = ['db_cluster_identifier', 'engine', 'allocated_storage', 'copy_tags_to_snapshot', + 'db_cluster_instance_class', 'database_name', 'db_cluster_parameter_group_name', + 'db_subnet_group_name', 'deletion_protection', 'engine_version', 'master_username', + 'master_user_password', 'manage_master_user_password', 'option_group_name', 'port', + 'replication_source_identifier', 'storage_encrypted', 'storage_type', 'tags', + 'created_at', 'status', 'primary_writer_instance', 'reader_instances', 'cluster_endpoint', + 'instances_endpoints', 'pk_column', 'pk_value'] + + #update configurations + current_cluster = self.describe(cluster_identifier) + cluster_dict = dict(zip(columns, current_cluster[0])) + instance_controller.delete_db_instance(db_instance_identifier = cluster_dict['primary_writer_instance'],final_db_snapshot_identifier = True) + if cluster_dict['reader_instances'] != [] : + for id in cluster_dict['reader_instances']: + instance_controller.delete_db_instance(db_instance_identifier = id, final_db_snapshot_identifier = True) + self.dal.deleteInMemoryDBCluster(cluster_identifier) @@ -232,7 +216,6 @@ def modify(self, cluster_id: str, **kwargs): file_path = self.get_file_path(cluster_id+'_configurations') self.storage_manager.delete_file(file_path) self.storage_manager.create_file(file_path, cluster_string) - # self.storage_manager.create_file(file_path, json.dumps(current_cluster)) diff --git a/DB/NEW_KT_DB/Service/Classes/DBInstanceService.py b/DB/NEW_KT_DB/Service/Classes/DBInstanceService.py index df436f91..125126d7 100644 --- a/DB/NEW_KT_DB/Service/Classes/DBInstanceService.py +++ b/DB/NEW_KT_DB/Service/Classes/DBInstanceService.py @@ -3,7 +3,7 @@ import sys from typing import Dict, Optional from Exception.exception import DBInstanceNotFoundError, ParamValidationError -from Validation.DBInstanceValidition import check_extra_params, check_required_params,is_valid_db_instance_identifier +from Validation.DBInstanceValidition import check_extra_params, check_required_params, is_valid_db_instance_identifier from Models.DBInstanceModel import DBInstance from Service.Abc.DBO import DBO from DataAccess.DBInstanceManager import DBInstanceManager @@ -13,89 +13,155 @@ class DBInstanceService(DBO): - def __init__(self, dal:DBInstanceManager ): + + def __init__(self, dal: DBInstanceManager): self.dal = dal - def create(self,**attributes): - '''Create a new DBCluster.''' + def create(self, **attributes): + """ + Create a new DBInstance. + + Params: attributes: dict containing attributes for the new DBInstance. + Required fields: 'db_instance_identifier', 'master_username', 'master_user_password' + Optional fields: 'db_name', 'port', 'allocated_storage' + + Raises: ValueError: if the db_instance_identifier is invalid. + AlreadyExistsError: if a DB instance with the given identifier already exists. + + Return: dict with a 'DBInstance' key containing the created instance's details. + """ required_params = ['db_instance_identifier', 'master_username', 'master_user_password'] - all_params = ['db_name', 'port','allocated_storage'] + all_params = ['db_name', 'port', 'allocated_storage'] all_params.extend(required_params) - check_required_params(required_params, attributes) # check if there are all required parameters - check_extra_params(all_params, attributes) #check if there are not extra params that the function can't get - db_instance_identifier=attributes['db_instance_identifier'] - if not is_valid_db_instance_identifier(db_instance_identifier,63): + + # Validate required and extra parameters + check_required_params(required_params, attributes) + check_extra_params(all_params, attributes) + + db_instance_identifier = attributes['db_instance_identifier'] + + # Validate DB instance identifier format + if not is_valid_db_instance_identifier(db_instance_identifier, 63): raise ValueError('db_instance_identifier is invalid') - if self.dal.is_db_instance_identifier_exist(db_instance_identifier): - raise AlreadyExistsError(f"the id {db_instance_identifier} is already exist") + + if self.dal.is_db_instance_exist(db_instance_identifier): + raise AlreadyExistsError(f"The ID {db_instance_identifier} already exists") + db_instance = DBInstance(**attributes) self.dal.createInMemoryDBInstance(db_instance) + return {'DBInstance': db_instance.to_dict()} - def delete(self,kwargs): - """Delete a DB instance.""" - required_params = ['db_instance_identifier'] - all_params = ['skip_final_snapshot', 'final_db_snapshot_identifier', 'delete_automated_backups'] - all_params.extend(required_params) - check_required_params(required_params, kwargs) # check if there are all required parameters - check_extra_params(all_params, kwargs) #check if there are not extra params that the function can't get - db_instance_identifier = kwargs['db_instance_identifier'] - if not self.dal.is_db_instance_identifier_exist(db_instance_identifier): # check if db to delete exists - raise DBInstanceNotFoundError('This DB instance identifier does not exist') - db_instance=self.get(db_instance_identifier) - if 'skip_final_snapshot' not in kwargs or kwargs['skip_final_snapshot'] == False: #if need to do final snapshot - if 'final_db_snapshot_identifier' not in kwargs: #raise when snapshot id was not given - raise ParamValidationError('If you do not enable skip_final_snapshot parameter, you must specify the FinalDBSnapshotIdentifier parameter') - create_db_snapshot(db_instance_identifier=kwargs['db_instance_identifier'], #create final snapshot - db_snapshot_identifier=kwargs['final_db_snapshot_identifier']) + def delete(self, db_instance_identifier,skip_final_snapshot=False,final_db_snapshot_identifier=None,delete_automated_backups=False): + """ + Delete a DBInstance by its identifier with options to handle final snapshots and automated backups. + + Params: db_instance_identifier: The primary key (ID) of the DBInstance to delete. + skip_final_snapshot: If True, skip creating a final snapshot before deletion. Defaults to False. + final_db_snapshot_identifier: If skip_final_snapshot is False, specify an identifier for the final DB snapshot. + delete_automated_backups: If True, delete automated backups along with the DBInstance. Defaults to False. + """ + if not self.dal.is_db_instance_exist(db_instance_identifier): + raise DBInstanceNotFoundError(db_instance_identifier,'This DB instance identifier does not exist') + + # Get the DBInstance to delete - object + db_instance = self.get(db_instance_identifier) + + # Handle final snapshot before deletion if required + # print(skip_final_snapshot) + if skip_final_snapshot == False: + if final_db_snapshot_identifier is None: + raise ParamValidationError('If skip_final_snapshot is False, final_db_snapshot_identifier must be specified') + create_db_snapshot( + db_instance_identifier=db_instance_identifier, + db_snapshot_identifier=final_db_snapshot_identifier + ) + self.dal.deleteInMemoryDBInstance(db_instance_identifier) + + # Clean up storage (delete associated directory) endpoint = db_instance.endpoint - storageManager=StorageManager(DBInstance.BASE_PATH) + storageManager = StorageManager(DBInstance.BASE_PATH) storageManager.delete_directory(endpoint) + + # Delete the DBInstance object del db_instance - def describe(self,db_instance_identifier): - '''Describe the details of DBCluster.''' - if not self.dal.is_db_instance_identifier_exist(db_instance_identifier): # check if db to delete exists + def describe(self, db_instance_identifier): + """ + Describe the details of a DBInstance. + + Params: db_instance_identifier: str, identifier of the DBInstance to describe. + + Raises: DBInstanceNotFoundError: if the DB instance does not exist. + + Return: dict with a 'DBInstance' key containing the details of the instance. + """ + if not self.dal.is_db_instance_exist(db_instance_identifier): raise DBInstanceNotFoundError('This DB instance identifier does not exist') - describe_db_instance=self.dal.describeDBInstance(db_instance_identifier)[0] + + # Retrieve and structure the DBInstance description + describe_db_instance = self.dal.describeDBInstance(db_instance_identifier)[0] describe_db_instance_dict = { - 'db_instance_identifier': describe_db_instance[0], - 'allocated_storage': describe_db_instance[1], - 'master_username': describe_db_instance[2], - 'master_user_password': describe_db_instance[3], - 'db_name': describe_db_instance[4], - 'port': describe_db_instance[5], - 'status': describe_db_instance[6], - 'created_time': str(describe_db_instance[7]), - 'endpoint': describe_db_instance[8], - 'databases': describe_db_instance[9], - 'pk_value': describe_db_instance[10] + 'db_instance_identifier': describe_db_instance[0], + 'allocated_storage': describe_db_instance[1], + 'master_username': describe_db_instance[2], + 'master_user_password': describe_db_instance[3], + 'db_name': describe_db_instance[4], + 'port': describe_db_instance[5], + 'status': describe_db_instance[6], + 'created_time': str(describe_db_instance[7]), + 'endpoint': describe_db_instance[8], + 'databases': describe_db_instance[9], + 'pk_value': describe_db_instance[10] } - return {'DBInstance':describe_db_instance_dict} + return {'DBInstance': describe_db_instance_dict} def modify(self, **updates): - '''Modify an existing DBCluster.''' + """ + Modify an existing DBInstance. + + Params: updates: dict containing the attributes to modify in the DBInstance. + Required field: 'db_instance_identifier' + Optional fields: 'port', 'allocated_storage', 'master_user_password' + + Raises: DBInstanceNotFoundError: if the DB instance does not exist. + + Return: dict with a 'DBInstance' key containing the updated instance's details. + """ required_params = ['db_instance_identifier'] - all_params = ['port','allocated_storage', 'master_user_password'] + all_params = ['port', 'allocated_storage', 'master_user_password'] all_params.extend(required_params) - check_required_params(required_params, updates) # check if there are all required parameters - check_extra_params(all_params, updates) #check if there are not extra params that the function can't get - db_instance_identifier=updates['db_instance_identifier'] + + # Validate required and extra parameters + check_required_params(required_params, updates) + check_extra_params(all_params, updates) + + db_instance_identifier = updates['db_instance_identifier'] + + # Prepare the SQL-like set clause for updates filtered_updates = {key: value for key, value in updates.items() if key != 'db_instance_identifier'} set_clause = ', '.join([f"{key} = '{value}'" for key, value in filtered_updates.items()]) - self.dal.modifyDBInstance(db_instance_identifier,set_clause) - update_db_instance=self.get(db_instance_identifier) - return {'DBInstance':update_db_instance} + + self.dal.modifyDBInstance(db_instance_identifier, set_clause) + + # Return the updated DB instance details + update_db_instance = self.get(db_instance_identifier) + return {'DBInstance': update_db_instance} - def get(self,db_instance_identifier): - '''get code object.''' - describe_result=self.describe(db_instance_identifier) - if describe_result : - describe_result=describe_result['DBInstance'] - return DBInstance( - **describe_result - ) - return None + def get(self, db_instance_identifier): + """ + Retrieve a DBInstance object from the database. + Params: db_instance_identifier: str, identifier of the DBInstance to retrieve. + + Return: DBInstance object or None if not found. + """ + describe_result = self.describe(db_instance_identifier) + + if describe_result: + describe_result = describe_result['DBInstance'] + return DBInstance(**describe_result) + + return None diff --git a/DB/NEW_KT_DB/Validation/DBClusterValiditions.py b/DB/NEW_KT_DB/Validation/DBClusterValiditions.py index 84bd1319..752119cd 100644 --- a/DB/NEW_KT_DB/Validation/DBClusterValiditions.py +++ b/DB/NEW_KT_DB/Validation/DBClusterValiditions.py @@ -125,55 +125,3 @@ def check_required_params(required_params, **kwargs): if param not in kwargs.keys(): return False return True - -# import sqlite3 -# from sqlite3 import OperationalError -# import re -# import sys - -# def string_in_dict(string: str, values: dict) -> bool: -# """Check if the string is in dict.""" -# return string in values - -# def is_valid_length(string: str, min_length: int, max_length: int) -> bool: -# """Check if the string is valid based on the length.""" -# return min_length <= len(string) <= max_length - -# def is_valid_pattern(string: str, pattern: str) -> bool: -# """Check if the optionGroupName is valid based on the pattern.""" -# return bool(re.match(pattern, string)) - -# def exist_key_value_in_json_column(conn: sqlite3.Connection, table_name: str, column_name: str, key: str, value: str) -> bool: -# """Check if a specific key-value pair exists within a JSON column in the given table.""" -# try: -# c = conn.cursor() -# c.execute(f''' -# SELECT COUNT(*) FROM {table_name} -# WHERE {column_name} LIKE ? -# ''', (f'%"{key}": "{value}"%',)) -# return c.fetchone()[0] > 0 -# except OperationalError as e: -# print(f"Error: {e}") - -# def exist_value_in_column(conn: sqlite3.Connection, table_name: str, column_name: str, value: str) -> bool: -# """Check if a specific value exists within a column in the given table.""" -# try: -# c = conn.cursor() -# c.execute(f''' -# SELECT COUNT(*) FROM {table_name} -# WHERE {column_name} LIKE ? -# ''', (value,)) -# return c.fetchone()[0] > 0 -# except OperationalError as e: -# print(f"Error: {e}") - - - -# def is_valid_number(num: int, min: int = -sys.maxsize - 1, max: int = sys.maxsize) -> bool: -# return min <= num <= max - -# def check_required_params(required_params, **kwargs): -# for param in required_params: -# if param not in kwargs.keys(): -# return False -# return True \ No newline at end of file From aa0624d986424918d2596512eeab0940437b5b24 Mon Sep 17 00:00:00 2001 From: sara-lea Date: Wed, 18 Sep 2024 22:27:04 +0300 Subject: [PATCH 14/16] delete not related files --- .../Untitled-checkpoint.ipynb | 6 - DB/ELTS/Untitled.ipynb | 52 ---- .../Controller/DBClusterController.py | 63 +++++ .../Controller/DBInstanceController.py | 46 ---- .../Controller/DBSubnetGroupController.py | 24 -- DB/NEW_KT_DB/DataAccess/DBClusterManager.py | 19 +- DB/NEW_KT_DB/DataAccess/DBInstanceManager.py | 76 ------ .../DataAccess/DBSubnetGroupManager.py | 38 --- DB/NEW_KT_DB/Models/DBInstanceModel.py | 59 ----- DB/NEW_KT_DB/Models/DBSubnetGroupModel.py | 71 ------ .../Service/Classes/DBClusterService.py | 27 +- .../Service/Classes/DBInstanceService.py | 167 ------------ .../Service/Classes/DBSubnetGroupService.py | 91 ------- DB/NEW_KT_DB/Test/DBClusterTests.py | 241 +++++++++++++++--- DB/NEW_KT_DB/Test/DBSubnetGroupTests.py | 228 ----------------- .../Validation/DBInstanceValidition.py | 22 -- 16 files changed, 304 insertions(+), 926 deletions(-) delete mode 100644 DB/ELTS/.ipynb_checkpoints/Untitled-checkpoint.ipynb delete mode 100644 DB/ELTS/Untitled.ipynb delete mode 100644 DB/NEW_KT_DB/Controller/DBInstanceController.py delete mode 100644 DB/NEW_KT_DB/Controller/DBSubnetGroupController.py delete mode 100644 DB/NEW_KT_DB/DataAccess/DBInstanceManager.py delete mode 100644 DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py delete mode 100644 DB/NEW_KT_DB/Models/DBInstanceModel.py delete mode 100644 DB/NEW_KT_DB/Models/DBSubnetGroupModel.py delete mode 100644 DB/NEW_KT_DB/Service/Classes/DBInstanceService.py delete mode 100644 DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py delete mode 100644 DB/NEW_KT_DB/Test/DBSubnetGroupTests.py delete mode 100644 DB/NEW_KT_DB/Validation/DBInstanceValidition.py diff --git a/DB/ELTS/.ipynb_checkpoints/Untitled-checkpoint.ipynb b/DB/ELTS/.ipynb_checkpoints/Untitled-checkpoint.ipynb deleted file mode 100644 index 363fcab7..00000000 --- a/DB/ELTS/.ipynb_checkpoints/Untitled-checkpoint.ipynb +++ /dev/null @@ -1,6 +0,0 @@ -{ - "cells": [], - "metadata": {}, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/DB/ELTS/Untitled.ipynb b/DB/ELTS/Untitled.ipynb deleted file mode 100644 index d389bf14..00000000 --- a/DB/ELTS/Untitled.ipynb +++ /dev/null @@ -1,52 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 40, - "id": "d05cb473", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "# Get the desktop path\n", - "desktop_path = os.path.join(os.path.expanduser('~'), 'Desktop')\n", - "db_cluster_identifier= \"cluster\"\n", - "# Create the full directory path on the desktop\n", - "cluster_directory = os.path.join(desktop_path, f'clusters/{db_cluster_identifier}')\n", - "\n", - "# Create the directory (and any necessary intermediate directories)\n", - "os.makedirs(cluster_directory, exist_ok=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "76a40da7", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/DB/NEW_KT_DB/Controller/DBClusterController.py b/DB/NEW_KT_DB/Controller/DBClusterController.py index 5a4c0cbd..407d4ab8 100644 --- a/DB/NEW_KT_DB/Controller/DBClusterController.py +++ b/DB/NEW_KT_DB/Controller/DBClusterController.py @@ -30,5 +30,68 @@ def modify_db_cluster(self, cluster_identifier, **updates): def describe_db_cluster(self, cluster_id): return self.service.describe(cluster_id) + def get_all_db_clusters(self): + return self.service.get_all_cluster() + + + +if __name__=='__main__': + + # desktop_path = os.path.join(os.path.expanduser('~'), 'Desktop') + # cluster_directory = os.path.join(desktop_path, f'Clusters/clusters.db') + # base = os.path.join(desktop_path, f'Clusters') + storage_manager = StorageManager.StorageManager('Instances') + db_file = ObjectManager.ObjectManager('Clusters/instances.db') + instance_manager = DBInstanceManager.DBInstanceManager(db_file) + instanceService = DBInstanceService(instance_manager) + instanceController = DBInstanceController.DBInstanceController(instanceService) + + storage_manager = StorageManager.StorageManager('Clusters') + clusterManager = DBClusterManager.DBClusterManager('Clusters/clusters.db') + clusterService = DBClusterService(clusterManager,storage_manager, 'Clusters') + clusterController = DBClusterController(clusterService,instanceController) + + # storage_manager = StorageManager.StorageManager('Instances') + # db_file = ObjectManager.ObjectManager('Clusters/instances.db') + # instance_manager = DBInstanceManager.DBInstanceManager(db_file) + # instanceService = DBInstanceService(instance_manager) + # instanceController = DBInstanceController.DBInstanceController(instanceService) + cluster_data = { + 'db_cluster_identifier': 'Cluster27', + 'engine': 'mysql', + 'allocated_storage':5, + 'db_subnet_group_name': 'my-subnet-group' + } + + # clusterController.create_db_cluster(**cluster_data) + clusterController.delete_db_cluster('ClusterTest') + + # aa = clusterController.get_all_db_clusters() + # print(aa) + update_data = { + 'engine': 'postgres', + 'allocated_storage':3, + } + # clusterController.modify_db_cluster('myCluster1', **update_data) + + # dfgh = clusterController.describe_db_cluster('myCluster3') + # print(dfgh) + columns = ['db_cluster_identifier', 'engine', 'allocated_storage', 'copy_tags_to_snapshot', + 'db_cluster_instance_class', 'database_name', 'db_cluster_parameter_group_name', + 'db_subnet_group_name', 'deletion_protection', 'engine_version', 'master_username', + 'master_user_password', 'manage_master_user_password', 'option_group_name', 'port', + 'replication_source_identifier', 'storage_encrypted', 'storage_type', 'tags', + 'created_at', 'status', 'primary_writer_instance', 'reader_instances', 'cluster_endpoint', + 'instances_endpoints', 'pk_column', 'pk_value'] + + #update configurations + # current_cluster = clusterController.describe_db_cluster('Cluster26') + # print(type(current_cluster)) + # cluster_dict = dict(zip(columns, current_cluster[0])) + # print(type(cluster_dict['reader_instances'])) + # if cluster_dict['reader_instances']!='[]': + # for id in cluster_dict['reader_instances']: + # print(id) + diff --git a/DB/NEW_KT_DB/Controller/DBInstanceController.py b/DB/NEW_KT_DB/Controller/DBInstanceController.py deleted file mode 100644 index a285d6fa..00000000 --- a/DB/NEW_KT_DB/Controller/DBInstanceController.py +++ /dev/null @@ -1,46 +0,0 @@ -import datetime -from typing import Optional, Dict -from Service.Classes.DBInstanceService import DBInstanceService - -class DBInstanceController: - - def __init__(self, service: DBInstanceService): - self.service = service - - def create_db_instance(self, **kwargs): - """ - Create a new DBInstance by passing the necessary attributes to the service. - - Params: kwargs: The required and optional attributes for creating a DBInstance. - Return: A dictionary containing the newly created DBInstance. - """ - return self.service.create(**kwargs) - - def delete_db_instance(self,db_instance_identifier,skip_final_snapshot=False,final_db_snapshot_identifier=None,delete_automated_backups=False): - """ - Delete a DBInstance by its identifier with options to handle final snapshots and automated backups. - - Params: db_instance_identifier: The primary key (ID) of the DBInstance to delete. - skip_final_snapshot: If True, skip creating a final snapshot before deletion. Defaults to False. - final_db_snapshot_identifier: If skip_final_snapshot is False, specify an identifier for the final DB snapshot. - delete_automated_backups: If True, delete automated backups along with the DBInstance. Defaults to False. - """ - self.service.delete(db_instance_identifier,skip_final_snapshot,final_db_snapshot_identifier,delete_automated_backups) - - def modify_db_instance(self, **kwargs): - """ - Modify an existing DBInstance by passing updated attributes. - - Params: kwargs: The attributes to modify for the DBInstance. - Return: A dictionary containing the modified DBInstance. - """ - return self.service.modify(**kwargs) - - def describe_db_instance(self, db_instance_identifier: str): - """ - Retrieve details of a DBInstance by its identifier. - - Params: db_instance_identifier: The primary key (ID) of the DBInstance to describe. - Return: A dictionary containing the details of the DBInstance. - """ - return self.service.describe(db_instance_identifier) diff --git a/DB/NEW_KT_DB/Controller/DBSubnetGroupController.py b/DB/NEW_KT_DB/Controller/DBSubnetGroupController.py deleted file mode 100644 index 027dbcc9..00000000 --- a/DB/NEW_KT_DB/Controller/DBSubnetGroupController.py +++ /dev/null @@ -1,24 +0,0 @@ -from Service.Classes import DBSubnetGroupService - -class DBSubnetGroupController: - def __init__(self, service: DBSubnetGroupService): - self.service = service - - - def create_db_subnet_group(self, **kwargs): - self.service.create_db_subnet_group(**kwargs) - - - def delete_db_subnet_group(self, name): - self.service.delete_db_subnet_group(name) - - - def modify_db_subnet_group(self, name, updates): - self.service.modify_db_subnet_group(name, updates) - - def get_db_subnet_group(self, name): - return self.service.get_db_subnet_group(name) - - def describe_db_subnet_group(self, name): - return self.service.describe_db_subnet_group(name) - \ No newline at end of file diff --git a/DB/NEW_KT_DB/DataAccess/DBClusterManager.py b/DB/NEW_KT_DB/DataAccess/DBClusterManager.py index 7af84f8b..bf20a262 100644 --- a/DB/NEW_KT_DB/DataAccess/DBClusterManager.py +++ b/DB/NEW_KT_DB/DataAccess/DBClusterManager.py @@ -62,4 +62,21 @@ def get(self, cluster_id: str): data_mapping[key] = value return Cluster(**data_mapping) else: - raise ValueError(f"subnet group with name '{cluster_id}' not found") \ No newline at end of file + None + + def is_db_instance_exist(self, db_cluster_identifier: int) -> bool: + """ + Check if a DBInstance with the given identifier exists in memory. + + Params: db_instance_identifier: The primary key (ID) of the DBInstance to check. + + Return: True if the DBInstance exists, otherwise False. + """ + # Check if the object exists by its primary key in the management table + return bool(self.object_manager.db_manager.is_object_exist( + self.object_manager._convert_object_name_to_management_table_name(self.object_name), + criteria=f"{self.pk_column} = '{db_cluster_identifier}'" + )) + + def get_all_clusters(self): + return self.object_manager.get_all_objects_from_memory(self.object_name) \ No newline at end of file diff --git a/DB/NEW_KT_DB/DataAccess/DBInstanceManager.py b/DB/NEW_KT_DB/DataAccess/DBInstanceManager.py deleted file mode 100644 index 42b0d7c9..00000000 --- a/DB/NEW_KT_DB/DataAccess/DBInstanceManager.py +++ /dev/null @@ -1,76 +0,0 @@ -import os -import sys -from typing import Dict, Any, List -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -from DataAccess.ObjectManager import ObjectManager -from Models.DBInstanceModel import DBInstance - -class DBInstanceManager: - def __init__(self, object_manager: ObjectManager): - self.object_manager = object_manager - # Create the management table for DBInstance using its object name and table structure - self.object_manager.create_management_table(DBInstance.object_name, DBInstance.table_structure) - - def createInMemoryDBInstance(self, db_instance: DBInstance): - """ - Create a new DBInstance in memory. - - Params db_instance: DBInstance object to be saved in memory. - """ - self.object_manager.save_in_memory(DBInstance.object_name, db_instance.to_sql()) - - def deleteInMemoryDBInstance(self, db_instance_identifier: str): - """ - Delete a DBInstance from memory by its identifier. - - Params: db_instance_identifier: The primary key (ID) of the DBInstance to delete. - """ - self.object_manager.delete_from_memory_by_pk( - pk_column=DBInstance.pk_column, - pk_value=db_instance_identifier, - object_name=DBInstance.object_name - ) - - def describeDBInstance(self, db_instance_identifier: str): - """ - Retrieve the details of a DBInstance based on its identifier. - - Params: db_instance_identifier: The primary key (ID) of the DBInstance to describe. - - Return: List of DBInstance attributes matching the criteria. - """ - # Fetch DBInstance from memory using its primary key - return self.object_manager.get_from_memory( - criteria=f"{DBInstance.pk_column} = '{db_instance_identifier}'", - object_name=DBInstance.object_name, - columns='*' - ) - - def modifyDBInstance(self, db_instance_identifier: str, updates: str): - """ - Modify the attributes of an existing DBInstance in memory. - - Params: db_instance_identifier: The primary key (ID) of the DBInstance to modify. - updates: SQL-like string containing the updates to be applied (e.g., "port = '3306'"). - """ - # Update the instance's attributes based on the provided update string - self.object_manager.update_in_memory( - criteria=f"{DBInstance.pk_column} = '{db_instance_identifier}'", - object_name=DBInstance.object_name, - updates=updates - ) - - def is_db_instance_exist(self, db_instance_identifier: int) -> bool: - """ - Check if a DBInstance with the given identifier exists in memory. - - Params: db_instance_identifier: The primary key (ID) of the DBInstance to check. - - Return: True if the DBInstance exists, otherwise False. - """ - # Check if the object exists by its primary key in the management table - return bool(self.object_manager.db_manager.is_object_exist( - self.object_manager._convert_object_name_to_management_table_name(DBInstance.object_name), - criteria=f"{DBInstance.pk_column} = '{db_instance_identifier}'" - )) diff --git a/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py b/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py deleted file mode 100644 index 68c8af52..00000000 --- a/DB/NEW_KT_DB/DataAccess/DBSubnetGroupManager.py +++ /dev/null @@ -1,38 +0,0 @@ -from typing import Dict, Any, List - -import sys -import os -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - -from Models.DBSubnetGroupModel import DBSubnetGroup - -class DBSubnetGroupManager: - def __init__(self, object_manager): - self.object_manager = object_manager - self.object_manager._create_management_table(DBSubnetGroup.table_name, DBSubnetGroup.table_structure) - - def create(self, subnet_group: DBSubnetGroup): - self.object_manager.save_in_memory(DBSubnetGroup.table_name, subnet_group) - - def get(self, name: str): - data = self.object_manager.get_from_memory_by_id(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, name) - if data: - data_mapping = {'db_subnet_group_name':name} - for key, value in data[name].items(): - data_mapping[key] = value - return DBSubnetGroup(**data_mapping) - else: - raise ValueError(f"subnet group with name '{name}' not found") - - - def delete(self, name: str): - self.object_manager.delete_from_memory_by_id(DBSubnetGroup.pk_column, name, DBSubnetGroup.table_name) - - def describe(self, name: str): - return self.get(name).to_dict() - - def modify(self, subnet_group: DBSubnetGroup): - updates = subnet_group.to_dict() - del updates['db_subnet_group_name'] - self.object_manager.update_in_memory_by_id(DBSubnetGroup.pk_column, DBSubnetGroup.table_name, updates, subnet_group.db_subnet_group_name) diff --git a/DB/NEW_KT_DB/Models/DBInstanceModel.py b/DB/NEW_KT_DB/Models/DBInstanceModel.py deleted file mode 100644 index 75a155ed..00000000 --- a/DB/NEW_KT_DB/Models/DBInstanceModel.py +++ /dev/null @@ -1,59 +0,0 @@ -import json -from datetime import datetime -import os -import sys -from DataAccess.ObjectManager import ObjectManager -sys.path.insert(0, os.path.abspath( - os.path.join(os.path.dirname(__file__), '../../../'))) -from Storage.NEW_KT_Storage.DataAccess.StorageManager import StorageManager - - -class DBInstance: - BASE_PATH = "db_instances" - object_name = 'db_instance' - pk_column = 'db_instance_id' - pk_column_data_type = 'TEXT' - table_structure = 'db_instance_id VARCHAR(255) PRIMARY KEY NOT NULL, allocated_storage INT NOT NULL, master_username VARCHAR(255) NOT NULL, master_user_password VARCHAR(255) NOT NULL, db_name VARCHAR(255) NOT NULL, port INT NOT NULL, status VARCHAR(50) NOT NULL, created_time DATETIME NOT NULL, endpoint VARCHAR(255) NOT NULL, databases TEXT NOT NULL, pk_value VARCHAR(255) NOT NULL' - - def __init__(self, **kwargs): - """ - Initialize a new DBInstance with the given parameters. - """ - self.db_instance_identifier = kwargs['db_instance_identifier'] - self.allocated_storage = kwargs.get('allocated_storage', 20) - self.master_username = kwargs['master_username'] - self.master_user_password = kwargs['master_user_password'] - self.db_name = kwargs.get('db_name', None) - self.port = kwargs.get('port', 3306) - self.status = 'available' - self.created_time = datetime.now() - self.endpoint = self.db_instance_identifier - storageManager = StorageManager(DBInstance.BASE_PATH) - storageManager.create_directory(self.endpoint) - self.databases = kwargs.get('databases', {}) - self.pk_value = kwargs.get('pk_value', self.db_instance_identifier) - - def to_dict(self): - """Retrieve the metadata of the DB instance as a dictionary.""" - return ObjectManager.convert_object_attributes_to_dictionary( - db_instance_identifier=self.db_instance_identifier, - allocated_storage=self.allocated_storage, - master_username=self.master_username, - master_user_password=self.master_user_password, - db_name=self.db_name, - port=self.port, - status=self.status, - created_time=str(self.created_time), - endpoint=self.endpoint, - databases=self.databases, - pk_value=self.pk_value - - ) - - def to_sql(self): - # Convert the model instance to a dictionary - data_dict = self.to_dict() - values = '(' + ", ".join(f'\'{json.dumps(v)}\'' if isinstance(v, dict) or isinstance(v, list) else f'\'{v}\'' if isinstance(v, str) else f'\'{str(v)}\'' - for v in data_dict.values()) + ')' - # values='(\''+self.db_instance_identifier+'\',\''+json.dumps(self.to_dict())+'\')' - return values diff --git a/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py b/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py deleted file mode 100644 index 7ed37365..00000000 --- a/DB/NEW_KT_DB/Models/DBSubnetGroupModel.py +++ /dev/null @@ -1,71 +0,0 @@ -from typing import List, Dict, Any -import ast -import json -import sys -import os -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - -from DataAccess.ObjectManager import ObjectManager - -class DBSubnetGroup: - - pk_column = 'db_subnet_group_name' - table_name = 'db_subnet_groups' - table_structure = f""" - db_subnet_group_name primary key not null, - db_subnet_group_description TEXT NOT NULL, - vpc_id VARCHAR(255) NOT NULL, - subnets JSONB DEFAULT '{{}}', - db_subnet_group_arn VARCHAR(255), - status VARCHAR(50) DEFAULT 'pending' - """ - - def __init__(self, **kwargs): - try: - print(kwargs) - self.db_subnet_group_name = kwargs['db_subnet_group_name'] - self.db_subnet_group_description = kwargs['db_subnet_group_description'] - self.vpc_id = kwargs['vpc_id'] - self.subnets = kwargs.get('subnets', None) - if not self.subnets: - self.subnets = [] - if type(self.subnets) is not list: - self.subnets = ast.literal_eval(self.subnets) - self.db_subnet_group_arn = kwargs.get('db_subnet_group_arn', None) - - except KeyError as e: - raise ValueError(f"Missing required attribute for DBSubnetGroup: {str(e)}") - - # Ideally: - # self.db_subnet_group_arn should be dynamically created according to vpc-id, account-id and - # subnet-group-name, and then dynamically added to the routing table - - self.status = 'pending' - self.pk_value = self.db_subnet_group_name - - def to_dict(self) -> Dict[str, Any]: - return ObjectManager.convert_object_attributes_to_dictionary( - db_subnet_group_name=self.db_subnet_group_name, - db_subnet_group_description = self.db_subnet_group_description, - vpc_id = self.vpc_id, - subnets = self.subnets, - db_subnet_group_arn = self.db_subnet_group_arn, - status = self.status - ) - - def to_bytes(self): - bytes = json.dumps(self.to_dict()) - bytes = bytes.encode('utf-8') - return bytes - - def from_bytes_to_dict(bytes): - return json.loads(bytes.decode('utf-8')) - - - def to_sql(self): - # Convert the model instance to a dictionary - data_dict = self.to_dict() - values = '(' + ", ".join(f'\'{json.dumps(v)}\'' if isinstance(v, dict) or isinstance(v, list) else f'\'{v}\'' if isinstance(v, str) else f'\'{str(v)}\'' - for v in data_dict.values()) + ')' - return values \ No newline at end of file diff --git a/DB/NEW_KT_DB/Service/Classes/DBClusterService.py b/DB/NEW_KT_DB/Service/Classes/DBClusterService.py index 9086ab25..2fc08c7c 100644 --- a/DB/NEW_KT_DB/Service/Classes/DBClusterService.py +++ b/DB/NEW_KT_DB/Service/Classes/DBClusterService.py @@ -62,8 +62,8 @@ def create(self, instance_controller, **kwargs): raise ValueError("Missing required parameters") # Perform validations - # if self.dal.is_exists(kwargs.get('db_cluster_identifier')): - # raise ValueError(f"Cluster {kwargs.get('db_cluster_identifier')} already exists") + if self.dal.is_db_instance_exist(kwargs.get('db_cluster_identifier')): + raise ValueError(f"Cluster {kwargs.get('db_cluster_identifier')} already exists") if not validate_db_cluster_identifier(kwargs.get('db_cluster_identifier')): raise ValueError(f"Invalid DBClusterIdentifier: {kwargs.get('db_cluster_identifier')}") @@ -128,8 +128,8 @@ def create(self, instance_controller, **kwargs): def delete(self,instance_controller, cluster_identifier:str): '''Delete an existing DBCluster.''' - # if not self.is_cluster_exist(cluster_identifier): - # raise ValueError("Cluster does not exist!!") + if not self.dal.is_db_instance_exist(cluster_identifier): + raise ValueError("Cluster does not exist!!") file_path = self.get_file_path(cluster_identifier+"_configurations") self.storage_manager.delete_file(file_path=file_path) @@ -148,10 +148,10 @@ def delete(self,instance_controller, cluster_identifier:str): #update configurations current_cluster = self.describe(cluster_identifier) cluster_dict = dict(zip(columns, current_cluster[0])) - instance_controller.delete_db_instance(db_instance_identifier = cluster_dict['primary_writer_instance'],final_db_snapshot_identifier = True) - if cluster_dict['reader_instances'] != [] : + instance_controller.delete_db_instance(db_instance_identifier = cluster_dict['primary_writer_instance'],skip_final_snapshot = True) + if cluster_dict['reader_instances'] != '[]' : for id in cluster_dict['reader_instances']: - instance_controller.delete_db_instance(db_instance_identifier = id, final_db_snapshot_identifier = True) + instance_controller.delete_db_instance(db_instance_identifier = id, skip_final_snapshot = True) self.dal.deleteInMemoryDBCluster(cluster_identifier) @@ -159,8 +159,8 @@ def delete(self,instance_controller, cluster_identifier:str): def describe(self, cluster_id): '''Describe the details of DBCluster.''' - # if not self.is_cluster_exist(cluster_id): - # raise ValueError("Cluster does not exist!!") + if not self.dal.is_db_instance_exist(cluster_id): + raise ValueError("Cluster does not exist!!") return self.dal.describeDBCluster(cluster_id) @@ -168,8 +168,8 @@ def describe(self, cluster_id): def modify(self, cluster_id: str, **kwargs): '''Modify an existing DBCluster.''' - # if not self.is_cluster_exist(cluster_id): - # raise ValueError("Cluster does not exist!!") + if not self.dal.is_db_instance_exist(cluster_id): + raise ValueError("Cluster does not exist!!") if 'db_cluster_identifier' in kwargs and not validate_db_cluster_identifier(kwargs.get('db_cluster_identifier')): raise ValueError(f"Invalid DBClusterIdentifier: {kwargs.get('db_cluster_identifier')}") @@ -218,5 +218,10 @@ def modify(self, cluster_id: str, **kwargs): self.storage_manager.create_file(file_path, cluster_string) + def get_all_cluster(self): + return self.dal.get_all_clusters() + + + diff --git a/DB/NEW_KT_DB/Service/Classes/DBInstanceService.py b/DB/NEW_KT_DB/Service/Classes/DBInstanceService.py deleted file mode 100644 index 125126d7..00000000 --- a/DB/NEW_KT_DB/Service/Classes/DBInstanceService.py +++ /dev/null @@ -1,167 +0,0 @@ -import os -import shutil -import sys -from typing import Dict, Optional -from Exception.exception import DBInstanceNotFoundError, ParamValidationError -from Validation.DBInstanceValidition import check_extra_params, check_required_params, is_valid_db_instance_identifier -from Models.DBInstanceModel import DBInstance -from Service.Abc.DBO import DBO -from DataAccess.DBInstanceManager import DBInstanceManager -from Exception.exception import AlreadyExistsError -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../'))) -from Storage.NEW_KT_Storage.DataAccess.StorageManager import StorageManager - - -class DBInstanceService(DBO): - - def __init__(self, dal: DBInstanceManager): - self.dal = dal - - def create(self, **attributes): - """ - Create a new DBInstance. - - Params: attributes: dict containing attributes for the new DBInstance. - Required fields: 'db_instance_identifier', 'master_username', 'master_user_password' - Optional fields: 'db_name', 'port', 'allocated_storage' - - Raises: ValueError: if the db_instance_identifier is invalid. - AlreadyExistsError: if a DB instance with the given identifier already exists. - - Return: dict with a 'DBInstance' key containing the created instance's details. - """ - required_params = ['db_instance_identifier', 'master_username', 'master_user_password'] - all_params = ['db_name', 'port', 'allocated_storage'] - all_params.extend(required_params) - - # Validate required and extra parameters - check_required_params(required_params, attributes) - check_extra_params(all_params, attributes) - - db_instance_identifier = attributes['db_instance_identifier'] - - # Validate DB instance identifier format - if not is_valid_db_instance_identifier(db_instance_identifier, 63): - raise ValueError('db_instance_identifier is invalid') - - if self.dal.is_db_instance_exist(db_instance_identifier): - raise AlreadyExistsError(f"The ID {db_instance_identifier} already exists") - - db_instance = DBInstance(**attributes) - self.dal.createInMemoryDBInstance(db_instance) - - return {'DBInstance': db_instance.to_dict()} - - def delete(self, db_instance_identifier,skip_final_snapshot=False,final_db_snapshot_identifier=None,delete_automated_backups=False): - """ - Delete a DBInstance by its identifier with options to handle final snapshots and automated backups. - - Params: db_instance_identifier: The primary key (ID) of the DBInstance to delete. - skip_final_snapshot: If True, skip creating a final snapshot before deletion. Defaults to False. - final_db_snapshot_identifier: If skip_final_snapshot is False, specify an identifier for the final DB snapshot. - delete_automated_backups: If True, delete automated backups along with the DBInstance. Defaults to False. - """ - if not self.dal.is_db_instance_exist(db_instance_identifier): - raise DBInstanceNotFoundError(db_instance_identifier,'This DB instance identifier does not exist') - - # Get the DBInstance to delete - object - db_instance = self.get(db_instance_identifier) - - # Handle final snapshot before deletion if required - # print(skip_final_snapshot) - if skip_final_snapshot == False: - if final_db_snapshot_identifier is None: - raise ParamValidationError('If skip_final_snapshot is False, final_db_snapshot_identifier must be specified') - create_db_snapshot( - db_instance_identifier=db_instance_identifier, - db_snapshot_identifier=final_db_snapshot_identifier - ) - - self.dal.deleteInMemoryDBInstance(db_instance_identifier) - - # Clean up storage (delete associated directory) - endpoint = db_instance.endpoint - storageManager = StorageManager(DBInstance.BASE_PATH) - storageManager.delete_directory(endpoint) - - # Delete the DBInstance object - del db_instance - - def describe(self, db_instance_identifier): - """ - Describe the details of a DBInstance. - - Params: db_instance_identifier: str, identifier of the DBInstance to describe. - - Raises: DBInstanceNotFoundError: if the DB instance does not exist. - - Return: dict with a 'DBInstance' key containing the details of the instance. - """ - if not self.dal.is_db_instance_exist(db_instance_identifier): - raise DBInstanceNotFoundError('This DB instance identifier does not exist') - - # Retrieve and structure the DBInstance description - describe_db_instance = self.dal.describeDBInstance(db_instance_identifier)[0] - describe_db_instance_dict = { - 'db_instance_identifier': describe_db_instance[0], - 'allocated_storage': describe_db_instance[1], - 'master_username': describe_db_instance[2], - 'master_user_password': describe_db_instance[3], - 'db_name': describe_db_instance[4], - 'port': describe_db_instance[5], - 'status': describe_db_instance[6], - 'created_time': str(describe_db_instance[7]), - 'endpoint': describe_db_instance[8], - 'databases': describe_db_instance[9], - 'pk_value': describe_db_instance[10] - } - - return {'DBInstance': describe_db_instance_dict} - - def modify(self, **updates): - """ - Modify an existing DBInstance. - - Params: updates: dict containing the attributes to modify in the DBInstance. - Required field: 'db_instance_identifier' - Optional fields: 'port', 'allocated_storage', 'master_user_password' - - Raises: DBInstanceNotFoundError: if the DB instance does not exist. - - Return: dict with a 'DBInstance' key containing the updated instance's details. - """ - required_params = ['db_instance_identifier'] - all_params = ['port', 'allocated_storage', 'master_user_password'] - all_params.extend(required_params) - - # Validate required and extra parameters - check_required_params(required_params, updates) - check_extra_params(all_params, updates) - - db_instance_identifier = updates['db_instance_identifier'] - - # Prepare the SQL-like set clause for updates - filtered_updates = {key: value for key, value in updates.items() if key != 'db_instance_identifier'} - set_clause = ', '.join([f"{key} = '{value}'" for key, value in filtered_updates.items()]) - - self.dal.modifyDBInstance(db_instance_identifier, set_clause) - - # Return the updated DB instance details - update_db_instance = self.get(db_instance_identifier) - return {'DBInstance': update_db_instance} - - def get(self, db_instance_identifier): - """ - Retrieve a DBInstance object from the database. - - Params: db_instance_identifier: str, identifier of the DBInstance to retrieve. - - Return: DBInstance object or None if not found. - """ - describe_result = self.describe(db_instance_identifier) - - if describe_result: - describe_result = describe_result['DBInstance'] - return DBInstance(**describe_result) - - return None diff --git a/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py b/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py deleted file mode 100644 index 8fc33513..00000000 --- a/DB/NEW_KT_DB/Service/Classes/DBSubnetGroupService.py +++ /dev/null @@ -1,91 +0,0 @@ -from sqlite3 import IntegrityError -from typing import List, Dict, Any - -import sys -import os -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..'))) -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) - - -from Models.DBSubnetGroupModel import DBSubnetGroup -from DataAccess.DBSubnetGroupManager import DBSubnetGroupManager -from Storage.KT_Storage.DataAccess.StorageManager import StorageManager -from Storage.KT_Storage.DataAccess.VersionManager import VersionManager -from Validation.GeneralValidations import * -class DBSubnetGroupService: - def __init__(self, db_subnet_group_manager: DBSubnetGroupManager): - self.manager = db_subnet_group_manager - self.bucket = 'db_subnet_groups' - self.storage_manager = StorageManager() - self.storage_manager.create_bucket(self.bucket) - self.version_manager = VersionManager() - self.subnet_groups = dict() - - def create_db_subnet_group(self, **kwargs): - # object - if not kwargs.get('db_subnet_group_name'): - raise ValueError('Missing required argument db_subnet_group_name') - - if not is_length_in_range(kwargs['db_subnet_group_name'], 1, 255): - raise ValueError("invalid length for subnet group db_subnet_group_name: " + len(kwargs['db_subnet_group_name'])) - - if kwargs['db_subnet_group_name'] in self.subnet_groups: - raise ValueError(f"db_subnet_group_name {kwargs['db_subnet_group_name']} already exists") - - if kwargs.get('description') and not is_length_in_range('description', 1, 255): - raise ValueError("invalid length for subnet group description: " + len(kwargs['description'])) - - subnet_group = DBSubnetGroup(**kwargs) - # save in management table - # in try except block in case the server was shut down and re-run and local collection doesn't include all subnetGroups - try: - self.manager.create(subnet_group) - except IntegrityError as e: - raise ValueError(f"db_subnet_group_name {kwargs['db_subnet_group_name']} already exists") - - # physical object - # version = 0 assume created for the first time - self.storage_manager.create(self.bucket, subnet_group.db_subnet_group_name, subnet_group.to_bytes(), '0') - # save in local collection (hash table) for quick access - self.subnet_groups[kwargs['db_subnet_group_name']] = subnet_group - - def get_db_subnet_group(self, db_subnet_group_name: str) -> DBSubnetGroup: - data = self.manager.get(db_subnet_group_name) - return data - - def modify_db_subnet_group(self, db_subnet_group_name: str, updates: Dict[str, Any]) -> DBSubnetGroup: - if not db_subnet_group_name: - raise ValueError('Missing required argument db_subnet_group_name') - - if updates.get('description') and not is_length_in_range(updates['description'], 1, 255): - raise ValueError("invalid length for subnet group description: " + len(updates['description'])) - - subnet_group = self.get_db_subnet_group(db_subnet_group_name) - - for key, value in updates.items(): - setattr(subnet_group, key, value) - - self.manager.modify(subnet_group) - # version = str(int(self.version_manager.get(self.bucket, subnet_group.db_subnet_group_name).version_id)+1) - # for now we override the basic version, when the latest version id can be retrieved, we will make a new version as old_version_id + 1 - self.storage_manager.create(self.bucket, db_subnet_group_name, subnet_group.to_bytes(), '0') - - def delete_db_subnet_group(self, db_subnet_group_name: str) -> None: - if not db_subnet_group_name: - raise ValueError('Missing required argument db_subnet_group_name') - - # delete from management table - self.manager.delete(db_subnet_group_name) - # for now version id is 0 - # delete physical object from storage - self.storage_manager.delete_by_name(bucket_name=self.bucket, version_id='0', key=db_subnet_group_name) - # delete from local collection (hash table) - if db_subnet_group_name in self.subnet_groups: - del self.subnet_groups[db_subnet_group_name] - - def describe_db_subnet_group(self, db_subnet_group_name: str) -> Dict: - if not db_subnet_group_name: - raise ValueError('Missing required argument db_subnet_group_name') - - return self.manager.describe(db_subnet_group_name) \ No newline at end of file diff --git a/DB/NEW_KT_DB/Test/DBClusterTests.py b/DB/NEW_KT_DB/Test/DBClusterTests.py index 5619ccc8..2569cfc9 100644 --- a/DB/NEW_KT_DB/Test/DBClusterTests.py +++ b/DB/NEW_KT_DB/Test/DBClusterTests.py @@ -1,43 +1,216 @@ +import os +import sys import pytest import sqlite3 -import sys -import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..'))) -# Add paths to sys.path -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '.'))) - -from Service.Classes.DBClusterService import DBClusterService -from DataAccess.DBClusterManager import DBClusterManager +from Storage.NEW_KT_Storage.DataAccess.StorageManager import StorageManager +from DB.NEW_KT_DB.Service.Classes.DBClusterService import DBClusterService +from DB.NEW_KT_DB.DataAccess.DBClusterManager import DBClusterManager from Controller.DBClusterController import DBClusterController -from Storage.KT_Storage.DataAccess.StorageManager import StorageManager from DataAccess.ObjectManager import ObjectManager -from Models.DBClusterModel import Cluster - -@pytest.fixture(scope = 'module',autouse=True) -def setup_services(): - - # Initialize the managers, services, controllers, and storage managers - manager = DBClusterManager('Clusters/clusters.db') - service = DBClusterService(manager,storage_manager, 'Clusters') - controller = DBClusterController(service) - storage_manager = StorageManager('Clusters') +from Service.Classes.DBInstanceService import DBInstanceManager,DBInstanceService,AlreadyExistsError,ParamValidationError,DBInstanceNotFoundError +from Exception.exception import MissingRequireParamError +from Controller.DBInstanceController import DBInstanceController + +@pytest.fixture +def object_manager(): + return ObjectManager('Clusters/instances.db') + +@pytest.fixture +def db_instance_manager(object_manager): + return DBInstanceManager(object_manager) + +@pytest.fixture +def db_instance_service(db_instance_manager): + return DBInstanceService(db_instance_manager) + +# @pytest.fixture +# def snapshot_service(object_manager): +# return SnapShotService(SnapShotManager(object_manager)) + +@pytest.fixture +def db_instance_controller(db_instance_service): + return DBInstanceController(db_instance_service) + +@pytest.fixture +def db_cluster_manager(): + return DBClusterManager('Clusters/clusters.db') + +@pytest.fixture +def storage_manager(): + return StorageManager('Clusters') + +@pytest.fixture +def db_cluster_service(db_cluster_manager,storage_manager): + return DBClusterService(db_cluster_manager,storage_manager, 'Clusters') + +@pytest.fixture +def db_cluster_controller(db_cluster_service, db_instance_controller): + return DBClusterController(db_cluster_service, db_instance_controller) + +@pytest.fixture +def db_cluster_controller_with_cleanup(db_cluster_controller): + + # Return the controller, but ensure cleanup happens after the test + yield db_cluster_controller - # Provide the variables as a dictionary or as individual items - return { - 'manager': manager, - 'service': service, - 'controller': controller, - 'storage_manager': storage_manager + # Finalizer to clean up after the test + db_cluster_controller.delete_db_cluster('ClusterTest') + +def test_create_cluster_works(db_cluster_controller_with_cleanup): + cluster_data = { + 'db_cluster_identifier': 'ClusterTest', + 'engine': 'mysql', + 'allocated_storage':5, + 'db_subnet_group_name': 'my-subnet-group' + } + + res = db_cluster_controller_with_cleanup.create_db_cluster(**cluster_data) + # db_cluster_controller.delete_db_cluster('Cluster21') + assert res == None + +def test_create_cluster_missing_required_fields(db_cluster_controller): + cluster_data = { + 'db_cluster_identifier': 'ClusterTest', + 'allocated_storage':5, + 'db_subnet_group_name': 'my-subnet-group' + } + with pytest.raises(ValueError): + db_cluster_controller.create_db_cluster(**cluster_data) + +def test_create_cluster_invalide_identifier(db_cluster_controller): + cluster_data = { + 'db_cluster_identifier': 'Cluster__Test', + 'engine': 'mysql', + 'allocated_storage':5, + 'db_subnet_group_name': 'my-subnet-group' + } + with pytest.raises(ValueError): + db_cluster_controller.create_db_cluster(**cluster_data) + + +def test_create_cluster_already_exist(db_cluster_controller): + cluster_data = { + 'db_cluster_identifier': 'ClusterTest', + 'engine': 'mysql', + 'allocated_storage':5, + 'db_subnet_group_name': 'my-subnet-group' + } + db_cluster_controller.create_db_cluster(**cluster_data) + with pytest.raises(ValueError): + db_cluster_controller.create_db_cluster(**cluster_data) + + db_cluster_controller.delete_db_cluster('ClusterTest') + + + +def test_delete_cluster_works(db_cluster_controller): + cluster_data = { + 'db_cluster_identifier': 'ClusterTest', + 'engine': 'mysql', + 'allocated_storage':5, + 'db_subnet_group_name': 'my-subnet-group' + } + + db_cluster_controller.create_db_cluster(**cluster_data) + db_cluster_controller.delete_db_cluster('ClusterTest') + with pytest.raises(ValueError): + db_cluster_controller.delete_db_cluster('ClusterTest') + + +def test_delete_cluster_does_not_exist(db_cluster_controller): + with pytest.raises(ValueError): + db_cluster_controller.delete_db_cluster('ClusterTest') + + + +def test_describe_cluster_works(db_cluster_controller_with_cleanup): + cluster_data = { + 'db_cluster_identifier': 'ClusterTest', + 'engine': 'mysql', + 'allocated_storage':5, + 'db_subnet_group_name': 'my-subnet-group' + } + + db_cluster_controller_with_cleanup.create_db_cluster(**cluster_data) + res = db_cluster_controller_with_cleanup.describe_db_cluster('ClusterTest') + assert isinstance(res, list) + +def test_describe_cluster_does_not_exist(db_cluster_controller): + with pytest.raises(ValueError): + db_cluster_controller.describe_db_cluster('ClusterTest') + + +def test_modify_cluster_works(db_cluster_controller_with_cleanup): + cluster_data = { + 'db_cluster_identifier': 'ClusterTest', + 'engine': 'mysql', + 'allocated_storage':5, + 'db_subnet_group_name': 'my-subnet-group' + } + + db_cluster_controller_with_cleanup.create_db_cluster(**cluster_data) + update_data = { + 'engine': 'postgres', + } + db_cluster_controller_with_cleanup.modify_db_cluster('ClusterTest', **update_data) + res = db_cluster_controller_with_cleanup.describe_db_cluster('ClusterTest') + columns = ['db_cluster_identifier', 'engine', 'allocated_storage', 'copy_tags_to_snapshot', + 'db_cluster_instance_class', 'database_name', 'db_cluster_parameter_group_name', + 'db_subnet_group_name', 'deletion_protection', 'engine_version', 'master_username', + 'master_user_password', 'manage_master_user_password', 'option_group_name', 'port', + 'replication_source_identifier', 'storage_encrypted', 'storage_type', 'tags', + 'created_at', 'status', 'primary_writer_instance', 'reader_instances', 'cluster_endpoint', + 'instances_endpoints', 'pk_column', 'pk_value'] + + cluster_dict = dict(zip(columns, res[0])) + assert cluster_dict['engine'] == 'postgres' + +def test_modify_cluster_does_not_exist(db_cluster_controller): + update_data = { + 'engine': 'postgres', + } + with pytest.raises(ValueError): + db_cluster_controller.modify_db_cluster('ClusterTest',**update_data) + + +def test_modify_cluster_invalide_engine(db_cluster_controller_with_cleanup): + cluster_data = { + 'db_cluster_identifier': 'ClusterTest', + 'engine': 'mysql', + 'allocated_storage':5, + 'db_subnet_group_name': 'my-subnet-group' + } + + db_cluster_controller_with_cleanup.create_db_cluster(**cluster_data) + update_data = { + 'engine': 'invalid', + } + with pytest.raises(ValueError): + db_cluster_controller_with_cleanup.modify_db_cluster('ClusterTest',**update_data) + +def test_get_all_clusters(db_cluster_controller_with_cleanup): + cluster_data = { + 'db_cluster_identifier': 'ClusterTest', + 'engine': 'mysql', + 'allocated_storage':5, + 'db_subnet_group_name': 'my-subnet-group' } -# def test_create_DBCluster_function(): -# cluster_data = { -# 'db_cluster_identifier': 'myCluster5', -# 'engine': 'mysql', -# 'allocated_storage':5, -# 'db_subnet_group_name': 'my-subnet-group' -# } + db_cluster_controller_with_cleanup.create_db_cluster(**cluster_data) + res = db_cluster_controller_with_cleanup.get_all_db_clusters() + assert isinstance(res, list) + + + + + + + + + + + + - # assert setup_services['controller'].create_db_cluster(**cluster_data) \ No newline at end of file diff --git a/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py b/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py deleted file mode 100644 index b2fe7674..00000000 --- a/DB/NEW_KT_DB/Test/DBSubnetGroupTests.py +++ /dev/null @@ -1,228 +0,0 @@ -from sqlite3 import IntegrityError -import pytest -import json - -import sys -import os -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))) -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) -sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '.'))) - -from Service.Classes.DBSubnetGroupService import DBSubnetGroupService -from DataAccess.DBSubnetGroupManager import DBSubnetGroupManager -from Controller.DBSubnetGroupController import DBSubnetGroupController -from Storage.KT_Storage.DataAccess.StorageManager import StorageManager -from DataAccess.ObjectManager import ObjectManager -from Models.DBSubnetGroupModel import DBSubnetGroup -import sqlite3 - -object_manager = ObjectManager('../object_management_db.db') -manager = DBSubnetGroupManager(object_manager=object_manager) -service = DBSubnetGroupService(manager) -controller = DBSubnetGroupController(service) -storage_manager = StorageManager() - -@pytest.fixture -def clear_table(): - # Connect to the SQLite database - conn = conn = sqlite3.connect('../object_management_db.db') - cursor = conn.cursor() - - # Clear the table if it exists - table_name = "db_subnet_groups" - cursor.execute(f"DELETE FROM {table_name};") - - # Commit changes and close the connection - conn.commit() - conn.close() - - # Yield to allow tests to run - yield - -def test_create(clear_table): - - # remove existing subnet group from previous tests - controller.create_db_subnet_group( - db_subnet_group_name='subnet_group_1', - subnets=[ - {'subnet_id': 'subnet-12345678'}, - {'subnet_id': 'subnet-87654321'} - ], - db_subnet_group_description='Test subnet group', - vpc_id='vpc-12345678', - db_subnet_group_arn='arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' - ) - - # check that file was created (no error raised on get) - storage_manager.get('db_subnet_groups', 'subnet_group_1', '0') - # check that object was saved to management table (no error raised on get) - controller.get_db_subnet_group('subnet_group_1') - - # check that file content is correct - from_storage = DBSubnetGroup(**DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', 'subnet_group_1', '0')['content'])) - from_db = controller.get_db_subnet_group('subnet_group_1') - # check values were stored correctly in management table as well as storage - - # group name - assert from_storage.db_subnet_group_name == 'subnet_group_1' - assert from_db.db_subnet_group_name == 'subnet_group_1' - - # subnets - for subnet in from_storage.subnets: - assert subnet in [ - {'subnet_id': 'subnet-12345678'}, - {'subnet_id': 'subnet-87654321'} - ] - for subnet in from_db.subnets: - assert subnet in [ - {'subnet_id': 'subnet-12345678'}, - {'subnet_id': 'subnet-87654321'} - ] - - # description - assert from_storage.db_subnet_group_description == 'Test subnet group' - assert from_db.db_subnet_group_description == 'Test subnet group' - - # vpc_id - assert from_storage.vpc_id == 'vpc-12345678' - assert from_db.vpc_id == 'vpc-12345678' - - # arn - assert from_storage.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' - assert from_db.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' - - -def test_unique_constraint(): - with pytest.raises(ValueError): - controller.create_db_subnet_group( - db_subnet_group_name='subnet_group_1', - subnets=[ - {'subnet_id': 'subnet-87654321'}, - {'subnet_id': 'subnet-12345678'} - ], - db_subnet_group_description='Another subnet group with same name', - vpc_id='vpc-87654321', - db_subnet_group_arn='arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' - ) - - -def test_get(): - subnet_group_1 = controller.get_db_subnet_group('subnet_group_1') - assert subnet_group_1 != None - assert subnet_group_1.db_subnet_group_name == 'subnet_group_1' - assert subnet_group_1.db_subnet_group_description == 'Test subnet group' - assert subnet_group_1.vpc_id == 'vpc-12345678' - assert subnet_group_1.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' - assert subnet_group_1.status == 'pending' - for subnet in subnet_group_1.subnets: - assert subnet in [ - {"subnet_id": "subnet-12345678"}, - {"subnet_id": "subnet-87654321"} - ] - -def test_modify(): - controller.modify_db_subnet_group( - name='subnet_group_1', - updates= {'subnets':[ - {'subnet_id': 'subnet-12345988'}, - {'subnet_id': 'subnet-876543881'}]} - ) - - from_storage = DBSubnetGroup(**DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', 'subnet_group_1', '0')['content'])) - from_db = controller.get_db_subnet_group('subnet_group_1') - assert from_storage.db_subnet_group_name == from_db.db_subnet_group_name - for subnet in from_storage.subnets: - assert subnet in from_db.subnets - assert from_storage.db_subnet_group_description == from_db.db_subnet_group_description - assert from_storage.vpc_id == from_db.vpc_id - assert from_storage.db_subnet_group_arn == from_db.db_subnet_group_arn - assert from_storage.status == from_db.status - -def test_describe(): - subnet_group_1 = controller.describe_db_subnet_group('subnet_group_1') - assert type(subnet_group_1) == dict - assert type(subnet_group_1['subnets']) == list - assert type(subnet_group_1['subnets'][0]) == dict - subnet_group_1 = DBSubnetGroup(**subnet_group_1) - assert subnet_group_1 != None - assert subnet_group_1.db_subnet_group_name == 'subnet_group_1' - assert subnet_group_1.db_subnet_group_description == 'Test subnet group' - assert subnet_group_1.vpc_id == 'vpc-12345678' - assert subnet_group_1.db_subnet_group_arn == 'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_1' - assert subnet_group_1.status == 'pending' - for subnet in subnet_group_1.subnets: - assert subnet in [ - {'subnet_id': 'subnet-12345988'}, - {'subnet_id': 'subnet-876543881'} - ] - - -def test_delete(): - controller.delete_db_subnet_group('subnet_group_1') - with pytest.raises(FileNotFoundError): - storage_manager.get('db_subnet_groups', 'subnet_group_1', '0') - with pytest.raises(Exception): - controller.get_db_subnet_group('subnet_group_1') - - -@pytest.mark.parametrize("index", range(20)) -def test_insert_many(index): - controller.create_db_subnet_group( - db_subnet_group_name=f'subnet_group_{index}', - subnets=[ - {'subnet_id': f'subnet-1234567{index}'}, - {'subnet_id': f'subnet-8765432{index}'} - ], - db_subnet_group_description=f'Test subnet group {index}', - vpc_id=f'vpc-1234567{index}', - db_subnet_group_arn=f'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_{index}' - ) - - # check that file was created (no error raised on get) - storage_manager.get('db_subnet_groups', f'subnet_group_{index}', '0') - # check that object was saved to management table (no error raised on get) - controller.get_db_subnet_group(f'subnet_group_{index}') - - # check that file content is correct - from_storage = DBSubnetGroup(**DBSubnetGroup.from_bytes_to_dict(storage_manager.get('db_subnet_groups', f'subnet_group_{index}', '0')['content'])) - from_db = controller.get_db_subnet_group(f'subnet_group_{index}') - # check values were stored correctly in management table as well as storage - - # group name - assert from_storage.db_subnet_group_name == f'subnet_group_{index}' - assert from_db.db_subnet_group_name == f'subnet_group_{index}' - - # subnets - for subnet in from_storage.subnets: - assert subnet in [ - {'subnet_id': f'subnet-1234567{index}'}, - {'subnet_id': f'subnet-8765432{index}'} - ] - for subnet in from_db.subnets: - assert subnet in [ - {'subnet_id': f'subnet-1234567{index}'}, - {'subnet_id': f'subnet-8765432{index}'} - ] - - # description - assert from_storage.db_subnet_group_description == f'Test subnet group {index}' - assert from_db.db_subnet_group_description == f'Test subnet group {index}' - - # vpc_id - assert from_storage.vpc_id == f'vpc-1234567{index}' - assert from_db.vpc_id == f'vpc-1234567{index}' - - # arn - assert from_storage.db_subnet_group_arn == f'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_{index}' - assert from_db.db_subnet_group_arn == f'arn:aws:rds:us-west-2:123456789012:subgrp:subnet_group_{index}' - - - -@pytest.mark.parametrize("index", range(20)) -def test_delete_many_from_prev_test(index): - db_subnet_group_name=f'subnet_group_{index}' - controller.delete_db_subnet_group(db_subnet_group_name) - with pytest.raises(FileNotFoundError): - storage_manager.get('db_subnet_groups', db_subnet_group_name, '0') - with pytest.raises(Exception): - controller.get_db_subnet_group(db_subnet_group_name) diff --git a/DB/NEW_KT_DB/Validation/DBInstanceValidition.py b/DB/NEW_KT_DB/Validation/DBInstanceValidition.py deleted file mode 100644 index ff0d4393..00000000 --- a/DB/NEW_KT_DB/Validation/DBInstanceValidition.py +++ /dev/null @@ -1,22 +0,0 @@ -import re - -from Exception.exception import ParamValidationError, MissingRequireParamError - - -def check_required_params(required_params, kwargs): - """Check if all required parameters are present in kwargs.""" - for param in required_params: - if param not in kwargs: - raise MissingRequireParamError(f"Missing required parameter in input: {param}") - - -def check_extra_params(all_params, kwargs): - string_all_params = ", ".join(all_params) - for param in kwargs: - if param not in all_params: - raise ParamValidationError(f"Unknown parameter in input: {param}, must be one of: {string_all_params}") - - -def is_valid_db_instance_identifier(identifier, length): - pattern = r'^[a-zA-Z][a-zA-Z0-9-]*[a-zA-Z0-9]$' - return ((1 <= len(identifier) <= length) and re.match(pattern, identifier) and '--' not in identifier) \ No newline at end of file From 6ecb06fe51ed568040a1d127602ea3a21540da2c Mon Sep 17 00:00:00 2001 From: sara-lea Date: Wed, 18 Sep 2024 22:28:38 +0300 Subject: [PATCH 15/16] open pull request --- DB/NEW_KT_DB/Test/DBClusterTests.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DB/NEW_KT_DB/Test/DBClusterTests.py b/DB/NEW_KT_DB/Test/DBClusterTests.py index 2569cfc9..c2c15ebc 100644 --- a/DB/NEW_KT_DB/Test/DBClusterTests.py +++ b/DB/NEW_KT_DB/Test/DBClusterTests.py @@ -203,6 +203,8 @@ def test_get_all_clusters(db_cluster_controller_with_cleanup): assert isinstance(res, list) +zxzxzx + From d573fa07b767940e4901272193624f12b0d3b136 Mon Sep 17 00:00:00 2001 From: sara-lea Date: Wed, 18 Sep 2024 22:28:45 +0300 Subject: [PATCH 16/16] open pull request --- DB/NEW_KT_DB/Test/DBClusterTests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DB/NEW_KT_DB/Test/DBClusterTests.py b/DB/NEW_KT_DB/Test/DBClusterTests.py index c2c15ebc..0f4b9635 100644 --- a/DB/NEW_KT_DB/Test/DBClusterTests.py +++ b/DB/NEW_KT_DB/Test/DBClusterTests.py @@ -203,7 +203,7 @@ def test_get_all_clusters(db_cluster_controller_with_cleanup): assert isinstance(res, list) -zxzxzx +