diff --git a/cterasdk/core/cloudfs.py b/cterasdk/core/cloudfs.py index 96d03fb7..7e47905c 100644 --- a/cterasdk/core/cloudfs.py +++ b/cterasdk/core/cloudfs.py @@ -5,7 +5,7 @@ from .base_command import BaseCommand from . import query, devices from .enum import ListFilter, PolicyType -from .types import ComplianceSettingsBuilder, ExtendedAttributesBuilder +from .types import ArchiveSettingsBuilder, ComplianceSettingsBuilder, ExtendedAttributesBuilder from ..common import union, Object from ..exceptions import CTERAException, ObjectNotFoundException @@ -151,19 +151,24 @@ def __init__(self, core): def _get_entire_object(self, name, owner): return self._core.api.get(f'{self.find(name, owner, include=["baseObjectRef"]).baseObjectRef}') - def add(self, name, group, owner, winacls=True, description=None, # pylint: disable=too-many-arguments - quota=None, compliance_settings=None, xattrs=None, gfl=False, lock_extensions=None): + def add(self, name, group=None, owner=None, winacls=True, description=None, # pylint: disable=too-many-arguments, too-many-locals + quota=None, archive_settings=None, compliance_settings=None, native_format_settings=None, + xattrs=None, gfl=False, lock_extensions=None): """ Create a new Cloud Drive Folder (Cloud Volume) - :param str name: Name of the new folder - :param str group: Folder Group to assign this folder to + :param str name: Name of the new cloud folder + :param str group,optional: Folder Group to assign this folder to :param cterasdk.core.types.UserAccount owner: User account, the owner of the new folder :param bool,optional winacls: Use Windows ACLs, defaults to True :param str,optional description: Cloud drive folder description :param str,optional quota: Cloud drive folder quota in GB + :param cterasdk.common.object.Object,optional archive_settings: Archive settings. + Use :func:`cterasdk.core.types.ArchiveSettingsBuilder` to build the archive settings object :param cterasdk.common.object.Object,optional compliance_settings: Compliance settings, defaults to disabled. Use :func:`cterasdk.core.types.ComplianceSettingsBuilder` to build the compliance settings object + :param cterasdk.common.object.Object,optional native_format_settings: Native format settings, defaults to disabled. + Use :func:`cterasdk.core.types.NativeFormatSettingsBuilder` to build the native format settings object :param cterasdk.common.object.Object,optional xattrs: Extended attributes, defaults to MacOS. Use :func:`cterasdk.core.types.ExtendedAttributesBuilder` to build the extended attributes object :param bool,optional gfl: Enable global file locking @@ -171,15 +176,45 @@ def add(self, name, group, owner, winacls=True, description=None, # pylint: dis :returns: Path to the Cloud Drive folder :rtype: str """ - param = Object() + param = Object(owner=self._core.users.get(owner, ['baseObjectRef']).baseObjectRef) + + self._configure_native_or_encrypted(param, group, native_format_settings) + + CloudDrives._configure_attributes(param, name, winacls, quota, description, xattrs) + + CloudDrives._configure_archive(param, archive_settings) + + CloudDrives._configure_compliance(param, compliance_settings) + + if gfl: + CloudDrives._configure_locking(param, lock_extensions) + + try: + response = self._core.api.execute('', 'addCloudDrive', param) + logger.info('Cloud drive folder created. %s', + {'name': name, 'owner': param.owner, 'folder_group': group, 'winacls': winacls}) + return re.search(r'/Users\/(.+)', response).group() + except CTERAException as error: + logger.error('Cloud drive folder creation failed. %s', + {'name': name, 'folder_group': group, 'owner': str(owner), 'win_acls': winacls}) + raise error + + def _configure_native_or_encrypted(self, param, group, native_format_settings): + if native_format_settings: + param.openStorageEnabled = True + param.openFabricSettings = native_format_settings + param.group = None + else: + param.group = self._core.cloudfs.groups.get(group, ['baseObjectRef']).baseObjectRef + + @staticmethod + def _configure_attributes(param, name, winacls, quota, description, xattrs): param.name = name - param.owner = self._core.users.get(owner, ['baseObjectRef']).baseObjectRef - param.group = self._core.cloudfs.groups.get(group, ['baseObjectRef']).baseObjectRef param.enableSyncWinNtExtendedAttributes = winacls param.folderQuota = quota if description: param.description = description - param.wormSettings = compliance_settings if compliance_settings else ComplianceSettingsBuilder.default().build() + if xattrs: param.extendedAttributes = xattrs elif not winacls: # Only override default when winacls is False @@ -193,31 +228,27 @@ def add(self, name, group, owner, winacls=True, description=None, # pylint: dis else: param.extendedAttributes = ExtendedAttributesBuilder.default().build() - if gfl: - param.globalFileLockSettings = Object() - param.globalFileLockSettings._classname = 'GlobalFileLockSettings' # pylint: disable=protected-access - param.globalFileLockSettings.enabled = True - param.globalFileLockSettings.globalFileLockExtensions = ( - lock_extensions if lock_extensions else CloudDrives.default_extensions - ) + @staticmethod + def _configure_archive(param, archive_settings): + param.archiveSettings = archive_settings if archive_settings else ArchiveSettingsBuilder.default().build() - try: - response = self._core.api.execute('', 'addCloudDrive', param) - logger.info( - 'Cloud drive folder created. %s', - {'name': name, 'owner': param.owner, 'folder_group': group, 'winacls': winacls} - ) - return re.search(r'/Users\/(.+)', response).group() - except CTERAException as error: - logger.error( - 'Cloud drive folder creation failed. %s', - {'name': name, 'folder_group': group, 'owner': str(owner), 'win_acls': winacls} - ) - raise error + @staticmethod + def _configure_compliance(param, compliance_settings): + param.wormSettings = compliance_settings if compliance_settings else ComplianceSettingsBuilder.default().build() - def modify(self, current_name, owner, new_name=None, # pylint: disable=too-many-arguments, too-many-locals - new_owner=None, new_group=None, description=None, winacls=None, quota=None, compliance_settings=None, xattrs=None, - gfl=None, lock_extensions=None): + @staticmethod + def _configure_locking(param, lock_extensions): + param.globalFileLockSettings = Object() + param.globalFileLockSettings._classname = 'GlobalFileLockSettings' # pylint: disable=protected-access + param.globalFileLockSettings.enabled = True + param.globalFileLockSettings.globalFileLockExtensions = ( + lock_extensions if lock_extensions else CloudDrives.default_extensions + ) + + def modify(self, current_name, owner, new_name=None, # pylint: disable=too-many-arguments, too-many-locals, too-many-branches + new_owner=None, new_group=None, description=None, winacls=None, quota=None, + archive_settings=None, compliance_settings=None, native_format_settings=None, + xattrs=None, gfl=None, lock_extensions=None): """ Modify a Cloud Drive Folder (Cloud Volume) @@ -229,8 +260,12 @@ def modify(self, current_name, owner, new_name=None, # pylint: disable=too-many :param str,optional description: Folder description :param bool,optional winacls: Enable or disable Windows ACLs :param str,optional quota: Folder quota in GB + :param cterasdk.common.object.Object,optional archive_settings: Archive settings. + Use :func:`cterasdk.core.types.ArchiveSettingsBuilder` to build the archive settings object :param cterasdk.common.object.Object,optional compliance_settings: Compliance settings. Use :func:`cterasdk.core.types.ComplianceSettingsBuilder` to build the compliance settings object + :param cterasdk.common.object.Object,optional native_format_settings: Native format settings. + Use :func:`cterasdk.core.types.NativeFormatSettingsBuilder` to build the native format settings object :param cterasdk.common.object.Object,optional xattrs: Extended attributes. Use :func:`cterasdk.core.types.ExtendedAttributesBuilder` to build the extended attributes object :param bool,optional gfl: Enable global file locking @@ -249,8 +284,12 @@ def modify(self, current_name, owner, new_name=None, # pylint: disable=too-many param.enableSyncWinNtExtendedAttributes = winacls if quota: param.folderQuota = quota + if archive_settings: + param.archiveSettings = archive_settings if compliance_settings: param.wormSettings = compliance_settings + if native_format_settings: + param.openFabricSettings = native_format_settings if xattrs: param.extendedAttributes = xattrs if gfl is not None: diff --git a/cterasdk/core/enum.py b/cterasdk/core/enum.py index ed6809bd..80e7060a 100644 --- a/cterasdk/core/enum.py +++ b/cterasdk/core/enum.py @@ -439,6 +439,7 @@ class BucketType: :ivar str Wasabi: Wasabi S3 :ivar str Google: Google S3 :ivar str NetAppStorageGRID: NetApp StorageGRID WebScale (S3) + :ivar str Cloudian: Cloudian (S3) """ Azure = 'Azure' Scality = 'ScalityS3' @@ -449,6 +450,7 @@ class BucketType: Wasabi = 'WasabiS3' Google = 'GoogleS3' NetAppStorageGRID = 'NTAP' + Cloudian = 'CD' class EnvironmentVariables: @@ -737,3 +739,16 @@ class ResourceError: FileWithTheSameNameExist = "FileWithTheSameNameExist" InvalidName = "InvalidName" ReservedName = "ReservedName" + + +class NativeFormat: + """ + Native Format - Fusion Direct + + :ivar str Filesystem: Read-write through the filesystem only. Bucket access is read-only. + :ivar str Bucket: Read-write through the bucket only. Filesystem access is read-only. + :ivar str Bidirectional: Read-write through both the filesystem and the bucket. + """ + Filesystem = 'Filesystem' + Bucket = 'Bucket' + Bidirectional = 'Bidirectional' diff --git a/cterasdk/core/servers.py b/cterasdk/core/servers.py index 52b21aaa..aa3d928a 100644 --- a/cterasdk/core/servers.py +++ b/cterasdk/core/servers.py @@ -130,7 +130,7 @@ def enable(self, bucket, interval): :param int interval: Backup interval in minutes """ database = self._core.servers.system_database - database.backupToBucket = Object(enabled=True, exportSchedulePeriod=interval, details=bucket.database_backup_server_object()) + database.backupToBucket = Object(enabled=True, exportSchedulePeriod=interval, details=bucket.to_database_backup_server_object()) logger.info("Enabling database backup. %s", {'server': database.name}) response = self._core.api.put(f'/servers/{database.name}', database) logger.info("Database backup enabled. %s", {'server': database.name}) diff --git a/cterasdk/core/types.py b/cterasdk/core/types.py index 3436566e..8fd0f140 100644 --- a/cterasdk/core/types.py +++ b/cterasdk/core/types.py @@ -4,7 +4,7 @@ from ..lib.storage import commonfs from .enum import PortalAccountType, CollaboratorType, FileAccessMode, PlanCriteria, TemplateCriteria, ProtectionLevel, \ - BucketType, LocationType, Platform, RetentionMode, Duration, ExtendedAttributes, ConflictHandler + BucketType, LocationType, Platform, RetentionMode, Duration, ExtendedAttributes, ConflictHandler, NativeFormat CloudFSFolderFindingHelper = namedtuple('CloudFSFolderFindingHelper', ('name', 'owner')) @@ -331,7 +331,21 @@ def __init__(self, bucket, driver, access_key, secret_key, endpoint, https, dire def trust_all_certificates(self): return not self.verify_ssl - def database_backup_server_object(self): + def to_native_format_server_object(self): + return Object( + _classname='OpenFabricS3DataStorage', + storage=self.driver, + bucket=self.bucket, + accessKey=self.access_key, + secretKey=self.secret_key, + endPoint=self.endpoint, + useHttps=self.https, + trustAllCertificates=self.trust_all_certificates, + masterHost=None, + usePathStyleAddressing=False + ) + + def to_database_backup_server_object(self): return Object( storage=self.driver, bucket=self.bucket, @@ -400,6 +414,13 @@ def __init__(self, bucket, access_key, secret_key, super().__init__(bucket, BucketType.Nutanix, access_key, secret_key, endpoint, https, direct, verify_ssl) +class Cloudian(S3Compatible): + + def __init__(self, bucket, access_key, secret_key, + endpoint, https=False, direct=False, verify_ssl=True): + super().__init__(bucket, BucketType.Cloudian, access_key, secret_key, endpoint, https, direct, verify_ssl) + + class Wasabi(S3Compatible): def __init__(self, bucket, access_key, secret_key, @@ -741,6 +762,58 @@ def build(self): return self.settings +class ArchiveSettingsBuilder: + + def __init__(self, enabled, mode, retain_for): + self.settings = Object() + self.settings._classname = 'ArchiveSettings' # pylint: disable=protected-access + self.settings.archive = enabled + self.settings.retentionMode = mode + self.settings.retentionPeriod = retain_for + self.settings.gracePeriod = ComplianceSettingsBuilder._get_retention_period(0, Duration.Minutes) + + @staticmethod + def default(): + return ArchiveSettingsBuilder(False, RetentionMode.Enterprise, None) + + @staticmethod + def enterprise(amount, duration): + return ArchiveSettingsBuilder(True, RetentionMode.Enterprise, + ComplianceSettingsBuilder._get_retention_period(amount, duration)) # pylint: disable=protected-access + + @staticmethod + def compliance(amount, duration): + return ArchiveSettingsBuilder(True, RetentionMode.Compliance, + ComplianceSettingsBuilder._get_retention_period(amount, duration)) # pylint: disable=protected-access + + def build(self): + return self.settings + + +class NativeFormatSettingsBuilder: + + def __init__(self, mode, bucket, sqs, region): + self.settings = Object(_classname='OpenFabricSettings', storageMode=mode) + self.settings.dataStorage = bucket.to_native_format_server_object() + self.settings.dataStorage.sqsUrl = sqs + self.settings.dataStorage.region = region + + @staticmethod + def filesystem(bucket, region): + return NativeFormatSettingsBuilder(NativeFormat.Filesystem, bucket, None, region) + + @staticmethod + def bucket(bucket, sqs, region): + return NativeFormatSettingsBuilder(NativeFormat.Bucket, bucket, sqs, region) + + @staticmethod + def bidirectional(bucket, sqs, region): + return NativeFormatSettingsBuilder(NativeFormat.Bidirectional, bucket, sqs, region) + + def build(self): + return self.settings + + class ExtendedAttribute(Object): def __init__(self, name, supported): diff --git a/docs/source/UserGuides/Portal/Administration.rst b/docs/source/UserGuides/Portal/Administration.rst index 23d4023b..85234c71 100644 --- a/docs/source/UserGuides/Portal/Administration.rst +++ b/docs/source/UserGuides/Portal/Administration.rst @@ -1425,10 +1425,13 @@ Cloud Drive Folders wbruce = core_types.UserAccount('wbruce', 'ctera.local') admin.cloudfs.drives.add('DIR-002', 'FG-002', wbruce) +Code examples to create immutable Cloud Drive folders + +.. code:: python + """Create immutable Cloud Drive folders""" svc_account = core_types.UserAccount('svc_account') - """ Mode: Enterprise (i.e., allow privileged delete by the CTERA Compliance Officer role) Retention Period: 7 Years. @@ -1447,6 +1450,34 @@ Cloud Drive Folders settings = core_types.ComplianceSettingsBuilder.enterprise(1, core_enum.Duration.Years).grace_period(1, core_enum.Duration.Hours).build() admin.cloudfs.drives.add('Compliance', 'FG-Compliance', svc_account, compliance_settings=settings) +Code examples to create archive Cloud Drive folders + +.. code:: python + + archive_settings = core_types.ArchiveSettingsBuilder.enterprise(30, core_enum.Duration.Days).build() + admin.cloudfs.groups.add( + 'ARCHIVE-DIR-001', + 'FG-001', + core_types.UserAccount('svc_account'), + archive_settings=archive_settings + ) + +Code examples to create Cloud Drive folders using Fusion Direct + +.. code:: python + + name, access, secret = 'target-s3-bucket-name', 'access-key', 'secret-access-key' + endpoint, https = 's3.eu-west-1.amazonaws.com', True + bucket = core_types.AmazonS3(name, access, secret, endpoint, https) # use verify_ssl=False to trust all certificates + + sqs = 'https://sqs.ctera.local:18090/7ed20456b4eebb2c21c0079edcefe2ce/bucket' + native_format_settings = core_types.NativeFormatSettingsBuilder.bidirectional(bucket, sqs).build() + + user.cloudfs.drives.add( + 'DIR-001-Native', + owner=core_types.UserAccount('svc_account'), + native_format_settings=native_format_settings) + ) .. automethod:: cterasdk.core.cloudfs.CloudDrives.modify :noindex: diff --git a/tests/ut/core/admin/test_cloudfs_cloud_drives.py b/tests/ut/core/admin/test_cloudfs_cloud_drives.py index beeb58ab..153e2083 100644 --- a/tests/ut/core/admin/test_cloudfs_cloud_drives.py +++ b/tests/ut/core/admin/test_cloudfs_cloud_drives.py @@ -3,7 +3,7 @@ from cterasdk import exceptions from cterasdk.core import cloudfs -from cterasdk.core.types import UserAccount, ComplianceSettingsBuilder, ExtendedAttributesBuilder +from cterasdk.core.types import UserAccount, ArchiveSettingsBuilder, ComplianceSettingsBuilder, ExtendedAttributesBuilder from cterasdk.core import query from cterasdk.common import Object, union from tests.ut.core.admin import base_admin @@ -191,8 +191,8 @@ def test_undelete_with_local_owner(self): self._global_admin.users.get.assert_called_once_with(self._local_user_account, ['displayName']) self._global_admin.files.undelete.assert_called_once_with(f'Users/{self._owner}/{self._name}') - def _get_add_cloud_drive_object(self, winacls=True, description=None, quota=None, compliance_settings=None, xattrs=None, - gfl=None, lock_extensions=None): + def _get_add_cloud_drive_object(self, winacls=True, description=None, quota=None, archive_settings=None, + compliance_settings=None, xattrs=None, gfl=None, lock_extensions=None): add_cloud_drive_param = Object() add_cloud_drive_param.name = self._name add_cloud_drive_param.owner = self._owner @@ -202,6 +202,7 @@ def _get_add_cloud_drive_object(self, winacls=True, description=None, quota=None if description: add_cloud_drive_param.description = description add_cloud_drive_param.wormSettings = compliance_settings if compliance_settings else ComplianceSettingsBuilder.default().build() + add_cloud_drive_param.archiveSettings = archive_settings if archive_settings else ArchiveSettingsBuilder.default().build() add_cloud_drive_param.extendedAttributes = xattrs if xattrs else ExtendedAttributesBuilder.default().build() if gfl: add_cloud_drive_param.globalFileLockSettings = Object()