diff --git a/CMakeLists.txt b/CMakeLists.txt index d92311448..7545b4e9a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,6 +113,17 @@ ecbuild_add_option( FEATURE RADOS DESCRIPTION "Ceph/Rados storage support" REQUIRED_PACKAGES Rados ) +### S3 Support + +ecbuild_add_option( FEATURE AWSSDK_S3 + DEFAULT OFF + REQUIRED_PACKAGES "AWSSDK COMPONENTS s3 QUIET" + DESCRIPTION "Enables S3 storage support via AWS SDK C++ library" ) + +ecbuild_add_option( FEATURE S3 + CONDITION HAVE_AWSSDK_S3 + DESCRIPTION "Enables S3 storage support" ) + ### Armadillo ecbuild_add_option( FEATURE ARMADILLO diff --git a/src/eckit/CMakeLists.txt b/src/eckit/CMakeLists.txt index d3a2523cc..5d70231ab 100644 --- a/src/eckit/CMakeLists.txt +++ b/src/eckit/CMakeLists.txt @@ -283,6 +283,44 @@ if(HAVE_RADOS) ) endif() +if( eckit_HAVE_S3 ) + + if( eckit_HAVE_AWSSDK_S3 ) + list( APPEND eckit_io_srcs + io/s3/aws/S3ClientAWS.cc + io/s3/aws/S3ClientAWS.h + io/s3/aws/S3ContextAWS.cc + io/s3/aws/S3ContextAWS.h + ) + endif( eckit_HAVE_AWSSDK_S3 ) + + list( APPEND eckit_io_srcs + io/s3/S3BucketName.cc + io/s3/S3BucketName.h + io/s3/S3BucketPath.h + io/s3/S3Client.cc + io/s3/S3Client.h + io/s3/S3Config.cc + io/s3/S3Config.h + io/s3/S3Credential.cc + io/s3/S3Credential.h + io/s3/S3Exception.cc + io/s3/S3Exception.h + io/s3/S3Handle.cc + io/s3/S3Handle.h + io/s3/S3Name.cc + io/s3/S3Name.h + io/s3/S3ObjectName.cc + io/s3/S3ObjectName.h + io/s3/S3ObjectPath.h + io/s3/S3Session.cc + io/s3/S3Session.h + io/s3/S3URIManager.cc + io/s3/S3URIManager.h + ) + +endif( eckit_HAVE_S3 ) + list( APPEND eckit_filesystem_srcs filesystem/BasePathName.cc filesystem/BasePathName.h @@ -925,6 +963,7 @@ ecbuild_add_library( $ PRIVATE_INCLUDES + $ "${CURL_INCLUDE_DIRS}" "${SNAPPY_INCLUDE_DIRS}" "${LZ4_INCLUDE_DIRS}" @@ -933,6 +972,7 @@ ecbuild_add_library( "${RADOS_INCLUDE_DIRS}" "${OPENSSL_INCLUDE_DIR}" "${AIO_INCLUDE_DIRS}" + "${AWSSDK_INCLUDE_DIRS}" PRIVATE_LIBS "${SNAPPY_LIBRARIES}" @@ -944,6 +984,7 @@ ecbuild_add_library( "${CURL_LIBRARIES}" "${AIO_LIBRARIES}" "${RADOS_LIBRARIES}" + "${AWSSDK_LINK_LIBRARIES}" PUBLIC_LIBS ${CMATH_LIBRARIES} diff --git a/src/eckit/io/s3/S3BucketName.cc b/src/eckit/io/s3/S3BucketName.cc new file mode 100644 index 000000000..1ef0f54a8 --- /dev/null +++ b/src/eckit/io/s3/S3BucketName.cc @@ -0,0 +1,110 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +#include "eckit/io/s3/S3BucketName.h" + +#include "eckit/config/LibEcKit.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/s3/S3BucketPath.h" +#include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/S3Name.h" +#include "eckit/io/s3/S3ObjectName.h" +#include "eckit/io/s3/S3ObjectPath.h" +#include "eckit/log/CodeLocation.h" +#include "eckit/log/Log.h" +#include "eckit/net/Endpoint.h" + +#include +#include +#include +#include +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3BucketName::parse(const std::string& name) -> S3BucketPath { + const auto parsed = S3Name::parse(name); + if (const auto size = parsed.size(); size < 1 || size > 2) { + throw S3SeriousBug("Could not parse bucket from name: " + name, Here()); + } + return {parsed[0]}; +} + +//---------------------------------------------------------------------------------------------------------------------- + +S3BucketName::S3BucketName(const net::Endpoint& endpoint, S3BucketPath path) + : S3Name(endpoint), path_ {std::move(path)} { } + +S3BucketName::S3BucketName(const URI& uri) : S3Name(uri), path_ {parse(uri.name())} { } + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3BucketName::uri() const -> URI { + auto uri = S3Name::uri(); + uri.path(path_); + return uri; +} + +auto S3BucketName::asString() const -> std::string { + return S3Name::asString() + '/' + path_.asString(); +} + +void S3BucketName::print(std::ostream& out) const { + out << "S3BucketName["; + S3Name::print(out); + out << ',' << path_ << ']'; +} + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3BucketName::makeObject(const std::string& object) const -> std::unique_ptr { + return std::make_unique(endpoint(), S3ObjectPath {path_, object}); +} + +auto S3BucketName::exists() const -> bool { + return client().bucketExists(path_); +} + +void S3BucketName::create() { + client().createBucket(path_); +} + +void S3BucketName::destroy() { + client().deleteBucket(path_); +} + +void S3BucketName::ensureCreated() { + try { + create(); + } catch (S3EntityAlreadyExists& e) { LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; } +} + +void S3BucketName::ensureDestroyed() { + try { + client().emptyBucket(path_); + client().deleteBucket(path_); + } catch (S3EntityNotFound& e) { LOG_DEBUG_LIB(LibEcKit) << e.what() << std::endl; } +} + +auto S3BucketName::listObjects() const -> std::vector { + return client().listObjects(path_); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3BucketName.h b/src/eckit/io/s3/S3BucketName.h new file mode 100644 index 000000000..26ba193df --- /dev/null +++ b/src/eckit/io/s3/S3BucketName.h @@ -0,0 +1,81 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +/// @file S3BucketName.h +/// @author Metin Cakircali +/// @author Nicolau Manubens +/// @date Feb 2024 + +#pragma once + +#include "eckit/io/s3/S3BucketPath.h" +#include "eckit/io/s3/S3Name.h" +#include "eckit/net/Endpoint.h" + +#include +#include +#include +#include + +namespace eckit { + +class S3ObjectName; + +//---------------------------------------------------------------------------------------------------------------------- + +class S3BucketName : public S3Name { +public: // factory + static auto parse(const std::string& name) -> S3BucketPath; + + auto makeObject(const std::string& object) const -> std::unique_ptr; + +public: // methods + S3BucketName(const net::Endpoint& endpoint, S3BucketPath path); + + explicit S3BucketName(const URI& uri); + + // accessors + + auto asString() const -> std::string override; + + auto uri() const -> URI override; + + auto exists() const -> bool override; + + auto path() const -> const S3BucketPath& { return path_; } + + /// @todo return S3 object iterator but first add prefix + auto listObjects() const -> std::vector; + + // modifiers + + void create(); + + void destroy(); + + void ensureCreated(); + + void ensureDestroyed(); + +private: // methods + void print(std::ostream& out) const override; + +private: // members + S3BucketPath path_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3BucketPath.h b/src/eckit/io/s3/S3BucketPath.h new file mode 100644 index 000000000..8eba5f9f7 --- /dev/null +++ b/src/eckit/io/s3/S3BucketPath.h @@ -0,0 +1,54 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the Horizon Europe programme funded project DaFab + * (Grant agreement: 101128693) https://www.dafab-ai.eu/ + */ + +/// @file S3BucketPath.h +/// @author Metin Cakircali +/// @date Dec 2024 + +#pragma once + +#include +#include +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +struct S3BucketPath { + + S3BucketPath(std::string bucket) : bucket {std::move(bucket)} { } + + auto asString() const -> std::string { return bucket; } + + operator std::string() const { return asString(); } + + bool operator==(const S3BucketPath& other) const { return bucket == other.bucket; } + + bool operator!=(const S3BucketPath& other) const { return !(*this == other); } + + friend std::ostream& operator<<(std::ostream& out, const S3BucketPath& path) { + out << "bucket=" << path.bucket; + return out; + } + + // members + + std::string bucket; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Client.cc b/src/eckit/io/s3/S3Client.cc new file mode 100644 index 000000000..894dc2a96 --- /dev/null +++ b/src/eckit/io/s3/S3Client.cc @@ -0,0 +1,56 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +#include "eckit/io/s3/S3Client.h" + +#include "eckit/exception/Exceptions.h" +#include "eckit/io/s3/S3Config.h" +#include "eckit/io/s3/S3Session.h" +#include "eckit/io/s3/aws/S3ClientAWS.h" +#include "eckit/log/CodeLocation.h" + +#include +#include +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +S3Client::S3Client(S3Config config) : config_ {std::move(config)} { } + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3Client::makeUnique(const S3Config& config) -> std::unique_ptr { + + if (config.backend == S3Backend::AWS) { return std::make_unique(config); } + + throw UserError("Unsupported S3 backend! Supported backend = AWS ", Here()); +} + +//---------------------------------------------------------------------------------------------------------------------- + +void S3Client::print(std::ostream& out) const { + out << "S3Client[config=" << config_ << "]"; +} + +std::ostream& operator<<(std::ostream& out, const S3Client& client) { + client.print(out); + return out; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Client.h b/src/eckit/io/s3/S3Client.h new file mode 100644 index 000000000..338d345b0 --- /dev/null +++ b/src/eckit/io/s3/S3Client.h @@ -0,0 +1,95 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +/// @file S3Client.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#pragma once + +#include "eckit/io/s3/S3BucketPath.h" +#include "eckit/io/s3/S3Config.h" +#include "eckit/io/s3/S3ObjectPath.h" + +#include +#include +#include +#include +#include + +namespace eckit { + +class S3Context; + +//---------------------------------------------------------------------------------------------------------------------- + +class S3Client { +public: // methods + S3Client(const S3Client&) = delete; + S3Client& operator=(const S3Client&) = delete; + S3Client(S3Client&&) = default; + S3Client& operator=(S3Client&&) = default; + virtual ~S3Client() = default; + + static auto makeUnique(const S3Config& config) -> std::unique_ptr; + + static auto makeShared(const S3Config& config) -> std::shared_ptr { return makeUnique(config); } + + auto config() const -> const S3Config& { return config_; } + + // bucket operations + + virtual void createBucket(const S3BucketPath& path) const = 0; + + virtual void emptyBucket(const S3BucketPath& path) const = 0; + + virtual void deleteBucket(const S3BucketPath& path) const = 0; + + virtual auto bucketExists(const S3BucketPath& path) const -> bool = 0; + + virtual auto listBuckets() const -> std::vector = 0; + + // object operations + + virtual auto putObject(const S3ObjectPath& path, const void* buffer, uint64_t length) const -> long long = 0; + + virtual auto getObject(const S3ObjectPath& path, void* buffer, uint64_t offset, uint64_t length) const -> long long = 0; + + virtual void deleteObject(const S3ObjectPath& path) const = 0; + + virtual void deleteObjects(const S3BucketPath& path, const std::vector& objects) const = 0; + + virtual auto listObjects(const S3BucketPath& path) const -> std::vector = 0; + + virtual auto objectExists(const S3ObjectPath& path) const -> bool = 0; + + virtual auto objectSize(const S3ObjectPath& path) const -> long long = 0; + +protected: // methods + S3Client(); + + explicit S3Client(S3Config config); + + virtual void print(std::ostream& out) const; + + friend std::ostream& operator<<(std::ostream& out, const S3Client& client); + +private: // members + S3Config config_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Config.cc b/src/eckit/io/s3/S3Config.cc new file mode 100644 index 000000000..63973ae16 --- /dev/null +++ b/src/eckit/io/s3/S3Config.cc @@ -0,0 +1,124 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +#include "eckit/io/s3/S3Config.h" + +#include "eckit/config/LibEcKit.h" +#include "eckit/config/LocalConfiguration.h" +#include "eckit/config/Resource.h" +#include "eckit/config/YAMLConfiguration.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/log/CodeLocation.h" +#include "eckit/log/Log.h" +#include "eckit/net/Endpoint.h" + +#include +#include +#include +#include +#include +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3Config::make(const LocalConfiguration& config) -> S3Config { + const auto endpoint = config.getString("endpoint"); + + if (endpoint.empty()) { throw eckit::UserError("No S3 endpoint found in configuration!", Here()); } + + const auto region = config.getString("region", defaultRegion); + + S3Config s3config(endpoint, region); + + if (config.has("backend")) { + const auto backendStr = config.getString("backend"); + if (backendStr == "AWS") { + s3config.backend = S3Backend::AWS; + } else if (backendStr == "REST") { + s3config.backend = S3Backend::REST; + } else if (backendStr == "MinIO") { + s3config.backend = S3Backend::MINIO; + } else { + throw UserError("Invalid backend: " + backendStr, Here()); + } + } + + if (config.has("secure")) { s3config.secure = config.getBool("secure"); } + + return s3config; +} + +auto S3Config::make(std::string path) -> std::vector { + + if (path.empty()) { + path = Resource("s3ConfigFile;$ECKIT_S3_CONFIG_FILE", "~/.config/eckit/S3Config.yaml"); + } + + PathName configPath(path); + + if (!configPath.exists()) { + Log::debug() << "S3 configuration file does not exist: " << configPath << std::endl; + return {}; + } + + if (configPath.isDir()) { + Log::debug() << "Path " << configPath << " is a directory. Expecting a file!" << std::endl; + return {}; + } + + const auto servers = YAMLConfiguration(configPath).getSubConfigurations("servers"); + + std::vector result; + result.reserve(servers.size()); + for (const auto& server : servers) { result.emplace_back(make(server)); } + + return result; +} + +S3Config::S3Config(const net::Endpoint& endpoint, std::string region) + : endpoint {endpoint}, region {std::move(region)} { } + +S3Config::S3Config(const std::string& host, const uint16_t port, std::string region) + : endpoint {host, port}, region {std::move(region)} { } + +S3Config::S3Config(const URI& uri) : S3Config(uri.host(), uri.port()) { } + +//---------------------------------------------------------------------------------------------------------------------- + +void S3Config::print(std::ostream& out) const { + out << "S3Config[endpoint=" << endpoint << ",region=" << region << ",backend="; + if (backend == S3Backend::AWS) { + out << "AWS"; + } else if (backend == S3Backend::REST) { + out << "REST"; + } else if (backend == S3Backend::MINIO) { + out << "MinIO"; + } + out << ",secure=" << secure << "]"; +} + +//---------------------------------------------------------------------------------------------------------------------- + +std::ostream& operator<<(std::ostream& out, const S3Config& config) { + config.print(out); + return out; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Config.h b/src/eckit/io/s3/S3Config.h new file mode 100644 index 000000000..3a9a69824 --- /dev/null +++ b/src/eckit/io/s3/S3Config.h @@ -0,0 +1,99 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +/// @file S3Config.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#pragma once + +#include "eckit/net/Endpoint.h" + +#include +#include +#include +#include + +namespace eckit { + +class LocalConfiguration; + +enum class S3Backend : std::uint8_t { AWS, REST, MINIO }; + +//---------------------------------------------------------------------------------------------------------------------- + +/// @brief S3 configurations for a given endpoint +/// +/// @example Example YAML S3 configuration file: +/// +/// ECKIT_S3_CONFIG_FILE = ~/.config/eckit/S3Config.yaml +/// +/// --- +/// servers: +/// - endpoint: "127.0.0.1:9000" +/// region: "default" # (default) +/// secure: true # (default) +/// backend: "AWS" # (default) +/// +/// - endpoint: "minio:9000" +/// region: "eu-central-1" +/// +/// - endpoint: "https://localhost:9000" +/// region: "eu-central-1" +/// +/// # region is inferred from the endpoint +/// - endpoint: "https://eu-central-1.ecmwf.int:9000" +/// +struct S3Config { + static constexpr auto defaultRegion = "default"; + static constexpr auto defaultBackend = S3Backend::AWS; + + // static methods + + static auto make(const LocalConfiguration& config) -> S3Config; + + static auto make(std::string path) -> std::vector; + + // constructors + + explicit S3Config(const net::Endpoint& endpoint, std::string region = defaultRegion); + + explicit S3Config(const std::string& host, uint16_t port, std::string region = defaultRegion); + + explicit S3Config(const URI& uri); + + // operators + + bool operator==(const S3Config& other) const { + return backend == other.backend && endpoint == other.endpoint && region == other.region; + } + + bool operator!=(const S3Config& other) const { return !(*this == other); } + + void print(std::ostream& out) const; + + friend std::ostream& operator<<(std::ostream& out, const S3Config& config); + + // members + + net::Endpoint endpoint; + std::string region {defaultRegion}; + S3Backend backend {defaultBackend}; + bool secure {true}; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Credential.cc b/src/eckit/io/s3/S3Credential.cc new file mode 100644 index 000000000..762fe8bdc --- /dev/null +++ b/src/eckit/io/s3/S3Credential.cc @@ -0,0 +1,77 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +#include "eckit/io/s3/S3Credential.h" + +#include "eckit/config/Configuration.h" +#include "eckit/config/LibEcKit.h" +#include "eckit/config/LocalConfiguration.h" +#include "eckit/config/Resource.h" +#include "eckit/config/YAMLConfiguration.h" +#include "eckit/filesystem/PathName.h" +#include "eckit/log/Log.h" + +#include +#include +#include +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +namespace { + +const std::string defaultCredFile = "~/.config/eckit/S3Credentials.yaml"; + +S3Credential fromYAML(const LocalConfiguration& config) { + const auto endpoint = config.getString("endpoint"); + const auto keyID = config.getString("accessKeyID"); + const auto secret = config.getString("secretKey"); + return {endpoint, keyID, secret}; +} + +} // namespace + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3Credential::fromFile(std::string path) -> std::vector { + + if (path.empty()) { path = Resource("s3CredentialsFile;$ECKIT_S3_CREDENTIALS_FILE", defaultCredFile); } + + PathName credPath(path); + + if (!credPath.exists()) { + Log::debug() << "S3 credentials file does not exist: " << credPath << std::endl; + return {}; + } + + if (credPath.isDir()) { + Log::debug() << "Path " << credPath << " is a directory. Expecting a file!" << std::endl; + return {}; + } + + const auto creds = YAMLConfiguration(credPath).getSubConfigurations("credentials"); + + std::vector result; + result.reserve(creds.size()); + for (const auto& cred : creds) { result.emplace_back(fromYAML(cred)); } + + return result; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Credential.h b/src/eckit/io/s3/S3Credential.h new file mode 100644 index 000000000..4cb9c594b --- /dev/null +++ b/src/eckit/io/s3/S3Credential.h @@ -0,0 +1,68 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +/// @file S3Credential.h +/// @author Metin Cakircali +/// @date Feb 2024 + +#pragma once + +#include "eckit/net/Endpoint.h" + +#include +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +/// @brief S3 credential information for a given endpoint +/// +/// @example Example YAML S3 credential file: +/// +/// ECKIT_S3_CREDENTIALS_FILE = ~/.config/eckit/S3Credentials.yaml +/// +/// --- +/// credentials: +/// - endpoint: '127.0.0.1:9000' +/// accessKeyID: 'minio' +/// secretKey: 'minio1234' +/// +/// - endpoint: 'minio:9000' +/// accessKeyID: 'minio' +/// secretKey: 'minio1234' +/// +/// - endpoint: 'localhost:9000' +/// accessKeyID: 'asd2' +/// secretKey: 'asd2' +/// +struct S3Credential { + + static auto fromFile(std::string path) -> std::vector; + + auto operator==(const S3Credential& other) const -> bool { + return endpoint == other.endpoint && keyID == other.keyID && secret == other.secret; + } + + auto operator!=(const S3Credential& other) const -> bool { return !(*this == other); } + + net::Endpoint endpoint; + std::string keyID; + std::string secret; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Exception.cc b/src/eckit/io/s3/S3Exception.cc new file mode 100644 index 000000000..90d04798e --- /dev/null +++ b/src/eckit/io/s3/S3Exception.cc @@ -0,0 +1,62 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +#include "eckit/io/s3/S3Exception.h" + +#include "eckit/exception/Exceptions.h" +#include "eckit/log/CodeLocation.h" + +#include +#include +#include + +//---------------------------------------------------------------------------------------------------------------------- + +namespace { + +inline auto addCode(const std::string& msg, int code) -> std::string { + std::ostringstream oss; + oss << "Code(" << strerror(-code) << ") " << msg; + return oss.str(); +}; + +} // namespace + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +S3SeriousBug::S3SeriousBug(const std::string& msg, const CodeLocation& loc) + : SeriousBug("[S3 Serious Bug] " + msg, loc) { } + +S3SeriousBug::S3SeriousBug(const std::string& msg, const int code, const CodeLocation& loc) + : S3SeriousBug(addCode(msg, code), loc) { } + +//---------------------------------------------------------------------------------------------------------------------- + +S3Exception::S3Exception(const std::string& msg, const CodeLocation& loc) : Exception("[S3 Exception] " + msg, loc) { } + +S3BucketNotEmpty::S3BucketNotEmpty(const std::string& msg, const CodeLocation& loc) + : S3Exception("[Not Empty] " + msg, loc) { } + +S3EntityNotFound::S3EntityNotFound(const std::string& msg, const CodeLocation& loc) + : S3Exception("[Not Found] " + msg, loc) { } + +S3EntityAlreadyExists::S3EntityAlreadyExists(const std::string& msg, const CodeLocation& loc) + : S3Exception("[Already Exists] " + msg, loc) { } + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Exception.h b/src/eckit/io/s3/S3Exception.h new file mode 100644 index 000000000..173409ab7 --- /dev/null +++ b/src/eckit/io/s3/S3Exception.h @@ -0,0 +1,67 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +/// @file S3Exception.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#pragma once + +#include "eckit/exception/Exceptions.h" +#include "eckit/log/CodeLocation.h" + +#include +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +class S3SeriousBug : public SeriousBug { +public: + S3SeriousBug(const std::string& msg, const CodeLocation& loc); + S3SeriousBug(const std::string& msg, int code, const CodeLocation& loc); + + S3SeriousBug(const std::ostringstream& msg, const CodeLocation& loc) : S3SeriousBug(msg.str(), loc) { } + + S3SeriousBug(const std::ostringstream& msg, int code, const CodeLocation& loc) + : S3SeriousBug(msg.str(), code, loc) { } +}; + +//---------------------------------------------------------------------------------------------------------------------- + +class S3Exception : public Exception { +public: + S3Exception(const std::string& msg, const CodeLocation& loc); +}; + +class S3BucketNotEmpty : public S3Exception { +public: + S3BucketNotEmpty(const std::string& msg, const CodeLocation& loc); +}; + +class S3EntityNotFound : public S3Exception { +public: + S3EntityNotFound(const std::string& msg, const CodeLocation& loc); +}; + +class S3EntityAlreadyExists : public S3Exception { +public: + S3EntityAlreadyExists(const std::string& msg, const CodeLocation& loc); +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Handle.cc b/src/eckit/io/s3/S3Handle.cc new file mode 100644 index 000000000..5416af8ce --- /dev/null +++ b/src/eckit/io/s3/S3Handle.cc @@ -0,0 +1,128 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +#include "eckit/io/s3/S3Handle.h" + +#include "eckit/config/LibEcKit.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/io/Length.h" +#include "eckit/io/Offset.h" +#include "eckit/io/s3/S3ObjectName.h" +#include "eckit/log/Log.h" + +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +S3Handle::S3Handle(const S3ObjectName& name, const Offset& offset) : name_(name), pos_(offset) { } + +S3Handle::S3Handle(const S3ObjectName& name) : S3Handle(name, 0) { } + +void S3Handle::print(std::ostream& out) const { + out << "S3Handle[name=" << name_ << ", position=" << pos_; + if (mode_ == Mode::CLOSED) { + out << ", mode=closed"; + } else if (mode_ == Mode::READ) { + out << ", mode=read"; + } else if (mode_ == Mode::WRITE) { + out << ", mode=write"; + } + out << "]"; +} + +//---------------------------------------------------------------------------------------------------------------------- + +Length S3Handle::openForRead() { + open(Mode::READ); + + return estimate(); +} + +void S3Handle::openForWrite(const Length& /*length*/) { + open(Mode::WRITE); + + ASSERT(name_.bucketExists()); + + /// @todo slow code, use of length ? + // if (length > 0 && name_.exists()) { ASSERT(size() == length); } +} + +//---------------------------------------------------------------------------------------------------------------------- + +long S3Handle::read(void* buffer, const long length) { + ASSERT(mode_ == Mode::READ); + ASSERT(0 <= pos_); + + if (size() <= pos_) { return 0; } + + const auto len = name_.get(buffer, pos_, length); + + pos_ += len; + + return len; +} + +long S3Handle::write(const void* buffer, const long length) { + ASSERT(mode_ == Mode::WRITE); + ASSERT(!name_.exists()); + + const auto len = name_.put(buffer, length); + + pos_ += len; + + return len; +} + +//---------------------------------------------------------------------------------------------------------------------- + +void S3Handle::open(const Mode mode) { + ASSERT(mode_ == Mode::CLOSED); + pos_ = 0; + mode_ = mode; +} + +void S3Handle::close() { + if (mode_ == Mode::WRITE) { flush(); } + pos_ = 0; + mode_ = Mode::CLOSED; +} + +void S3Handle::flush() { + LOG_DEBUG_LIB(LibEcKit) << "flush()!" << std::endl; +} + +//---------------------------------------------------------------------------------------------------------------------- + +Length S3Handle::size() { + return name_.size(); +} + +Length S3Handle::estimate() { + return size(); +} + +Offset S3Handle::seek(const Offset& offset) { + pos_ = pos_ + offset; + + ASSERT(0 <= pos_ && size() >= pos_); + + return pos_; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Handle.h b/src/eckit/io/s3/S3Handle.h new file mode 100644 index 000000000..30cb65d6d --- /dev/null +++ b/src/eckit/io/s3/S3Handle.h @@ -0,0 +1,78 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +/// @file S3Handle.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#pragma once + +#include "eckit/io/DataHandle.h" +#include "eckit/io/Length.h" +#include "eckit/io/s3/S3ObjectName.h" + +#include +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +class S3Handle : public DataHandle { +public: // methods + enum class Mode : std::uint8_t { CLOSED, READ, WRITE }; + + S3Handle(const S3ObjectName& name, const Offset& offset); + + explicit S3Handle(const S3ObjectName& name); + + Length openForRead() override; + + void openForWrite(const Length& length) override; + + long read(void* buffer, long length) override; + + long write(const void* buffer, long length) override; + + void flush() override; + + void close() override; + + Length size() override; + + Length estimate() override; + + Offset position() override { return pos_; } + + Offset seek(const Offset& offset) override; + + auto canSeek() const -> bool override { return true; } + +private: // methods + void print(std::ostream& out) const override; + + void open(Mode mode); + +private: // members + S3ObjectName name_; + + Offset pos_ {0}; + + Mode mode_ {Mode::CLOSED}; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Name.cc b/src/eckit/io/s3/S3Name.cc new file mode 100644 index 000000000..96c74fe73 --- /dev/null +++ b/src/eckit/io/s3/S3Name.cc @@ -0,0 +1,89 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +#include "eckit/io/s3/S3Name.h" + +#include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/s3/S3BucketName.h" +#include "eckit/io/s3/S3BucketPath.h" +#include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3ObjectName.h" +#include "eckit/io/s3/S3ObjectPath.h" +#include "eckit/io/s3/S3Session.h" +#include "eckit/log/CodeLocation.h" +#include "eckit/utils/Tokenizer.h" + +#include +#include +#include +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3Name::parse(const std::string& name) -> std::vector { + return Tokenizer("/").tokenize(name); +} + +auto S3Name::make(const net::Endpoint& endpoint, const std::string& name) -> std::unique_ptr { + const auto names = parse(name); + switch (names.size()) { + case 1: return std::make_unique(endpoint, S3BucketPath {names[0]}); + case 2: return std::make_unique(endpoint, S3ObjectPath {names[0], names[1]}); + default: throw SeriousBug("Could not parse S3 name: " + name, Here()); + } +} + +//---------------------------------------------------------------------------------------------------------------------- + +S3Name::S3Name(const net::Endpoint& endpoint) : endpoint_ {endpoint} { } + +S3Name::S3Name(const URI& uri) : endpoint_ {uri.host(), uri.port()} { + /// @todo is "s3://endpoint/bucket/object" a valid URI ? + ASSERT(uri.scheme() == type); +} + +S3Name::~S3Name() = default; + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3Name::client() const -> S3Client& { + return *S3Session::instance().getClient(endpoint_); +} + +auto S3Name::uri() const -> URI { + return {type, endpoint_.host(), endpoint_.port()}; +} + +auto S3Name::asString() const -> std::string { + return std::string(endpoint_); +} + +void S3Name::print(std::ostream& out) const { + out << "endpoint=" << endpoint_; +} + +//---------------------------------------------------------------------------------------------------------------------- + +std::ostream& operator<<(std::ostream& out, const S3Name& name) { + name.print(out); + return out; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Name.h b/src/eckit/io/s3/S3Name.h new file mode 100644 index 000000000..dbd2357fc --- /dev/null +++ b/src/eckit/io/s3/S3Name.h @@ -0,0 +1,82 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +/// @file S3Name.h +/// @author Metin Cakircali +/// @author Simon Smart +/// @date Jan 2024 + +#pragma once + +#include "eckit/net/Endpoint.h" + +#include +#include +#include +#include + +namespace eckit { + +class S3Client; + +//---------------------------------------------------------------------------------------------------------------------- + +class S3Name { +public: // statics + static constexpr auto type = "s3"; + + static auto parse(const std::string& name) -> std::vector; + + static auto make(const net::Endpoint& endpoint, const std::string& name) -> std::unique_ptr; + +public: // methods + explicit S3Name(const net::Endpoint& endpoint); + + explicit S3Name(const URI& uri); + + // rules + + S3Name(const S3Name&) = default; + S3Name& operator=(const S3Name&) = default; + + S3Name(S3Name&&) = delete; + S3Name& operator=(S3Name&&) = delete; + + virtual ~S3Name(); + + // accessors + + virtual auto uri() const -> URI; + + virtual auto exists() const -> bool = 0; + + virtual auto asString() const -> std::string; + + auto endpoint() const -> const net::Endpoint& { return endpoint_; } + +protected: // methods + virtual void print(std::ostream& out) const; + + friend std::ostream& operator<<(std::ostream& out, const S3Name& name); + + auto client() const -> S3Client&; + +private: // members + net::Endpoint endpoint_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3ObjectName.cc b/src/eckit/io/s3/S3ObjectName.cc new file mode 100644 index 000000000..507084bde --- /dev/null +++ b/src/eckit/io/s3/S3ObjectName.cc @@ -0,0 +1,102 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +#include "eckit/io/s3/S3ObjectName.h" + +#include "eckit/filesystem/URI.h" +#include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/S3Handle.h" +#include "eckit/io/s3/S3Name.h" +#include "eckit/io/s3/S3ObjectPath.h" +#include "eckit/log/CodeLocation.h" +#include "eckit/net/Endpoint.h" + +#include +#include +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3ObjectName::parse(const std::string& name) -> S3ObjectPath { + const auto parsed = S3Name::parse(name); + if (parsed.size() != 2) { throw S3SeriousBug("Could not parse bucket/object from name: " + name, Here()); } + return {parsed[0], parsed[1]}; +} + +//---------------------------------------------------------------------------------------------------------------------- + +S3ObjectName::S3ObjectName(const net::Endpoint& endpoint, S3ObjectPath path) + : S3Name(endpoint), path_ {std::move(path)} { } + +S3ObjectName::S3ObjectName(const URI& uri) : S3Name(uri), path_ {parse(uri.name())} { } + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3ObjectName::uri() const -> URI { + auto uri = S3Name::uri(); + uri.path(path_); + return uri; +} + +auto S3ObjectName::asString() const -> std::string { + return S3Name::asString() + path_.asString(); +} + +void S3ObjectName::print(std::ostream& out) const { + out << "S3ObjectName["; + S3Name::print(out); + out << ',' << path_ << ']'; +} + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3ObjectName::size() const -> long long { + return client().objectSize(path_); +} + +auto S3ObjectName::exists() const -> bool { + return client().objectExists(path_); +} + +auto S3ObjectName::bucketExists() const -> bool { + return client().bucketExists(path_.bucket); +} + +void S3ObjectName::remove() { + client().deleteObject(path_); +} + +auto S3ObjectName::put(const void* buffer, const long length) const -> long long { + return client().putObject(path_, buffer, length); +} + +auto S3ObjectName::get(void* buffer, const long offset, const long length) const -> long long { + return client().getObject(path_, buffer, offset, length); +} + +auto S3ObjectName::dataHandle() const -> DataHandle* { + return new S3Handle(*this); +} + +auto S3ObjectName::dataHandle(const Offset& offset) const -> DataHandle* { + return new S3Handle(*this, offset); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3ObjectName.h b/src/eckit/io/s3/S3ObjectName.h new file mode 100644 index 000000000..4699ba8b5 --- /dev/null +++ b/src/eckit/io/s3/S3ObjectName.h @@ -0,0 +1,83 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +/// @file S3ObjectName.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#pragma once + +#include "eckit/io/s3/S3Name.h" +#include "eckit/io/s3/S3ObjectPath.h" + +#include +#include + +namespace eckit { + +class Offset; +class DataHandle; + +//---------------------------------------------------------------------------------------------------------------------- + +class S3ObjectName : public S3Name { +public: // factory + static auto parse(const std::string& name) -> S3ObjectPath; + +public: // methods + S3ObjectName(const net::Endpoint& endpoint, S3ObjectPath path); + + explicit S3ObjectName(const URI& uri); + + // accessors + + auto uri() const -> URI override; + + auto exists() const -> bool override; + + auto asString() const -> std::string override; + + auto size() const -> long long; + + auto path() const -> const S3ObjectPath& { return path_; } + + auto bucketExists() const -> bool; + + // modifiers + + void remove(); + + // I/O + + auto put(const void* buffer, long length) const -> long long; + + auto get(void* buffer, long offset, long length) const -> long long; + + [[nodiscard]] + auto dataHandle() const -> DataHandle*; + + [[nodiscard]] + auto dataHandle(const Offset& offset) const -> DataHandle*; + +private: // methods + void print(std::ostream& out) const override; + +private: // members + S3ObjectPath path_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3ObjectPath.h b/src/eckit/io/s3/S3ObjectPath.h new file mode 100644 index 000000000..2fbc60d92 --- /dev/null +++ b/src/eckit/io/s3/S3ObjectPath.h @@ -0,0 +1,55 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the Horizon Europe programme funded project DaFab + * (Grant agreement: 101128693) https://www.dafab-ai.eu/ + */ + +/// @file S3ObjectPath.h +/// @author Metin Cakircali +/// @date Dec 2024 + +#pragma once + +#include +#include +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +struct S3ObjectPath { + + S3ObjectPath(std::string bucket, std::string object) : bucket {std::move(bucket)}, object {std::move(object)} { } + + auto asString() const -> std::string { return bucket + '/' + object; } + + operator std::string() const { return asString(); } + + bool operator==(const S3ObjectPath& other) const { return bucket == other.bucket && object == other.object; } + + bool operator!=(const S3ObjectPath& other) const { return !(*this == other); } + + friend std::ostream& operator<<(std::ostream& out, const S3ObjectPath& path) { + out << "bucket=" << path.bucket << ",object=" << path.object; + return out; + } + + // members + + std::string bucket; + std::string object; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Session.cc b/src/eckit/io/s3/S3Session.cc new file mode 100644 index 000000000..6bf1c6387 --- /dev/null +++ b/src/eckit/io/s3/S3Session.cc @@ -0,0 +1,148 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +#include "eckit/io/s3/S3Session.h" + +#include "eckit/config/LibEcKit.h" +#include "eckit/io/s3/S3Credential.h" +#include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/aws/S3ClientAWS.h" +#include "eckit/log/CodeLocation.h" +#include "eckit/log/Log.h" +#include "eckit/net/Endpoint.h" +#include "eckit/runtime/Main.h" + +#include +#include +#include + +namespace eckit { + +namespace { + +/// @brief Predicate to find a client or credential by its endpoint +struct EndpointMatch { + const net::Endpoint& endpoint_; + + bool operator()(const std::shared_ptr& cred) const { return cred->endpoint == endpoint_; } + + bool operator()(const std::shared_ptr& client) const { return client->config().endpoint == endpoint_; } +}; + +} // namespace + +//---------------------------------------------------------------------------------------------------------------------- + +S3Session& S3Session::instance() { + static S3Session session; + return session; +} + +//---------------------------------------------------------------------------------------------------------------------- + +S3Session::S3Session() { + if (!Main::ready()) { + // inform when called before Main::initialise + Log::debug() << "Skipping initialization of S3Session instance.\n"; + return; + } + loadClients(); + loadCredentials(); +} + +S3Session::~S3Session() = default; + +//---------------------------------------------------------------------------------------------------------------------- + +void S3Session::clear() { + Log::debug() << "Clearing S3 clients and credentials.\n"; + clients_.clear(); + credentials_.clear(); +} + +//---------------------------------------------------------------------------------------------------------------------- +// CLIENT + +void S3Session::loadClients(const std::string& path) { + const auto configs = S3Config::make(path); + for (const auto& config : configs) { addClient(config); } +} + +auto S3Session::findClient(const net::Endpoint& endpoint) const -> std::shared_ptr { + + auto client = std::find_if(clients_.begin(), clients_.end(), EndpointMatch {endpoint}); + + if (client != clients_.end()) { return *client; } + + // not found + return {}; +} + +auto S3Session::getClient(const net::Endpoint& endpoint) const -> std::shared_ptr { + + if (auto client = findClient(endpoint)) { return client; } + + throw S3EntityNotFound("Could not find the client for " + std::string(endpoint), Here()); +} + +auto S3Session::addClient(const S3Config& config) -> std::shared_ptr { + if (auto client = findClient(config.endpoint)) { return client; } + // not found, add new item + return clients_.emplace_back(S3Client::makeShared(config)); +} + +void S3Session::removeClient(const net::Endpoint& endpoint) { + clients_.remove_if(EndpointMatch {endpoint}); +} + +//---------------------------------------------------------------------------------------------------------------------- +// CREDENTIALS + +void S3Session::loadCredentials(const std::string& path) { + const auto creds = S3Credential::fromFile(path); + for (const auto& cred : creds) { addCredential(cred); } +} + +auto S3Session::findCredential(const net::Endpoint& endpoint) const -> std::shared_ptr { + + auto cred = std::find_if(credentials_.begin(), credentials_.end(), EndpointMatch {endpoint}); + + if (cred != credentials_.end()) { return *cred; } + + // not found + return {}; +} + +auto S3Session::getCredential(const net::Endpoint& endpoint) const -> std::shared_ptr { + + if (auto cred = findCredential(endpoint)) { return cred; } + + throw S3EntityNotFound("Could not find the credential for " + std::string(endpoint), Here()); +} + +auto S3Session::addCredential(const S3Credential& credential) -> std::shared_ptr { + // don't add duplicate credentials + if (auto cred = findCredential(credential.endpoint)) { return cred; } + // not found: add new item + return credentials_.emplace_back(std::make_shared(credential)); +} + +void S3Session::removeCredential(const net::Endpoint& endpoint) { + credentials_.remove_if(EndpointMatch {endpoint}); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3Session.h b/src/eckit/io/s3/S3Session.h new file mode 100644 index 000000000..50a042b3a --- /dev/null +++ b/src/eckit/io/s3/S3Session.h @@ -0,0 +1,98 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +/// @file S3Session.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#pragma once + +#include +#include +#include + +namespace eckit { + +class S3Client; +struct S3Credential; +struct S3Config; + +namespace net { +class Endpoint; +} + +//---------------------------------------------------------------------------------------------------------------------- + +class S3Session final { +public: // methods + S3Session(const S3Session&) = delete; + S3Session& operator=(const S3Session&) = delete; + S3Session(S3Session&&) = delete; + S3Session& operator=(S3Session&&) = delete; + + static S3Session& instance(); + + void clear(); + + // clients + + void loadClients(const std::string& path = ""); + + /// @brief Get an S3 client for the given configuration + /// @param config S3 configuration + /// @return S3 client + /// @throws S3EntityNotFound if the client does not exist + [[nodiscard]] + auto getClient(const net::Endpoint& endpoint) const -> std::shared_ptr; + + /// @brief Add an S3 client for the given configuration + /// @param config S3 configuration + /// @return S3 client + auto addClient(const S3Config& config) -> std::shared_ptr; + + void removeClient(const net::Endpoint& endpoint); + + // credentials + + void loadCredentials(const std::string& path = ""); + + /// @brief Get an S3 credential for the given configuration + /// @param endpoint S3 endpoint + /// @return S3 credential + /// @throws S3EntityNotFound if the credential does not exist + [[nodiscard]] + auto getCredential(const net::Endpoint& endpoint) const -> std::shared_ptr; + + auto addCredential(const S3Credential& credential) -> std::shared_ptr; + + void removeCredential(const net::Endpoint& endpoint); + +private: // methods + S3Session(); + + ~S3Session(); + + auto findClient(const net::Endpoint& endpoint) const -> std::shared_ptr; + + auto findCredential(const net::Endpoint& endpoint) const -> std::shared_ptr; + +private: // members + std::list> clients_; + std::list> credentials_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3URIManager.cc b/src/eckit/io/s3/S3URIManager.cc new file mode 100644 index 000000000..c8749f56a --- /dev/null +++ b/src/eckit/io/s3/S3URIManager.cc @@ -0,0 +1,55 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +#include "eckit/io/s3/S3URIManager.h" + +#include "eckit/filesystem/URIManager.h" +#include "eckit/io/Length.h" +#include "eckit/io/Offset.h" +#include "eckit/io/s3/S3ObjectName.h" + +#include + +namespace eckit { + +static S3URIManager manager_s3("s3"); + +//---------------------------------------------------------------------------------------------------------------------- + +S3URIManager::S3URIManager(const std::string& name) : URIManager(name) { } + +bool S3URIManager::exists(const URI& uri) { + return S3ObjectName(uri).exists(); +} + +DataHandle* S3URIManager::newWriteHandle(const URI& uri) { + return S3ObjectName(uri).dataHandle(); +} + +DataHandle* S3URIManager::newReadHandle(const URI& uri) { + return S3ObjectName(uri).dataHandle(); +} + +DataHandle* S3URIManager::newReadHandle(const URI& uri, const OffsetList& /*offsets*/, const LengthList& /*lengths*/) { + return S3ObjectName(uri).dataHandle(); +} + +std::string S3URIManager::asString(const URI& uri) const { + return S3ObjectName(uri).asString(); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/S3URIManager.h b/src/eckit/io/s3/S3URIManager.h new file mode 100644 index 000000000..9b3dc5e1a --- /dev/null +++ b/src/eckit/io/s3/S3URIManager.h @@ -0,0 +1,49 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +/// @file S3URIManager.h +/// @author Simon Smart +/// @author Metin Cakircali +/// @date Feb 2024 + +#pragma once + +#include "eckit/filesystem/URIManager.h" + +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +class S3URIManager : public URIManager { +public: // methods + explicit S3URIManager(const std::string& name); + + bool authority() override { return true; } + +private: // methods + bool exists(const URI& uri) override; + + DataHandle* newWriteHandle(const URI& uri) override; + DataHandle* newReadHandle(const URI& uri) override; + DataHandle* newReadHandle(const URI& uri, const OffsetList& offsets, const LengthList& lengths) override; + + std::string asString(const URI& uri) const override; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/aws/S3ClientAWS.cc b/src/eckit/io/s3/aws/S3ClientAWS.cc new file mode 100644 index 000000000..5ccfc943b --- /dev/null +++ b/src/eckit/io/s3/aws/S3ClientAWS.cc @@ -0,0 +1,364 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +#include "eckit/io/s3/aws/S3ClientAWS.h" + +#include "eckit/config/LibEcKit.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/io/s3/S3BucketPath.h" +#include "eckit/io/s3/S3Credential.h" +#include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/S3ObjectPath.h" +#include "eckit/io/s3/S3Session.h" +#include "eckit/io/s3/aws/S3ContextAWS.h" +#include "eckit/log/CodeLocation.h" +#include "eckit/log/Log.h" +#include "eckit/net/IPAddress.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +//---------------------------------------------------------------------------------------------------------------------- + +namespace { + +inline std::string awsErrorMessage(const std::string& msg, const Aws::S3::S3Error& error) { + std::ostringstream oss; + oss << msg << " AWS: " << error.GetMessage() << " Remote IP: " << error.GetRemoteHostIpAddress(); + return oss.str(); +} + +class BufferIOStream : public Aws::IOStream { +public: + BufferIOStream(void* buffer, const uint64_t length) + : Aws::IOStream(new Aws::Utils::Stream::PreallocatedStreamBuf(reinterpret_cast(buffer), length)) { + } + + ~BufferIOStream() override { delete rdbuf(); } +}; + +} // namespace + +//---------------------------------------------------------------------------------------------------------------------- + +namespace eckit { + +const auto allocTag = "S3ClientAWS"; + +S3ClientAWS::S3ClientAWS(const S3Config& config) : S3Client(config), ctx_ {S3ContextAWS::instance()} { } + +//---------------------------------------------------------------------------------------------------------------------- + +void S3ClientAWS::configure() const { + + LOG_DEBUG_LIB(LibEcKit) << "Configure S3 AWS client... "; + + Aws::Client::ClientConfigurationInitValues initVal; + initVal.shouldDisableIMDS = true; + Aws::Client::ClientConfiguration configAWS(initVal); + + // we are not an ec2 instance + configAWS.disableIMDS = true; + configAWS.disableImdsV1 = true; + + // setup region + if (!config().region.empty()) { configAWS.region = config().region; } + + // configuration.scheme = Aws::Http::Scheme::HTTPS; + configAWS.verifySSL = false; + + // setup endpoint + if (config().endpoint.host().empty()) { throw UserError("Empty endpoint hostname in configuration!", Here()); } + + configAWS.endpointOverride = net::IPAddress::hostAddress(config().endpoint.host()).asString(); + + ASSERT(config().endpoint.port() > 0); + configAWS.endpointOverride += ":" + std::to_string(config().endpoint.port()); + + LOG_DEBUG_LIB(LibEcKit) << "endpoint=" << configAWS.endpointOverride << std::endl; + + const auto cred = S3Session::instance().getCredential(config().endpoint); + // credentials provider + auto cProvider = Aws::MakeShared(allocTag, cred->keyID, cred->secret); + // endpoint provider + auto eProvider = Aws::MakeShared(allocTag); + // client + client_ = std::make_unique(cProvider, eProvider, configAWS); +} + +auto S3ClientAWS::client() const -> Aws::S3::S3Client& { + if (!client_) { configure(); } + if (!client_) { throw S3SeriousBug("Invalid client!", Here()); } + return *client_; +} + +//---------------------------------------------------------------------------------------------------------------------- +// BUCKET + +void S3ClientAWS::createBucket(const S3BucketPath& path) const { + Aws::S3::Model::CreateBucketRequest request; + request.SetBucket(path); + + auto outcome = client().CreateBucket(request); + + if (!outcome.IsSuccess()) { + const auto& err = outcome.GetError(); + const auto msg = awsErrorMessage("Failed to create bucket=" + path.asString(), err); + const auto eType = err.GetErrorType(); + if (eType == Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS || eType == Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU) { + throw S3EntityAlreadyExists(msg, Here()); + } + throw S3SeriousBug(msg, Here()); + } + + /// @todo do we wait for the bucket to propagate? + + LOG_DEBUG_LIB(LibEcKit) << "Created bucket=" << path << std::endl; +} + +void S3ClientAWS::emptyBucket(const S3BucketPath& path) const { + deleteObjects(path, listObjects(path)); +} + +//---------------------------------------------------------------------------------------------------------------------- + +void S3ClientAWS::deleteBucket(const S3BucketPath& path) const { + Aws::S3::Model::DeleteBucketRequest request; + request.SetBucket(path); + + auto outcome = client().DeleteBucket(request); + + if (!outcome.IsSuccess()) { + const auto& err = outcome.GetError(); + const auto msg = awsErrorMessage("Failed to delete bucket=" + path.asString(), err); + if (err.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_BUCKET) { throw S3EntityNotFound(msg, Here()); } + if (err.GetErrorType() == Aws::S3::S3Errors::UNKNOWN && err.GetExceptionName() == "BucketNotEmpty") { + throw S3BucketNotEmpty(msg, Here()); + } + throw S3SeriousBug(msg, Here()); + } + + LOG_DEBUG_LIB(LibEcKit) << "Deleted bucket=" << path << std::endl; +} + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3ClientAWS::bucketExists(const S3BucketPath& path) const -> bool { + Aws::S3::Model::HeadBucketRequest request; + + request.SetBucket(path); + + return client().HeadBucket(request).IsSuccess(); +} + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3ClientAWS::listBuckets() const -> std::vector { + auto outcome = client().ListBuckets(); + + if (outcome.IsSuccess()) { + std::vector buckets; + for (const auto& bucket : outcome.GetResult().GetBuckets()) { buckets.emplace_back(bucket.GetName()); } + return buckets; + } + + auto msg = awsErrorMessage("Failed to list buckets!", outcome.GetError()); + throw S3SeriousBug(msg, Here()); +} + +//---------------------------------------------------------------------------------------------------------------------- +// PUT OBJECT + +auto S3ClientAWS::putObject(const S3ObjectPath& path, const void* buffer, const uint64_t length) const -> long long { + Aws::S3::Model::PutObjectRequest request; + + request.SetBucket(path.bucket); + request.SetKey(path.object); + // request.SetContentLength(length); + + if (buffer && length > 0) { + auto streamBuffer = Aws::MakeShared(allocTag, const_cast(buffer), length); + request.SetBody(streamBuffer); + } else { + // empty object + request.SetBody(Aws::MakeShared(allocTag)); + } + + auto outcome = client().PutObject(request); + + if (outcome.IsSuccess()) { + LOG_DEBUG_LIB(LibEcKit) << "Put " << path << " with len=" << length << '\n'; + /// @todo actual size of written bytes + return length; + } + + auto msg = awsErrorMessage("Failed to put " + std::string(path), outcome.GetError()); + throw S3SeriousBug(msg, Here()); +} + +//---------------------------------------------------------------------------------------------------------------------- +// GET OBJECT + +auto S3ClientAWS::getObject(const S3ObjectPath& path, + void* buffer, + const uint64_t /*offset*/, + const uint64_t length) const -> long long { + Aws::S3::Model::GetObjectRequest request; + + request.SetBucket(path.bucket); + request.SetKey(path.object); + request.SetResponseStreamFactory([&buffer, length]() { return Aws::New(allocTag, buffer, length); }); + /// @todo range and streambuf + // request.SetRange(std::to_string(offset) + "-" + std::to_string(offset + length)); + + auto outcome = client().GetObject(request); + + if (outcome.IsSuccess()) { + LOG_DEBUG_LIB(LibEcKit) << "Get " << path << '\n'; + LOG_DEBUG_LIB(LibEcKit) << "Req. len=" << length << ", Obj. len=" << outcome.GetResult().GetContentLength() + << '\n'; + return outcome.GetResult().GetContentLength(); + } + + auto msg = awsErrorMessage("Failed to retrieve " + std::string(path), outcome.GetError()); + throw S3SeriousBug(msg, Here()); +} + +//---------------------------------------------------------------------------------------------------------------------- +// DELETE OBJECT + +void S3ClientAWS::deleteObject(const S3ObjectPath& path) const { + Aws::S3::Model::DeleteObjectRequest request; + + request.SetBucket(path.bucket); + request.SetKey(path.object); + + auto outcome = client().DeleteObject(request); + + if (!outcome.IsSuccess()) { + auto msg = awsErrorMessage("Failed to delete " + std::string(path), outcome.GetError()); + throw S3SeriousBug(msg, Here()); + } + + LOG_DEBUG_LIB(LibEcKit) << "Deleted " << path << '\n'; +} + +//---------------------------------------------------------------------------------------------------------------------- + +void S3ClientAWS::deleteObjects(const S3BucketPath& path, const std::vector& objects) const { + if (objects.empty()) { return; } + + Aws::S3::Model::DeleteObjectsRequest request; + + request.SetBucket(path); + + Aws::S3::Model::Delete deleteObject; + for (const auto& object : objects) { deleteObject.AddObjects(Aws::S3::Model::ObjectIdentifier().WithKey(object)); } + // deleteObject.SetQuiet(true); + + request.SetDelete(deleteObject); + + auto outcome = client().DeleteObjects(request); + + if (!outcome.IsSuccess()) { + auto msg = awsErrorMessage("Failed to delete objects in bucket=" + path.asString(), outcome.GetError()); + throw S3SeriousBug(msg, Here()); + } + + LOG_DEBUG_LIB(LibEcKit) << "Deleted " << objects.size() << " objects in bucket=" << path << std::endl; + for (const auto& object : objects) { LOG_DEBUG_LIB(LibEcKit) << "Deleted object=" << object << std::endl; } +} + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3ClientAWS::listObjects(const S3BucketPath& path) const -> std::vector { + Aws::S3::Model::ListObjectsV2Request request; + + request.SetBucket(path); + + auto outcome = client().ListObjectsV2(request); + + if (!outcome.IsSuccess()) { + const auto& err = outcome.GetError(); + const auto msg = awsErrorMessage("Failed to list objects in bucket=" + path.asString(), err); + if (err.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_BUCKET) { throw S3EntityNotFound(msg, Here()); } + throw S3SeriousBug(msg, Here()); + } + + std::vector result; + + const auto& objects = outcome.GetResult().GetContents(); + for (const auto& object : objects) { result.emplace_back(object.GetKey()); } + + return result; +} + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3ClientAWS::objectExists(const S3ObjectPath& path) const -> bool { + Aws::S3::Model::HeadObjectRequest request; + + request.SetBucket(path.bucket); + request.SetKey(path.object); + + return client().HeadObject(request).IsSuccess(); +} + +//---------------------------------------------------------------------------------------------------------------------- + +auto S3ClientAWS::objectSize(const S3ObjectPath& path) const -> long long { + Aws::S3::Model::HeadObjectRequest request; + + request.SetBucket(path.bucket); + request.SetKey(path.object); + + auto outcome = client().HeadObject(request); + + if (outcome.IsSuccess()) { return outcome.GetResult().GetContentLength(); } + + const auto msg = awsErrorMessage("Object '" + path.object + "' doesn't exist or no access!", outcome.GetError()); + throw S3SeriousBug(msg, Here()); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/aws/S3ClientAWS.h b/src/eckit/io/s3/aws/S3ClientAWS.h new file mode 100644 index 000000000..e8097224a --- /dev/null +++ b/src/eckit/io/s3/aws/S3ClientAWS.h @@ -0,0 +1,85 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +/// @file S3ClientAWS.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#pragma once + +#include "eckit/io/s3/S3BucketPath.h" +#include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3ObjectPath.h" + +#include + +#include +#include +#include +#include + +namespace eckit { + +class S3ContextAWS; +struct S3Config; + +//---------------------------------------------------------------------------------------------------------------------- + +class S3ClientAWS final : public S3Client { +public: // methods + explicit S3ClientAWS(const S3Config& config); + + // bucket operations + + void createBucket(const S3BucketPath& path) const override; + + void emptyBucket(const S3BucketPath& path) const override; + + void deleteBucket(const S3BucketPath& path) const override; + + auto bucketExists(const S3BucketPath& path) const -> bool override; + + auto listBuckets() const -> std::vector override; + + auto listObjects(const S3BucketPath& path) const -> std::vector override; + + // object operations + + auto putObject(const S3ObjectPath& path, const void* buffer, uint64_t length) const -> long long override; + + auto getObject(const S3ObjectPath& path, void* buffer, uint64_t offset, uint64_t length) const -> long long override; + + void deleteObject(const S3ObjectPath& path) const override; + + void deleteObjects(const S3BucketPath& path, const std::vector& objects) const override; + + auto objectExists(const S3ObjectPath& path) const -> bool override; + + auto objectSize(const S3ObjectPath& path) const -> long long override; + +private: // methods + void configure() const; + + auto client() const -> Aws::S3::S3Client&; + +private: // members + const S3ContextAWS& ctx_; + + mutable std::unique_ptr client_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/aws/S3ContextAWS.cc b/src/eckit/io/s3/aws/S3ContextAWS.cc new file mode 100644 index 000000000..bdcff27e7 --- /dev/null +++ b/src/eckit/io/s3/aws/S3ContextAWS.cc @@ -0,0 +1,41 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +#include "eckit/io/s3/aws/S3ContextAWS.h" + +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +S3ContextAWS& S3ContextAWS::instance() { + static S3ContextAWS ctx; + return ctx; +} + +//---------------------------------------------------------------------------------------------------------------------- + +S3ContextAWS::S3ContextAWS() { + Aws::InitAPI(options_); +} + +S3ContextAWS::~S3ContextAWS() { + Aws::ShutdownAPI(options_); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/s3/aws/S3ContextAWS.h b/src/eckit/io/s3/aws/S3ContextAWS.h new file mode 100644 index 000000000..cce8d281d --- /dev/null +++ b/src/eckit/io/s3/aws/S3ContextAWS.h @@ -0,0 +1,49 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the EC H2020 funded project IO-SEA + * (Project ID: 955811) iosea-project.eu + */ + +/// @file S3ContextAWS.h +/// @author Metin Cakircali +/// @date Jan 2024 + +#pragma once + +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +/// @brief Provides singleton for AWS API init/shutdown +class S3ContextAWS { +public: // methods + S3ContextAWS(const S3ContextAWS&) = delete; + S3ContextAWS& operator=(const S3ContextAWS&) = delete; + S3ContextAWS(S3ContextAWS&&) = delete; + S3ContextAWS& operator=(S3ContextAWS&&) = delete; + + static S3ContextAWS& instance(); + +private: // methods + explicit S3ContextAWS(); + + ~S3ContextAWS(); + +private: // members + Aws::SDKOptions options_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/net/Endpoint.cc b/src/eckit/net/Endpoint.cc index 8be1ff83d..ad58f976e 100644 --- a/src/eckit/net/Endpoint.cc +++ b/src/eckit/net/Endpoint.cc @@ -10,17 +10,22 @@ #include "eckit/net/Endpoint.h" -#include - #include "eckit/exception/Exceptions.h" +#include "eckit/filesystem/URI.h" #include "eckit/serialisation/Stream.h" #include "eckit/utils/Tokenizer.h" #include "eckit/utils/Translator.h" +#include + namespace eckit::net { //---------------------------------------------------------------------------------------------------------------------- +Endpoint::Endpoint(const URI& uri): host_(uri.host()), port_(uri.port()) { + validate(); +} + Endpoint::Endpoint(const std::string& s) { Tokenizer tokenize(":"); std::vector tokens; diff --git a/src/eckit/net/Endpoint.h b/src/eckit/net/Endpoint.h index fa53731c7..6cb300238 100644 --- a/src/eckit/net/Endpoint.h +++ b/src/eckit/net/Endpoint.h @@ -21,6 +21,7 @@ namespace eckit { class Stream; +class URI; namespace net { @@ -29,6 +30,7 @@ namespace net { class Endpoint { public: // methods + Endpoint(const URI& uri); // gets hostname:port from uri Endpoint(const std::string&); // parses the std::string formated as hostname:port Endpoint(const std::string& host, int port); Endpoint(Stream& s); diff --git a/src/eckit/value/StringContent.cc b/src/eckit/value/StringContent.cc index a39496af4..6f79021ac 100644 --- a/src/eckit/value/StringContent.cc +++ b/src/eckit/value/StringContent.cc @@ -66,7 +66,7 @@ int StringContent::compare(const Content& other) const { } int StringContent::compareString(const StringContent& other) const { - return ::strcmp(value_.c_str(), other.value_.c_str()); + return value_.compare(other.value_); } void StringContent::value(std::string& s) const { diff --git a/tests/io/CMakeLists.txt b/tests/io/CMakeLists.txt index b2a35896e..a567332d4 100644 --- a/tests/io/CMakeLists.txt +++ b/tests/io/CMakeLists.txt @@ -80,3 +80,5 @@ ecbuild_add_test( TARGET eckit_rados-performance INCLUDES ${RADOS_INCLUDE_DIRS} TEST_DEPENDS get_eckit_io_test_data LIBS eckit ) + +add_subdirectory( s3 ) diff --git a/tests/io/s3/CMakeLists.txt b/tests/io/s3/CMakeLists.txt new file mode 100644 index 000000000..405059c16 --- /dev/null +++ b/tests/io/s3/CMakeLists.txt @@ -0,0 +1,30 @@ + +if( eckit_HAVE_S3 ) + + set( S3_TEST_HOST "minio" ) + set( S3_TEST_PORT "9000" ) + set( S3_TEST_ENDPOINT ${S3_TEST_HOST}:${S3_TEST_PORT} ) + set( S3_TEST_REGION "eu-central-1" ) + set( S3_TEST_BUCKET "eckit-test-s3-bucket" ) + set( S3_TEST_OBJECT "eckit-test-s3-object" ) + + configure_file( test_s3_config.h.in test_s3_config.h @ONLY ) + + configure_file( S3Config.yaml.in S3Config.yaml @ONLY ) + configure_file( S3Credentials.yaml.in S3Credentials.yaml @ONLY ) + + set( test_srcs client handle ) + + foreach( _test ${test_srcs} ) + + ecbuild_add_test( + TARGET eckit_test_s3_${_test} + SOURCES test_s3_${_test}.cc + INCLUDES ${CMAKE_CURRENT_BINARY_DIR} + LIBS eckit + ENVIRONMENT "${_test_environment}" + ) + + endforeach() + +endif( eckit_HAVE_S3 ) diff --git a/tests/io/s3/S3Config.yaml.in b/tests/io/s3/S3Config.yaml.in new file mode 100644 index 000000000..217dfb2d9 --- /dev/null +++ b/tests/io/s3/S3Config.yaml.in @@ -0,0 +1,17 @@ +--- +servers: + - endpoint: @S3_TEST_ENDPOINT@ + region: @S3_TEST_REGION@ + + - endpoint: "127.0.0.1:9000" + region: "default" # (default) + secure: true # (default) + backend: "AWS" # (default) + + - endpoint: "localhost:9000" + region: "eu-central-1" + secure: true # (default) + backend: "AWS" # (default) + + # region is inferred from the endpoint + - endpoint: "eu-central-1.ecmwf.int:9000" diff --git a/tests/io/s3/S3Credentials.yaml.in b/tests/io/s3/S3Credentials.yaml.in new file mode 100644 index 000000000..3269ce3b7 --- /dev/null +++ b/tests/io/s3/S3Credentials.yaml.in @@ -0,0 +1,13 @@ +--- +credentials: + - endpoint: @S3_TEST_ENDPOINT@ + accessKeyID: minio + secretKey: minio1234 + + - endpoint: localhost:9000 + accessKeyID: asd1 + secretKey: asd1 + + - endpoint: 127.0.0.1:9000 + accessKeyID: asd2 + secretKey: asd2 diff --git a/tests/io/s3/test_s3_client.cc b/tests/io/s3/test_s3_client.cc new file mode 100644 index 000000000..142e1ac89 --- /dev/null +++ b/tests/io/s3/test_s3_client.cc @@ -0,0 +1,193 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @file test_s3client.cc +/// @author Metin Cakircali +/// @author Simon Smart +/// @date Jan 2024 + +#include "eckit/io/s3/S3BucketPath.h" +#include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3Config.h" +#include "eckit/io/s3/S3Exception.h" +#include "eckit/io/s3/S3Session.h" +#include "eckit/net/Endpoint.h" +#include "eckit/testing/Test.h" +#include "test_s3_config.h" + +#include +#include + +using namespace eckit; +using namespace eckit::testing; + +// this test requires a working S3 endpoint and credentials. such as, a local docker container MinIO instance + +namespace eckit::test { + +//---------------------------------------------------------------------------------------------------------------------- + +const std::vector testBuckets {"test-bucket-1", "test-bucket-2"}; + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("s3 client: API") { + + EXPECT(S3Session::instance().addClient(s3::TEST_CONFIG)); + EXPECT(S3Session::instance().addCredential(s3::TEST_CRED)); + + EXPECT_NO_THROW(s3::cleanup(testBuckets)); + + EXPECT_NO_THROW(S3Session::instance().removeClient(s3::TEST_ENDPOINT)); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("s3 client: read from file") { + + auto& session = S3Session::instance(); + + session.clear(); + + EXPECT_THROWS_AS(session.getClient({"127.0.0.1", 9000}), S3EntityNotFound); + EXPECT_THROWS_AS(session.getClient({"minio", 9000}), S3EntityNotFound); + EXPECT_THROWS_AS(session.getClient({"localhost", 9000}), S3EntityNotFound); + + session.loadClients("./S3Config.yaml"); + + EXPECT_NO_THROW(session.getClient({"127.0.0.1", 9000})); + EXPECT_NO_THROW(session.getClient({"minio", 9000})); + EXPECT_NO_THROW(session.getClient({"localhost", 9000})); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("s3 credentials: API") { + + EXPECT(S3Session::instance().addCredential(s3::TEST_CRED)); + + EXPECT_NO_THROW(s3::cleanup(testBuckets)); + + EXPECT_NO_THROW(S3Session::instance().removeCredential(s3::TEST_ENDPOINT)); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("s3 credentials: read from file") { + + auto& session = S3Session::instance(); + + session.clear(); + + EXPECT_THROWS_AS(session.getCredential({"127.0.0.1", 9000}), S3EntityNotFound); + EXPECT_THROWS_AS(session.getCredential({"minio", 9000}), S3EntityNotFound); + EXPECT_THROWS_AS(session.getCredential({"localhost", 9000}), S3EntityNotFound); + + session.loadCredentials("./S3Credentials.yaml"); + + EXPECT_NO_THROW(session.getCredential({"127.0.0.1", 9000})); + EXPECT_NO_THROW(session.getCredential({"minio", 9000})); + EXPECT_NO_THROW(session.getCredential({"localhost", 9000})); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("s3 backends") { + S3Config cfgTmp(s3::TEST_CONFIG); + + cfgTmp.backend = S3Backend::AWS; + EXPECT_NO_THROW(S3Client::makeUnique(cfgTmp)); + + cfgTmp.backend = S3Backend::REST; + EXPECT_THROWS(S3Client::makeUnique(cfgTmp)); + + cfgTmp.backend = S3Backend::MINIO; + EXPECT_THROWS(S3Client::makeUnique(cfgTmp)); +} + +// CASE("wrong credentials") { +// ensureClean(); +// +// S3Config cfgTmp(cfg); +// cfgTmp.region = "no-region-random"; +// +// EXPECT_THROWS(S3Client::makeUnique(cfgTmp)->createBucket("failed-bucket")); +// } + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("create s3 bucket in non-existing region") { + EXPECT_NO_THROW(s3::cleanup(testBuckets)); + + // this test requires an S3 endpoint that sets it's region + // a MinIO instance with empty region will not throw an exception + + auto cfgTmp = s3::TEST_CONFIG; + cfgTmp.region = "non-existing-region-random"; + + EXPECT_THROWS(S3Client::makeUnique(cfgTmp)->createBucket({"test-bucket-1"})); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("create s3 bucket") { + EXPECT_NO_THROW(s3::cleanup(testBuckets)); + + auto client = S3Client::makeUnique(s3::TEST_CONFIG); + + EXPECT_NO_THROW(client->createBucket({"test-bucket-1"})); + + EXPECT_THROWS(client->createBucket({"test-bucket-1"})); + + EXPECT_NO_THROW(client->createBucket({"test-bucket-2"})); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("list s3 buckets") { + EXPECT_NO_THROW(s3::cleanup(testBuckets)); + + auto client = S3Client::makeUnique(s3::TEST_CONFIG); + EXPECT_NO_THROW(client->createBucket({"test-bucket-1"})); + EXPECT_NO_THROW(client->createBucket({"test-bucket-2"})); + + { + const auto buckets = client->listBuckets(); + + EXPECT(s3::findString(buckets, "test-bucket-1")); + EXPECT(s3::findString(buckets, {"test-bucket-2"})); + + EXPECT_NO_THROW(client->deleteBucket({"test-bucket-1"})); + EXPECT_NO_THROW(client->deleteBucket({"test-bucket-2"})); + } + + { + const auto buckets = client->listBuckets(); + EXPECT_NOT(s3::findString(buckets, "test-bucket-1")); + EXPECT_NOT(s3::findString(buckets, {"test-bucket-2"})); + } + + EXPECT_THROWS(client->deleteBucket({"test-bucket-1"})); + EXPECT_THROWS(client->deleteBucket({"test-bucket-2"})); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit::test + +int main(int argc, char** argv) { + int ret = -1; + + ret = run_tests(argc, argv); + + test::s3::cleanup(test::testBuckets); + + return ret; +} diff --git a/tests/io/s3/test_s3_config.h.in b/tests/io/s3/test_s3_config.h.in new file mode 100644 index 000000000..aedef16a0 --- /dev/null +++ b/tests/io/s3/test_s3_config.h.in @@ -0,0 +1,79 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +#pragma once + +#include "eckit/io/Buffer.h" +#include "eckit/io/s3/S3BucketName.h" +#include "eckit/io/s3/S3BucketPath.h" +#include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3Config.h" +#include "eckit/io/s3/S3Credential.h" +#include "eckit/io/s3/S3ObjectName.h" +#include "eckit/log/Bytes.h" +#include "eckit/log/Timer.h" +#include "eckit/net/Endpoint.h" + +#include +#include +#include +#include + +#cmakedefine S3_TEST_HOST "@S3_TEST_HOST@" +#cmakedefine S3_TEST_PORT "@S3_TEST_PORT@" +#cmakedefine S3_TEST_ENDPOINT "@S3_TEST_ENDPOINT@" +#cmakedefine S3_TEST_REGION "@S3_TEST_REGION@" +#cmakedefine S3_TEST_BUCKET "@S3_TEST_BUCKET@" +#cmakedefine S3_TEST_OBJECT "@S3_TEST_OBJECT@" + +namespace eckit::test::s3 { + +//---------------------------------------------------------------------------------------------------------------------- + +const net::Endpoint TEST_ENDPOINT {S3_TEST_ENDPOINT}; + +const S3Config TEST_CONFIG {TEST_ENDPOINT, S3_TEST_REGION}; + +const S3Credential TEST_CRED {TEST_ENDPOINT, "minio", "minio1234"}; + +inline bool findString(const std::vector& list, const std::string& item) { + return (std::find(list.begin(), list.end(), item) != list.end()); +} + +inline void cleanup(const std::vector& bucketPaths) { + auto client = S3Client::makeUnique(TEST_CONFIG); + for (const auto& path : bucketPaths) { + if (client->bucketExists(path)) { + client->emptyBucket(path); + client->deleteBucket(path); + } + } +} + +inline void writePerformance(S3BucketName& bucket, const int count) { + eckit::Timer timer; + + Buffer buffer(1024 * 1024); + buffer.zero(); + + timer.start(); + for (int i = 0; i < count; i++) { + const auto objName = S3_TEST_OBJECT + std::to_string(i); + bucket.makeObject(objName)->put(buffer.data(), buffer.size()); + } + timer.stop(); + + std::cout << "Write performance: " << Bytes(buffer.size()) << " x " << count + << " objects, rate: " << Bytes(buffer.size() * 1000, timer) << std::endl; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit::test::s3 diff --git a/tests/io/s3/test_s3_handle.cc b/tests/io/s3/test_s3_handle.cc new file mode 100644 index 000000000..bf61df6df --- /dev/null +++ b/tests/io/s3/test_s3_handle.cc @@ -0,0 +1,306 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @file test_s3handle.cc +/// @author Metin Cakircali +/// @author Simon Smart +/// @date Jan 2024 + +#include "eckit/filesystem/PathName.h" +#include "eckit/filesystem/URI.h" +#include "eckit/io/MemoryHandle.h" +#include "eckit/io/s3/S3BucketName.h" +#include "eckit/io/s3/S3Client.h" +#include "eckit/io/s3/S3ObjectName.h" +#include "eckit/io/s3/S3Session.h" +#include "eckit/net/Endpoint.h" +#include "eckit/testing/Test.h" +#include "test_s3_config.h" + +#include + +#include +#include +#include +#include +#include + +using namespace std::string_literals; + +using namespace eckit; +using namespace eckit::testing; + +namespace eckit::test { + +//---------------------------------------------------------------------------------------------------------------------- + +constexpr std::string_view TEST_DATA {"abcdefghijklmnopqrstuvwxyz"}; + +const std::string TEST_BUCKET {"eckit-s3handle-test-bucket"}; +const std::string TEST_OBJECT {"eckit-s3handle-test-object"}; + +const std::vector testBuckets {TEST_BUCKET}; + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("initialize s3 session") { + + EXPECT(S3Session::instance().addCredential(s3::TEST_CRED)); + EXPECT(S3Session::instance().addClient(s3::TEST_CONFIG)); + + EXPECT_NO_THROW(s3::cleanup(testBuckets)); +} + +CASE("invalid s3 bucket") { + EXPECT_THROWS(S3BucketName(URI {"http://127.0.0.1:9000/" + TEST_BUCKET})); + EXPECT_THROWS(S3BucketName(URI {"s3://127.0.0.1" + TEST_BUCKET})); + EXPECT_THROWS(S3BucketName(URI {"s3://127.0.0.1/" + TEST_BUCKET})); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("S3BucketName: no bucket") { + EXPECT_NO_THROW(s3::cleanup(testBuckets)); + + S3BucketName bucket(s3::TEST_ENDPOINT, TEST_BUCKET); + + EXPECT_NOT(bucket.exists()); + + // LIST + EXPECT_THROWS(bucket.listObjects()); + + // CREATE OBJECT + EXPECT_THROWS(bucket.makeObject(TEST_OBJECT)->put(TEST_DATA.data(), TEST_DATA.size())); + + // DESTROY BUCKET + EXPECT_THROWS(bucket.destroy()); + EXPECT_NO_THROW(bucket.ensureDestroyed()); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("S3BucketName: create bucket") { + EXPECT_NO_THROW(s3::cleanup(testBuckets)); + + S3BucketName bucket(s3::TEST_ENDPOINT, TEST_BUCKET); + + EXPECT_NOT(bucket.exists()); + + // CREATE BUCKET + EXPECT_NO_THROW(bucket.create()); + EXPECT_THROWS(bucket.create()); + EXPECT_NO_THROW(bucket.ensureCreated()); + + // DESTROY BUCKET + EXPECT_NO_THROW(bucket.destroy()); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("S3BucketName: empty bucket") { + EXPECT_NO_THROW(s3::cleanup(testBuckets)); + + S3BucketName bucket(s3::TEST_ENDPOINT, TEST_BUCKET); + + // CREATE BUCKET + EXPECT_NO_THROW(bucket.ensureCreated()); + EXPECT(bucket.exists()); + + // LIST + EXPECT_EQUAL(bucket.listObjects().size(), 0); + + // DESTROY BUCKET + EXPECT_NO_THROW(bucket.destroy()); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("S3BucketName: bucket with object") { + EXPECT_NO_THROW(s3::cleanup(testBuckets)); + + S3BucketName bucket(s3::TEST_ENDPOINT, TEST_BUCKET); + + // CREATE BUCKET + EXPECT_NO_THROW(bucket.ensureCreated()); + + // CREATE OBJECT + EXPECT_NO_THROW(bucket.makeObject(TEST_OBJECT)->put(TEST_DATA.data(), TEST_DATA.size())); + + // LIST + EXPECT_EQUAL(bucket.listObjects().size(), 1); + + // DESTROY BUCKET + EXPECT_THROWS(bucket.destroy()); + EXPECT_NO_THROW(bucket.ensureDestroyed()); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("S3Handle: basic operations") { + EXPECT_NO_THROW(s3::cleanup(testBuckets)); + + S3Client::makeUnique(s3::TEST_CONFIG)->createBucket(TEST_BUCKET); + + const void* buffer = TEST_DATA.data(); + const long length = TEST_DATA.size(); + + const URI uri("s3://" + std::string(s3::TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); + + { + S3ObjectName object(uri); + + std::unique_ptr handle(object.dataHandle()); + EXPECT_NO_THROW(handle->openForWrite(length)); + EXPECT_EQUAL(handle->write(buffer, length), length); + EXPECT_NO_THROW(handle->close()); + } + + { + std::unique_ptr handle(uri.newReadHandle()); + EXPECT_NO_THROW(handle->openForRead()); + + std::string rbuf; + rbuf.resize(length); + + EXPECT_NO_THROW(handle->read(rbuf.data(), length)); + + EXPECT_EQUAL(rbuf, TEST_DATA); + + EXPECT_NO_THROW(handle->close()); + } + + { + std::unique_ptr handle(uri.newReadHandle()); + + MemoryHandle memHandle(length); + handle->saveInto(memHandle); + + EXPECT_EQUAL(memHandle.size(), Length(length)); + EXPECT_EQUAL(::memcmp(memHandle.data(), buffer, length), 0); + + EXPECT_NO_THROW(handle->close()); + } +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("S3Handle: openForRead") { + EXPECT_NO_THROW(s3::cleanup(testBuckets)); + + const URI uri("s3://" + std::string(s3::TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); + + EXPECT_NO_THROW(S3BucketName(uri).ensureCreated()); + + std::unique_ptr handle(uri.newReadHandle()); + + // NO OBJECT + EXPECT_THROWS(handle->openForRead()); + + // CREATE OBJECT + EXPECT_NO_THROW(S3ObjectName(uri).put(nullptr, 0)); + + // DOUBLE OPEN + EXPECT_THROWS(handle->openForRead()); + + // RE-OPEN + EXPECT_NO_THROW(handle->close()); + EXPECT_NO_THROW(handle->openForRead()); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("S3Handle: openForWrite") { + EXPECT_NO_THROW(s3::cleanup(testBuckets)); + + const URI uri("s3://" + std::string(s3::TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); + + { // NO BUCKET + std::unique_ptr handle(uri.newWriteHandle()); + EXPECT_THROWS(handle->openForWrite(0)); + EXPECT_NO_THROW(handle->close()); + } + + // CREATE BUCKET + EXPECT_NO_THROW(S3BucketName(uri).ensureCreated()); + + { // BUCKET EXISTS + std::unique_ptr handle(uri.newWriteHandle()); + EXPECT_NO_THROW(handle->openForWrite(0)); + EXPECT_NO_THROW(handle->close()); + } +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("S3Handle: read") { + EXPECT_NO_THROW(s3::cleanup(testBuckets)); + + const URI uri("s3://" + std::string(s3::TEST_ENDPOINT) + "/" + TEST_BUCKET + "/" + TEST_OBJECT); + + EXPECT_NO_THROW(S3BucketName(uri).ensureCreated()); + + // CREATE OBJECT + EXPECT_NO_THROW(S3ObjectName(uri).put(TEST_DATA.data(), TEST_DATA.size())); + + std::unique_ptr handle(uri.newReadHandle()); + + // OPEN + EXPECT_NO_THROW(handle->openForRead()); + + /// @todo range based read + // { + // const auto length = TEST_DATA.size(); + // + // std::string rbuf; + // rbuf.resize(length); + // auto len = handle->read(rbuf.data(), 10); + // std::cout << "========" << rbuf << std::endl; + // len = handle->read(rbuf.data(), length - 10); + // std::cout << "========" << rbuf << std::endl; + // EXPECT_EQUAL(rbuf, TEST_DATA); + // } + + // CLOSE + EXPECT_NO_THROW(handle->close()); +} + +//---------------------------------------------------------------------------------------------------------------------- + +CASE("s3 performance: write 1 10 100 objects") { + EXPECT_NO_THROW(s3::cleanup(testBuckets)); + + const URI uri("s3://" + std::string(s3::TEST_ENDPOINT) + "/" + TEST_BUCKET); + + S3BucketName bucket(uri); + + EXPECT_NO_THROW(bucket.ensureCreated()); + + s3::writePerformance(bucket, 1); + + s3::writePerformance(bucket, 10); + + s3::writePerformance(bucket, 100); +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit::test + +int main(int argc, char** argv) { + int ret = -1; + + ret = run_tests(argc, argv); + + test::s3::cleanup(test::testBuckets); + + return ret; +} + +//----------------------------------------------------------------------------------------------------------------------