diff --git a/duckdb b/duckdb index b390a7c..68d7555 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit b390a7c3760bd95926fe8aefde20d04b349b472e +Subproject commit 68d7555f68bd25c1a251ccca2e6338949c33986a diff --git a/extension-ci-tools b/extension-ci-tools index c098325..aac9640 160000 --- a/extension-ci-tools +++ b/extension-ci-tools @@ -1 +1 @@ -Subproject commit c098325d7e622b52747a0df810a8146ab10a9ab5 +Subproject commit aac9640615e51d6e7e8b72d4bf023703cfd8e479 diff --git a/src/block_manager.cpp b/src/block_manager.cpp index a204e94..f9a91af 100644 --- a/src/block_manager.cpp +++ b/src/block_manager.cpp @@ -300,13 +300,29 @@ void BlockManager::LoadFreeList() { void BlockManager::ValidateBlockId(block_id_t block_id) const { if (block_id == INVALID_BLOCK_ID) { - throw duckdb::InvalidInputException("Block ID cannot be INVALID_BLOCK_ID"); + throw duckdb::InvalidInputException( + "Block ID cannot be INVALID_BLOCK_ID", + duckdb::unordered_map{ + {"block_id", std::to_string(block_id)} + } + ); } if (block_id < 0) { - throw duckdb::InvalidInputException("Block ID cannot be negative", {{"block_id", std::to_string(block_id)}}); + throw duckdb::InvalidInputException( + "Block ID cannot be negative", + duckdb::unordered_map{ + {"block_id", std::to_string(block_id)} + } + ); } if (block_id >= max_block) { - throw duckdb::InvalidInputException("Block ID cannot exceed max_block", {{"block_id", std::to_string(block_id)}, {"max_block", std::to_string(max_block)}}); + throw duckdb::InvalidInputException( + "Block ID cannot exceed max_block", + duckdb::unordered_map{ + {"block_id", std::to_string(block_id)}, + {"max_block", std::to_string(max_block)} + } + ); } } diff --git a/src/include/quackstore_filesystem.hpp b/src/include/quackstore_filesystem.hpp index 6a6b435..0359f56 100644 --- a/src/include/quackstore_filesystem.hpp +++ b/src/include/quackstore_filesystem.hpp @@ -29,6 +29,22 @@ class QuackstoreFileSystem : public duckdb::FileSystem { duckdb::timestamp_t GetLastModifiedTime(duckdb::FileHandle &handle) override; bool IsManuallySet() override { return true; } + //! Check if a file exists + bool FileExists(const duckdb::string &filename, duckdb::optional_ptr opener = nullptr) override; + + //! Check if a directory exists + bool DirectoryExists(const duckdb::string &directory, duckdb::optional_ptr opener = nullptr) override; + + //! List files in a directory, invoking the callback method for each one with (filename, is_dir) + bool ListFiles(const duckdb::string &directory, + const std::function &callback, + duckdb::FileOpener *opener = nullptr) override; + + duckdb::unique_ptr OpenCompressedFile(duckdb::QueryContext context, + duckdb::unique_ptr handle, + bool write) override; + + private: Cache& cache; }; diff --git a/src/quackstore_filesystem.cpp b/src/quackstore_filesystem.cpp index 5d44832..8ec1c0b 100644 --- a/src/quackstore_filesystem.cpp +++ b/src/quackstore_filesystem.cpp @@ -11,6 +11,25 @@ namespace { duckdb::string StripPrefix(const duckdb::string &text, const duckdb::string &prefix) { return text.rfind(prefix, 0) == 0 ? text.substr(prefix.length()) : text; } + + bool TryGetUnderlyingFileSystem(duckdb::optional_ptr opener, duckdb::FileSystem*& out_fs) { + if (!opener) { + return false; + } + auto optional_cc = opener->TryGetClientContext(); + if (optional_cc) { + out_fs = &duckdb::FileSystem::GetFileSystem(*optional_cc); + return true; + } + + auto optional_db = opener->TryGetDatabase(); + if (optional_db) { + out_fs = &duckdb::FileSystem::GetFileSystem(*optional_db); + return true; + } + + return false; + } } namespace quackstore { @@ -322,26 +341,12 @@ int64_t QuackstoreFileSystem::Read(duckdb::FileHandle &handle, void *buffer, int } duckdb::vector QuackstoreFileSystem::Glob(const duckdb::string &path, duckdb::FileOpener *opener) { - duckdb::string actual_path = StripPrefix(path, SCHEMA_PREFIX); - - duckdb::FileSystem* ufs = nullptr; - auto optional_cc = opener->TryGetClientContext(); - auto optional_db = opener->TryGetDatabase(); - if (optional_cc) - { - ufs = &duckdb::FileSystem::GetFileSystem(*optional_cc); - } - else if (optional_db) - { - ufs = &duckdb::FileSystem::GetFileSystem(*optional_db); - } - else - { - throw duckdb::InvalidInputException("Unable to read QuackStore extension parameters"); + duckdb::FileSystem* underlying_fs_ptr = nullptr; + if (!TryGetUnderlyingFileSystem(opener, underlying_fs_ptr)) { + throw duckdb::InvalidInputException("Unable to get underlying FileSystem for Glob operation"); } - auto& underlying_fs = *ufs; - auto entries = underlying_fs.Glob(actual_path); + auto entries = underlying_fs_ptr->Glob(StripPrefix(path, SCHEMA_PREFIX)); if (path.rfind(SCHEMA_PREFIX, 0) == 0) { for (auto &e : entries) { e.path = SCHEMA_PREFIX + e.path; @@ -371,4 +376,42 @@ duckdb::timestamp_t QuackstoreFileSystem::GetLastModifiedTime(duckdb::FileHandle return caching_file_handle.GetFileLastModified(); } +bool QuackstoreFileSystem::FileExists(const duckdb::string &filename, duckdb::optional_ptr opener) { + duckdb::FileSystem* underlying_fs_ptr = nullptr; + if (!TryGetUnderlyingFileSystem(opener, underlying_fs_ptr)) { + throw duckdb::InvalidInputException("Unable to get underlying FileSystem for FileExists operation"); + } + + auto actual_path = StripPrefix(filename, SCHEMA_PREFIX); + return underlying_fs_ptr->FileExists(actual_path); +} + +bool QuackstoreFileSystem::DirectoryExists(const duckdb::string &directory, duckdb::optional_ptr opener) { + duckdb::FileSystem* underlying_fs_ptr = nullptr; + if (!TryGetUnderlyingFileSystem(opener, underlying_fs_ptr)) { + throw duckdb::InvalidInputException("Unable to get underlying FileSystem for DirectoryExists operation"); + } + + auto actual_path = StripPrefix(directory, SCHEMA_PREFIX); + return underlying_fs_ptr->DirectoryExists(actual_path); +} + +bool QuackstoreFileSystem::ListFiles(const duckdb::string &directory, + const std::function &callback, + duckdb::FileOpener *opener) { + duckdb::FileSystem* underlying_fs_ptr = nullptr; + if (!TryGetUnderlyingFileSystem(opener, underlying_fs_ptr)) { + throw duckdb::InvalidInputException("Unable to get underlying FileSystem for ListFiles operation"); + } + + auto actual_path = StripPrefix(directory, QuackstoreFileSystem::SCHEMA_PREFIX); + return underlying_fs_ptr->ListFiles(actual_path, callback); +} + +duckdb::unique_ptr QuackstoreFileSystem::OpenCompressedFile(duckdb::QueryContext context, + duckdb::unique_ptr handle, + bool write) { + throw duckdb::NotImplementedException("%s: OpenCompressedFile is not implemented!", GetName()); +} + } // namespace quackstore diff --git a/test/unittest/test_cachefs.cpp b/test/unittest/test_cachefs.cpp index d0ccef8..77cdfe1 100644 --- a/test/unittest/test_cachefs.cpp +++ b/test/unittest/test_cachefs.cpp @@ -851,4 +851,134 @@ TEST_CASE_METHOD(WithDuckDB, "CacheFileHandle constructor exception handling pre REQUIRE_NOTHROW(cache->Clear()); } } -} \ No newline at end of file +} + +TEST_CASE_METHOD(WithDuckDB, "Check QuackstoreFileSystem::FileExists", "[quackstore]") { + const auto CACHE_PATH = "/tmp/cache.bin"; + RemoveLocalFile(CACHE_PATH); + auto cache = Cache{16}; + cache.Open(CACHE_PATH); + + const duckdb::string FILE_PATH = QuackstoreFileSystem::SCHEMA_PREFIX + duckdb::string{"test/testdata/read_test.txt"}; + + auto& main_fs_ref = GetDBInstance().GetFileSystem(); + main_fs_ref.UnregisterSubSystem(QuackstoreFileSystem::FILESYSTEM_NAME); + CHECK_FALSE(main_fs_ref.FileExists(FILE_PATH)); + + main_fs_ref.RegisterSubSystem(duckdb::make_uniq(cache)); + CHECK(main_fs_ref.FileExists(FILE_PATH) == true); +} + + +// TEST_CASE_METHOD(WithDuckDB, "Check QuackstoreFileSystem::FileExists (via ATATCH)", "[quackstore]") { +// const auto CACHE_PATH = "/tmp/cache.bin"; +// RemoveLocalFile(CACHE_PATH); +// auto cache = Cache{16}; +// cache.Open(CACHE_PATH); +// +// const duckdb::string FILE_PATH = QuackstoreFileSystem::SCHEMA_PREFIX + duckdb::string{"test/testdata/read_test.txt"}; +// +// auto& main_fs_ref = GetDBInstance().GetFileSystem(); +// main_fs_ref.UnregisterSubSystem(QuackstoreFileSystem::FILESYSTEM_NAME); +// main_fs_ref.RegisterSubSystem(duckdb::make_uniq(cache)); +// +// auto& db = GetDBInstance(); +// auto con = duckdb::Connection{db}; +// auto result = con.Query("SELECT 1;"); +// +// result = con.Query("ATTACH 's3://dkosmakov/mydb3.duckdb' as remote_db_noncached;"); +// CHECK_FALSE(result->HasError()); +// +// result = con.Query("ATTACH 'quackstore://s3://dkosmakov/mydb3.duckdb' as remote_db_readonly (READ_ONLY);"); +// REQUIRE_FALSE(result->HasError()); +// +// result = con.Query("ATTACH 'quackstore://s3://dkosmakov/mydb3.duckdb' as remote_db;"); +// CHECK(result->HasError()); +// +// result = con.Query("SELECT * FROM remote_db_readonly.integers;"); +// REQUIRE_FALSE(result->HasError()); +// CHECK(result->ColumnCount() == 1); +// CHECK(result->RowCount() == 4); +// } + +TEST_CASE_METHOD(WithDuckDB, "Check QuackstoreFileSystem::DirectoryExists", "[quackstore]") { + const auto CACHE_PATH = "/tmp/cache_direxists_test.bin"; + RemoveLocalFile(CACHE_PATH); + auto cache = Cache{16}; + cache.Open(CACHE_PATH); + + const auto DIR_PATH = QuackstoreFileSystem::SCHEMA_PREFIX + duckdb::string{"test/testdata"}; + const auto NON_EXISTENT_DIR_PATH = QuackstoreFileSystem::SCHEMA_PREFIX + duckdb::string{"test/nonexistent_directory"}; + const auto FILE_PATH = QuackstoreFileSystem::SCHEMA_PREFIX + duckdb::string{"test/testdata/read_test.txt"}; + + auto& main_fs_ref = GetDBInstance().GetFileSystem(); + SECTION("Before registering QuackstoreFileSystem, directory should not be found") { + main_fs_ref.UnregisterSubSystem(QuackstoreFileSystem::FILESYSTEM_NAME); + CHECK_FALSE(main_fs_ref.DirectoryExists(DIR_PATH)); + CHECK_FALSE(main_fs_ref.DirectoryExists(NON_EXISTENT_DIR_PATH)); + CHECK_FALSE(main_fs_ref.DirectoryExists(FILE_PATH)); + } + + SECTION("After registering QuackstoreFileSystem") { + main_fs_ref.UnregisterSubSystem(QuackstoreFileSystem::FILESYSTEM_NAME); + main_fs_ref.RegisterSubSystem(duckdb::make_uniq(cache)); + CHECK(main_fs_ref.DirectoryExists(DIR_PATH) == true); + CHECK_FALSE(main_fs_ref.DirectoryExists(NON_EXISTENT_DIR_PATH)); + CHECK_FALSE(main_fs_ref.DirectoryExists(FILE_PATH)); + } +} + +TEST_CASE_METHOD(WithDuckDB, "Check QuackstoreFileSystem::ListFiles", "[quackstore]") { + const auto CACHE_PATH = "/tmp/cache_listfiles_test.bin"; + RemoveLocalFile(CACHE_PATH); + auto cache = Cache{16}; + cache.Open(CACHE_PATH); + + auto& main_fs_ref = GetDBInstance().GetFileSystem(); + main_fs_ref.UnregisterSubSystem(QuackstoreFileSystem::FILESYSTEM_NAME); + main_fs_ref.RegisterSubSystem(duckdb::make_uniq(cache)); + + SECTION("List files in test directory") { + const duckdb::string DIR_PATH = QuackstoreFileSystem::SCHEMA_PREFIX + duckdb::string{"test/testdata"}; + + duckdb::vector found_files; + duckdb::vector is_directory_flags; + + auto callback = [&](const duckdb::string& path, bool is_dir) { + found_files.push_back(path); + is_directory_flags.push_back(is_dir); + }; + + bool result = main_fs_ref.ListFiles(DIR_PATH, callback); + CHECK(result == true); + CHECK(found_files.size() > 0); + + // Check that we found our test file + bool found_test_file = false; + for (size_t i = 0; i < found_files.size(); ++i) { + INFO("Found: " << found_files[i] << " (is_dir: " << is_directory_flags[i] << ")"); + CHECK_FALSE(found_files[i].empty()); + if (found_files[i].find("read_test.txt") != duckdb::string::npos) { + found_test_file = true; + CHECK(is_directory_flags[i] == false); // Should be a file, not directory + } + } + CHECK(found_test_file == true); + } + + SECTION("List files in non-existent directory") { + const duckdb::string DIR_PATH = QuackstoreFileSystem::SCHEMA_PREFIX + duckdb::string{"test/nonexistent_directory"}; + + duckdb::vector found_files; + auto callback = [&](const duckdb::string& path, bool is_dir) { + found_files.push_back(path); + }; + + // ListFiles on non-existent directory should either return false or throw + // depending on underlying filesystem implementation + bool result = main_fs_ref.ListFiles(DIR_PATH, callback); + CHECK_FALSE(result); + // Either way, we shouldn't find any files + CHECK(found_files.size() == 0); + } +}