Fix Issue 6: ATTACH to remote duckdb database -- QuackstoreFileSystem: FileExists is not implemented

src/block_manager.cpp

@@ -300,13 +300,29 @@ void BlockManager::LoadFreeList() {
 
 void BlockManager::ValidateBlockId(block_id_t block_id) const {
     if (block_id == INVALID_BLOCK_ID) {
-        throw duckdb::InvalidInputException("Block ID cannot be INVALID_BLOCK_ID");
+        throw duckdb::InvalidInputException(
+            "Block ID cannot be INVALID_BLOCK_ID",
+            duckdb::unordered_map<duckdb::string, duckdb::string>{
+                {"block_id", std::to_string(block_id)}
+            }
+        );
     }
     if (block_id < 0) {
-        throw duckdb::InvalidInputException("Block ID cannot be negative", {{"block_id", std::to_string(block_id)}});
+        throw duckdb::InvalidInputException(
+            "Block ID cannot be negative",
+            duckdb::unordered_map<duckdb::string, duckdb::string>{
+                {"block_id", std::to_string(block_id)}
+            }
+        );
     }
     if (block_id >= max_block) {
-        throw duckdb::InvalidInputException("Block ID cannot exceed max_block", {{"block_id", std::to_string(block_id)}, {"max_block", std::to_string(max_block)}});
+        throw duckdb::InvalidInputException(
+            "Block ID cannot exceed max_block",
+            duckdb::unordered_map<duckdb::string, duckdb::string>{
+                {"block_id", std::to_string(block_id)},
+                {"max_block", std::to_string(max_block)}
+            }
+        );
     }
 }
 

src/include/quackstore_filesystem.hpp

@@ -29,6 +29,22 @@ class QuackstoreFileSystem : public duckdb::FileSystem {
     duckdb::timestamp_t GetLastModifiedTime(duckdb::FileHandle &handle) override;
     bool IsManuallySet() override { return true; }
 
+	//! Check if a file exists
+	bool FileExists(const duckdb::string &filename, duckdb::optional_ptr<duckdb::FileOpener> opener = nullptr) override;
+
+	//! Check if a directory exists
+	bool DirectoryExists(const duckdb::string &directory, duckdb::optional_ptr<duckdb::FileOpener> opener = nullptr) override;
+
+	//! List files in a directory, invoking the callback method for each one with (filename, is_dir)
+	bool ListFiles(const duckdb::string &directory,
+                           const std::function<void(const duckdb::string &, bool)> &callback,
+                           duckdb::FileOpener *opener = nullptr) override;
+
+	duckdb::unique_ptr<duckdb::FileHandle> OpenCompressedFile(duckdb::QueryContext context, 
+                                                                      duckdb::unique_ptr<duckdb::FileHandle> handle,
+	                                                                  bool write) override;
+
+
 private:
     Cache& cache;
 };

src/quackstore_filesystem.cpp

@@ -11,6 +11,25 @@ namespace {
     duckdb::string StripPrefix(const duckdb::string &text, const duckdb::string &prefix) {
         return text.rfind(prefix, 0) == 0 ? text.substr(prefix.length()) : text;
     }
+
+    bool TryGetUnderlyingFileSystem(duckdb::optional_ptr<duckdb::FileOpener> opener, duckdb::FileSystem*& out_fs) {
+        if (!opener) {
+            return false;
+        }
+        auto optional_cc = opener->TryGetClientContext();
+        if (optional_cc) {
+            out_fs = &duckdb::FileSystem::GetFileSystem(*optional_cc);
+            return true;
+        } 
+
+        auto optional_db = opener->TryGetDatabase();
+        if (optional_db) {
+            out_fs = &duckdb::FileSystem::GetFileSystem(*optional_db);
+            return true;
+        } 
+
+        return false;
+    }
 }
 
 namespace quackstore {
@@ -322,26 +341,12 @@ int64_t QuackstoreFileSystem::Read(duckdb::FileHandle &handle, void *buffer, int
 }
 
 duckdb::vector<duckdb::OpenFileInfo> QuackstoreFileSystem::Glob(const duckdb::string &path, duckdb::FileOpener *opener) {
-    duckdb::string actual_path = StripPrefix(path, SCHEMA_PREFIX);
-
-    duckdb::FileSystem* ufs = nullptr;
-    auto optional_cc = opener->TryGetClientContext();
-    auto optional_db = opener->TryGetDatabase();
-    if (optional_cc)
-    {
-        ufs = &duckdb::FileSystem::GetFileSystem(*optional_cc);
-    }
-    else if (optional_db)
-    {
-        ufs = &duckdb::FileSystem::GetFileSystem(*optional_db);
-    }
-    else
-    {
-        throw duckdb::InvalidInputException("Unable to read QuackStore extension parameters");
+    duckdb::FileSystem* underlying_fs_ptr = nullptr;
+    if (!TryGetUnderlyingFileSystem(opener, underlying_fs_ptr)) {
+        throw duckdb::InvalidInputException("Unable to get underlying FileSystem for Glob operation");
     }
-    auto& underlying_fs = *ufs;
 
-    auto entries = underlying_fs.Glob(actual_path);
+    auto entries = underlying_fs_ptr->Glob(StripPrefix(path, SCHEMA_PREFIX));
     if (path.rfind(SCHEMA_PREFIX, 0) == 0) {
         for (auto &e : entries) {
             e.path = SCHEMA_PREFIX + e.path;
@@ -371,4 +376,42 @@ duckdb::timestamp_t QuackstoreFileSystem::GetLastModifiedTime(duckdb::FileHandle
     return caching_file_handle.GetFileLastModified();
 }
 
+bool QuackstoreFileSystem::FileExists(const duckdb::string &filename, duckdb::optional_ptr<duckdb::FileOpener> opener) {
+    duckdb::FileSystem* underlying_fs_ptr = nullptr;
+    if (!TryGetUnderlyingFileSystem(opener, underlying_fs_ptr)) {
+        throw duckdb::InvalidInputException("Unable to get underlying FileSystem for FileExists operation");
+    }
+
+    auto actual_path = StripPrefix(filename, SCHEMA_PREFIX);
+    return underlying_fs_ptr->FileExists(actual_path);
+}
+
+bool QuackstoreFileSystem::DirectoryExists(const duckdb::string &directory, duckdb::optional_ptr<duckdb::FileOpener> opener) {
+    duckdb::FileSystem* underlying_fs_ptr = nullptr;
+    if (!TryGetUnderlyingFileSystem(opener, underlying_fs_ptr)) {
+        throw duckdb::InvalidInputException("Unable to get underlying FileSystem for DirectoryExists operation");
+    }
+
+    auto actual_path = StripPrefix(directory, SCHEMA_PREFIX);
+    return underlying_fs_ptr->DirectoryExists(actual_path);
+}
+
+bool QuackstoreFileSystem::ListFiles(const duckdb::string &directory, 
+                                     const std::function<void(const duckdb::string &, bool)> &callback, 
+                                     duckdb::FileOpener *opener) {
+    duckdb::FileSystem* underlying_fs_ptr = nullptr;
+    if (!TryGetUnderlyingFileSystem(opener, underlying_fs_ptr)) {
+        throw duckdb::InvalidInputException("Unable to get underlying FileSystem for ListFiles operation");
+    }
+
+    auto actual_path = StripPrefix(directory, QuackstoreFileSystem::SCHEMA_PREFIX);
+    return underlying_fs_ptr->ListFiles(actual_path, callback);
+}
+
+duckdb::unique_ptr<duckdb::FileHandle> QuackstoreFileSystem::OpenCompressedFile(duckdb::QueryContext context,
+                                                                    duckdb::unique_ptr<duckdb::FileHandle> handle,
+                                                                    bool write) {
+    throw duckdb::NotImplementedException("%s: OpenCompressedFile is not implemented!", GetName());
+}
+
 }  // namespace quackstore

test/unittest/test_cachefs.cpp

@@ -851,4 +851,134 @@ TEST_CASE_METHOD(WithDuckDB, "CacheFileHandle constructor exception handling pre
             REQUIRE_NOTHROW(cache->Clear());
         }
     }
-}
+}
+
+TEST_CASE_METHOD(WithDuckDB, "Check QuackstoreFileSystem::FileExists", "[quackstore]") {
+    const auto CACHE_PATH = "/tmp/cache.bin";
+    RemoveLocalFile(CACHE_PATH);
+    auto cache = Cache{16};
+    cache.Open(CACHE_PATH);
+
+    const duckdb::string FILE_PATH = QuackstoreFileSystem::SCHEMA_PREFIX + duckdb::string{"test/testdata/read_test.txt"};
+
+    auto& main_fs_ref = GetDBInstance().GetFileSystem();
+    main_fs_ref.UnregisterSubSystem(QuackstoreFileSystem::FILESYSTEM_NAME);
+    CHECK_FALSE(main_fs_ref.FileExists(FILE_PATH));
+
+    main_fs_ref.RegisterSubSystem(duckdb::make_uniq<QuackstoreFileSystem>(cache));
+    CHECK(main_fs_ref.FileExists(FILE_PATH) == true);
+}
+
+
+// TEST_CASE_METHOD(WithDuckDB, "Check QuackstoreFileSystem::FileExists (via ATATCH)", "[quackstore]") {
+//     const auto CACHE_PATH = "/tmp/cache.bin";
+//     RemoveLocalFile(CACHE_PATH);
+//     auto cache = Cache{16};
+//     cache.Open(CACHE_PATH);
+// 
+//     const duckdb::string FILE_PATH = QuackstoreFileSystem::SCHEMA_PREFIX + duckdb::string{"test/testdata/read_test.txt"};
+// 
+//     auto& main_fs_ref = GetDBInstance().GetFileSystem();
+//     main_fs_ref.UnregisterSubSystem(QuackstoreFileSystem::FILESYSTEM_NAME);
+//     main_fs_ref.RegisterSubSystem(duckdb::make_uniq<QuackstoreFileSystem>(cache));
+// 
+//     auto& db = GetDBInstance();
+//     auto con = duckdb::Connection{db};
+//     auto result = con.Query("SELECT 1;");
+// 
+//     result = con.Query("ATTACH 's3://dkosmakov/mydb3.duckdb' as remote_db_noncached;");
+//     CHECK_FALSE(result->HasError());
+// 
+//     result = con.Query("ATTACH 'quackstore://s3://dkosmakov/mydb3.duckdb' as remote_db_readonly (READ_ONLY);");
+//     REQUIRE_FALSE(result->HasError());
+// 
+//     result = con.Query("ATTACH 'quackstore://s3://dkosmakov/mydb3.duckdb' as remote_db;");
+//     CHECK(result->HasError());
+// 
+//     result = con.Query("SELECT * FROM remote_db_readonly.integers;");
+//     REQUIRE_FALSE(result->HasError());
+//     CHECK(result->ColumnCount() == 1);
+//     CHECK(result->RowCount() == 4);
+// }
+
+TEST_CASE_METHOD(WithDuckDB, "Check QuackstoreFileSystem::DirectoryExists", "[quackstore]") {
+    const auto CACHE_PATH = "/tmp/cache_direxists_test.bin";
+    RemoveLocalFile(CACHE_PATH);
+    auto cache = Cache{16};
+    cache.Open(CACHE_PATH);
+
+    const auto DIR_PATH = QuackstoreFileSystem::SCHEMA_PREFIX + duckdb::string{"test/testdata"};
+    const auto NON_EXISTENT_DIR_PATH = QuackstoreFileSystem::SCHEMA_PREFIX + duckdb::string{"test/nonexistent_directory"};
+    const auto FILE_PATH = QuackstoreFileSystem::SCHEMA_PREFIX + duckdb::string{"test/testdata/read_test.txt"};
+
+    auto& main_fs_ref = GetDBInstance().GetFileSystem();
+    SECTION("Before registering QuackstoreFileSystem, directory should not be found") {
+        main_fs_ref.UnregisterSubSystem(QuackstoreFileSystem::FILESYSTEM_NAME);
+        CHECK_FALSE(main_fs_ref.DirectoryExists(DIR_PATH));
+        CHECK_FALSE(main_fs_ref.DirectoryExists(NON_EXISTENT_DIR_PATH));
+        CHECK_FALSE(main_fs_ref.DirectoryExists(FILE_PATH));
+    }
+
+    SECTION("After registering QuackstoreFileSystem") {
+        main_fs_ref.UnregisterSubSystem(QuackstoreFileSystem::FILESYSTEM_NAME);
+        main_fs_ref.RegisterSubSystem(duckdb::make_uniq<QuackstoreFileSystem>(cache));
+        CHECK(main_fs_ref.DirectoryExists(DIR_PATH) == true);
+        CHECK_FALSE(main_fs_ref.DirectoryExists(NON_EXISTENT_DIR_PATH));
+        CHECK_FALSE(main_fs_ref.DirectoryExists(FILE_PATH));
+    }
+}
+
+TEST_CASE_METHOD(WithDuckDB, "Check QuackstoreFileSystem::ListFiles", "[quackstore]") {
+    const auto CACHE_PATH = "/tmp/cache_listfiles_test.bin";
+    RemoveLocalFile(CACHE_PATH);
+    auto cache = Cache{16};
+    cache.Open(CACHE_PATH);
+
+    auto& main_fs_ref = GetDBInstance().GetFileSystem();
+    main_fs_ref.UnregisterSubSystem(QuackstoreFileSystem::FILESYSTEM_NAME);
+    main_fs_ref.RegisterSubSystem(duckdb::make_uniq<QuackstoreFileSystem>(cache));
+
+    SECTION("List files in test directory") {
+        const duckdb::string DIR_PATH = QuackstoreFileSystem::SCHEMA_PREFIX + duckdb::string{"test/testdata"};
+
+        duckdb::vector<duckdb::string> found_files;
+        duckdb::vector<bool> is_directory_flags;
+
+        auto callback = [&](const duckdb::string& path, bool is_dir) {
+            found_files.push_back(path);
+            is_directory_flags.push_back(is_dir);
+        };
+
+        bool result = main_fs_ref.ListFiles(DIR_PATH, callback);
+        CHECK(result == true);
+        CHECK(found_files.size() > 0);
+
+        // Check that we found our test file
+        bool found_test_file = false;
+        for (size_t i = 0; i < found_files.size(); ++i) {
+            INFO("Found: " << found_files[i] << " (is_dir: " << is_directory_flags[i] << ")");
+            CHECK_FALSE(found_files[i].empty());
+            if (found_files[i].find("read_test.txt") != duckdb::string::npos) {
+                found_test_file = true;
+                CHECK(is_directory_flags[i] == false); // Should be a file, not directory
+            }
+        }
+        CHECK(found_test_file == true);
+    }
+
+    SECTION("List files in non-existent directory") {
+        const duckdb::string DIR_PATH = QuackstoreFileSystem::SCHEMA_PREFIX + duckdb::string{"test/nonexistent_directory"};
+
+        duckdb::vector<duckdb::string> found_files;
+        auto callback = [&](const duckdb::string& path, bool is_dir) {
+            found_files.push_back(path);
+        };
+
+        // ListFiles on non-existent directory should either return false or throw
+        // depending on underlying filesystem implementation
+        bool result = main_fs_ref.ListFiles(DIR_PATH, callback);
+        CHECK_FALSE(result);
+        // Either way, we shouldn't find any files
+        CHECK(found_files.size() == 0);
+    }
+}