Refactor Codebase using GLM5.3

.commandcode/settings.json

@@ -0,0 +1,9 @@
+{
+  "permissions": {
+    "allow": [
+      "Shell(python -m ruff check src tests 2 >& 1)"
+    ],
+    "deny": [],
+    "defaultMode": "default"
+  }
+}

.commandcode/taste/coding/taste.md

@@ -0,0 +1,4 @@
+# Coding
+
+- For Python projects lacking lint/type tooling, endorsed adding ruff (lint + import sorting) plus mypy with a lenient starter config as part of the work. Confidence: 0.7
+- Prefers behavior-preserving refactors: deduplicate and split overly complex functions without changing the public API surface, module paths, or documented architecture (keeps docs/roadmaps referencing the layout valid). Confidence: 0.6

.commandcode/taste/taste.md

@@ -0,0 +1,4 @@
+# Taste
+
+## Communication
+- After reviewing a detailed plan, approves with a terse "go ahead" and delegates full autonomous execution; wants key decisions surfaced as a small set of upfront questions before implementation, not repeated check-ins mid-work. Confidence: 0.6

.commandcode/taste/workflow/taste.md

@@ -0,0 +1,8 @@
+# Workflow
+
+- For refactoring tasks, capture a baseline test + coverage run before changing anything, re-run the full test suite after each phase so any breakage is attributable to a single phase, and run the suite twice at the end to catch test-order coupling. Confidence: 0.6
+- Track multi-phase work as an explicit phased todo list (tooling → mechanical cleanup → dedupe → complexity → architecture → verification) rather than one undifferentiated task. Confidence: 0.5
+- Verify the local copy is current (git log/status) before deep codebase analysis or planning; if it's stale, re-run exploration from scratch — explicitly ignoring prior findings — and update the plan, instead of incrementally patching a plan built on outdated code. Confidence: 0.6
+- When a long-running command (e.g., a test suite) appears stuck, don't block on it — explicitly asked to "proceed with next step"; keep making forward progress, run the stuck operation in the background and/or with a per-test timeout so hangs get diagnosed without halting the work. Confidence: 0.7
+- When the local environment can't run the full test suite (e.g., Windows dev box missing bash/WSL or native deps), don't stall on fixing it locally — explicitly directed to "proceed with all the changes for now. I will test the same inside my Github CI instead." Rely on static checks (ruff, mypy) and runnable local subsets meanwhile, and treat GitHub CI as the arbiter for full-suite validation. Confidence: 0.8
+- Local dev machine is Windows with known environmental test failures (tests spawning a POSIX shell via `bash -c` with no WSL installed, native Spark writers needing Hadoop winutils); classify those as pre-existing/environmental rather than regressions, and diff against an expected-failure baseline instead of chasing a fully green local run. Confidence: 0.7

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 name = "testbricks"
 description = "A set of proxy objects to facilitate testing of Databricks notebooks in CI/CD pipelines"
 dynamic = ["version"]
-requires-python = ">=3.8"
+requires-python = ">=3.10"
 authors = [
     {name = "Karan Gupta", email = "gkaran184@gmail.com"},
 ]
@@ -21,16 +21,35 @@ classifiers = [
     "Development Status :: 3 - Alpha",
     "Intended Audience :: Developers",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
     "Programming Language :: Python :: 3.14",
 ]
 
+[project.optional-dependencies]
+dev = ["pytest", "coverage", "ruff", "mypy"]
+
 [tool.setuptools.packages.find]
 where = ["src"]
 
 [tool.setuptools_scm]
+
+[tool.ruff]
+line-length = 100
+target-version = "py310"
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "W", "F", "I", "B"]
+
+[tool.ruff.lint.per-file-ignores]
+# Databricks notebook fixtures: spark/dbutils are injected at runtime.
+"tests/e2e_workflow/notebooks/*.py" = ["F821"]
+
+[tool.mypy]
+# numpy/pandas stubs use Python 3.12 syntax, so the checker must run as 3.12
+# even though the package supports 3.10+.
+python_version = "3.12"
+ignore_missing_imports = true

requirements.txt

@@ -5,3 +5,6 @@ pyarrow
 numpy
 py4j
 coverage
+ruff
+mypy
+pytest-timeout

src/testbricks/catalog/__init__.py

@@ -1,16 +1,28 @@
+from .csv_options import (
+    CSV_OPTION_KEYS,
+    DEFAULT_CSV_READ_OPTIONS,
+    normalize_csv_options,
+    option_flag,
+    option_lookup,
+)
 from .errors import InvalidTableNameError, SchemaMismatchError, SparkProxyError
 from .identifier import TableIdentifier
 from .spark_catalog import CatalogFacade
 from .sql_rewrite import is_maintenance_noop, rewrite_from_join_identifiers
 from .table_catalog import TableCatalog
 
 __all__ = [
+    "CSV_OPTION_KEYS",
     "CatalogFacade",
+    "DEFAULT_CSV_READ_OPTIONS",
     "InvalidTableNameError",
     "SchemaMismatchError",
     "SparkProxyError",
     "TableCatalog",
     "TableIdentifier",
     "is_maintenance_noop",
+    "normalize_csv_options",
+    "option_flag",
+    "option_lookup",
     "rewrite_from_join_identifiers",
 ]

src/testbricks/catalog/csv_options.py

@@ -0,0 +1,69 @@
+"""Canonical CSV option keys, aliases, and case-insensitive lookups.
+
+This module is the single source of truth for which ``option(...)`` keys the
+CSV catalog honors, how Spark-style aliases map to canonical names, and how
+options are matched case-insensitively.
+"""
+
+from __future__ import annotations
+
+from typing import Mapping, Optional
+
+DEFAULT_CSV_READ_OPTIONS: dict[str, str] = {"header": "true", "inferSchema": "true"}
+
+CSV_OPTION_KEYS = frozenset(
+    {
+        "delimiter",
+        "sep",
+        "quote",
+        "escape",
+        "nullvalue",
+        "dateformat",
+        "timestampformat",
+        "header",
+    }
+)
+
+_OPTION_ALIASES = {
+    "sep": "delimiter",
+    "delimiter": "delimiter",
+    "quote": "quote",
+    "escape": "escape",
+    "nullvalue": "nullValue",
+    "dateformat": "dateFormat",
+    "timestampformat": "timestampFormat",
+    "header": "header",
+}
+
+
+def normalize_csv_options(options: Optional[Mapping[str, str]]) -> dict[str, str]:
+    """Keep only known CSV options, mapping aliases to canonical names."""
+    if not options:
+        return {}
+    normalized: dict[str, str] = {}
+    for key, value in options.items():
+        canonical = _OPTION_ALIASES.get(key.lower())
+        if canonical is None:
+            continue
+        normalized[canonical] = str(value)
+    return normalized
+
+
+def option_lookup(options: Optional[Mapping[str, str]], *names: str) -> Optional[str]:
+    """Case-insensitive lookup returning the first match as ``str``."""
+    if not options:
+        return None
+    lowered = {key.lower(): value for key, value in options.items()}
+    for name in names:
+        if name.lower() in lowered:
+            return str(lowered[name.lower()])
+    return None
+
+
+def option_flag(options: Mapping[str, str], *names: str) -> bool:
+    """Case-insensitive truthy check for Spark-style boolean option values."""
+    wanted = {name.lower() for name in names}
+    for key, value in options.items():
+        if key.lower() in wanted:
+            return str(value).lower() in {"true", "1", "yes"}
+    return False

src/testbricks/catalog/errors.py

@@ -1,7 +1,15 @@
-"""Errors raised by the SparkProxy CSV catalog."""
+"""Errors raised by the CSV catalog layer.
 
+All catalog errors derive from both ``TestbricksError`` and the hierarchy
+below. ``InvalidTableNameError`` and ``SchemaMismatchError`` additionally
+inherit ``ValueError`` because callers legitimately treat invalid table
+names and schema mismatches as value-level input errors.
+"""
 
-class SparkProxyError(Exception):
+from testbricks.errors import TestbricksError
+
+
+class SparkProxyError(TestbricksError):
     """Base error for SparkProxy catalog and table operations."""
 
 

src/testbricks/catalog/identifier.py

@@ -7,11 +7,16 @@
 from .errors import InvalidTableNameError
 
 
+def strip_wrappers(text: str, wrapper_chars: str = "`") -> str:
+    """Strip one layer of symmetric wrapper characters (e.g. backticks, quotes)."""
+    text = text.strip()
+    if len(text) >= 2 and text[0] == text[-1] and text[0] in wrapper_chars:
+        return text[1:-1]
+    return text
+
+
 def _strip_identifier_part(part: str) -> str:
-    part = part.strip()
-    if len(part) >= 2 and part[0] == "`" and part[-1] == "`":
-        return part[1:-1]
-    return part
+    return strip_wrappers(part)
 
 
 def _split_qualified_name(table_name: str) -> list[str]:

src/testbricks/catalog/spark_catalog.py

@@ -22,6 +22,9 @@ class CatalogFacade:
     def __init__(self, tables: TableCatalog):
         self._tables = tables
 
+    def exists(self, ident: TableIdentifier) -> bool:
+        return self._tables.exists(ident)
+
     def tableExists(self, tableName: str, dbName: Optional[str] = None) -> bool:
         try:
             ident = (
@@ -67,6 +70,3 @@ def listDatabases(self, pattern: Optional[str] = None) -> List[Database]:
             for name in self._tables.iter_schema_names()
             if _matches(name, pattern)
         ]
-
-    def __getattr__(self, name):
-        return getattr(self._tables, name)