fix: harden indexing and prepare Steadlith 1.0.1

.github/workflows/ci.yml

@@ -121,6 +121,14 @@ jobs:
       - name: Validate distribution metadata
         run: python -m twine check dist/*
 
+      - name: Exercise the installed wheel
+        run: |
+          wheel_venv="$(mktemp -d)"
+          python -m venv "$wheel_venv"
+          "$wheel_venv/bin/python" -m pip install --no-cache-dir dist/*.whl
+          "$wheel_venv/bin/python" -m pip check
+          "$wheel_venv/bin/python" tests/smoke_installed.py
+
       - name: Upload distributions
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:

.github/workflows/release.yml

@@ -85,21 +85,9 @@ jobs:
           wheel_venv="$(mktemp -d)"
           python -m venv "$wheel_venv"
           wheel_python="$wheel_venv/bin/python"
-          wheel_cli="$wheel_venv/bin/steadlith"
           "$wheel_python" -m pip install --no-cache-dir dist/*.whl
           "$wheel_python" -m pip check
-          "$wheel_python" -c "import steadlith"
-          "$wheel_python" -m steadlith --version
-          "$wheel_cli" --version
-          smoke_dir="$(mktemp -d)"
-          cd "$smoke_dir"
-          "$wheel_cli" init --json > init.json
-          mkdir docs
-          printf '%s\n' 'alpha beta gamma delta epsilon' > docs/smoke.md
-          "$wheel_cli" plan --json > plan.json
-          "$wheel_cli" index --json > index.json
-          "$wheel_cli" query 'alpha beta' --json > query.json
-          "$wheel_cli" verify --json > verify.json
+          "$wheel_python" tests/smoke_installed.py
 
       - name: Attest distributions
         uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2

CHANGELOG.md

@@ -6,6 +6,26 @@ All notable changes to Steadlith are documented here. The format follows
 
 ## [Unreleased]
 
+## [1.0.1] - 2026-09-05
+
+### Fixed
+
+- Reject overlapping configuration, cache, index, SQLite sidecar, manifest, and migration paths before writes can corrupt project state.
+- Report invalid UTF-8 configuration files as actionable configuration errors, including JSON CLI errors.
+- Reject malformed cache import fields without coercing identities, vectors, or token counts, and bound import line reads.
+- Prevent forced cache exports from replacing configuration, databases and sidecars, manifest mirrors, migration journals, or receipts.
+- Roll back interrupted cache and index transactions so the same connection can be retried safely.
+- Keep related SQLite reads on one committed generation and report malformed stored metadata as verification failures.
+- Publish manifest mirrors from the latest committed SQLite state under a writer reservation, preventing delayed operations from restoring an older mirror.
+- Account for reusable active vectors when estimating embeddings for copied or renamed chunks after cache pruning.
+- Pin OpenAI clients to the official API endpoint so inherited `OPENAI_BASE_URL` values cannot redirect requests or disagree with cache identity.
+- Validate migration target size and the exact configuration change before publishing state; accept quoted and padded TOML table and key names.
+
+### Changed
+
+- Exercise installed-wheel indexing, querying, migration, cache, deletion, compaction, and fixture retrieval workflows in pull-request and release CI.
+- Activate the contributor virtual environment before installing development dependencies.
+
 ## [1.0.0] - 2026-08-22
 
 ### Added
@@ -92,7 +112,8 @@ All notable changes to Steadlith are documented here. The format follows
 - The bundled hash embedder is deterministic test infrastructure, not a production retrieval model.
 - Post-anchor structural snapping is experimental and requires project-specific legal review before use.
 
-[Unreleased]: https://github.com/satwiksps/steadlith/compare/v1.0.0...HEAD
+[Unreleased]: https://github.com/satwiksps/steadlith/compare/v1.0.1...HEAD
+[1.0.1]: https://github.com/satwiksps/steadlith/compare/v1.0.0...v1.0.1
 [1.0.0]: https://github.com/satwiksps/steadlith/compare/v0.3.0...v1.0.0
 [0.3.0]: https://github.com/satwiksps/steadlith/compare/v0.2.0...v0.3.0
 [0.2.0]: https://github.com/satwiksps/steadlith/compare/v0.1.0...v0.2.0

CITATION.cff

@@ -7,8 +7,8 @@ authors:
     given-names: "Satwik Sai Prakash"
     email: "sahoospsatwik@gmail.com"
 repository-code: "https://github.com/satwiksps/steadlith"
-version: 1.0.0
-date-released: 2026-08-22
+version: 1.0.1
+date-released: 2026-09-05
 license: Apache-2.0
 keywords:
   - retrieval-augmented generation

CONTRIBUTING.md

@@ -17,11 +17,17 @@ Steadlith supports Python 3.10 and newer.
 git clone https://github.com/satwiksps/steadlith.git
 cd steadlith
 python -m venv .venv
+```
+
+Activate the environment with `source .venv/bin/activate` on Linux or macOS, or
+`.venv\Scripts\Activate.ps1` in Windows PowerShell. Then install:
+
+```bash
 python -m pip install --upgrade pip
 python -m pip install -e ".[dev]"
 ```
 
-Activate the virtual environment using the command appropriate for your shell. Provider-specific work may also need the `openai` or `sentence-transformers` extra.
+Provider-specific work may also need the `openai` or `sentence-transformers` extra.
 
 The landing site is a separate Next.js application:
 

docs/cli.md

@@ -172,7 +172,7 @@ At least one limit is required and both must be non-negative. Age pruning remove
 steadlith cache export [-c PATH] [--json] [--force] DESTINATION
 ```
 
-Writes all cache entries as deterministic JSON Lines through an atomic temporary file. It refuses to overwrite an existing destination unless `--force` is supplied, and it never permits the live cache, its SQLite sidecars, or the configured index database as the destination. A missing cache produces an empty file.
+Writes all cache entries as deterministic JSON Lines through an atomic temporary file. It refuses to overwrite an existing destination unless `--force` is supplied. Managed project state is always protected: the configuration, cache and index databases, their SQLite sidecars, the manifest mirror, migration journal, and files in the migration-receipt directory cannot be export destinations, even with `--force`. A missing cache produces an empty file.
 
 ### `cache import`
 

docs/conf.py

@@ -12,7 +12,7 @@
 try:
     release = version("steadlith")
 except PackageNotFoundError:
-    release = "1.0.0"
+    release = "1.0.1"
 version = ".".join(release.split(".")[:2])
 
 extensions = [

docs/development/testing.md

@@ -12,6 +12,17 @@ python -m pytest --cov=steadlith --cov-branch --cov-report=term-missing
 python -m build
 ```
 
+Test the distribution in a separate environment with only runtime dependencies:
+
+```bash
+python -m venv tmp/wheel-check
+tmp/wheel-check/bin/python -m pip install dist/steadlith-1.0.1-py3-none-any.whl
+tmp/wheel-check/bin/python -m pip check
+tmp/wheel-check/bin/python tests/smoke_installed.py
+```
+
+Use `tmp/wheel-check/Scripts/python.exe` in PowerShell, and substitute the wheel filename for the version being tested. The smoke script runs outside the checkout and checks both entry points, exact quick-start results, repeat indexing, deletion guards, migration and rollback, cache transfer, and compaction. Pull-request and release CI run the same script against the built wheel.
+
 Build documentation with warnings as errors:
 
 ```bash

docs/getting-started/configuration.md

@@ -87,7 +87,7 @@ Provider and model settings participate in embedding identity. Dimensions are va
 | --- | --- | --- |
 | `cache` | `.steadlith/cache.sqlite3` | SQLite content-addressed embedding cache. |
 
-The cache path must stay below the configuration directory and must differ from the index database path.
+The cache path must stay below the configuration directory and must differ from the index database path. State paths cannot overlap the configuration, SQLite sidecars (`-wal`, `-shm`, `-journal`), index manifest (`.manifest.json`), migration journal, or migration receipt directory (`.migrations`). Validation rejects these collisions before writing state.
 
 ## Index fields
 

docs/guides/cache-management.md

@@ -53,7 +53,7 @@ The command writes deterministic JSON Lines through a temporary file and atomic
 steadlith cache export --force cache-backup.jsonl
 ```
 
-It also refuses destinations that resolve to the live cache, its SQLite sidecars, or the configured index database.
+Even with `--force`, exports cannot replace managed project state: the configuration, cache and index databases, their SQLite sidecars, the manifest mirror, migration journal, or anything in the migration-receipt directory. Choose a separate backup file.
 
 ## Import
 
@@ -71,6 +71,11 @@ Import validates:
 - duplicate-key consistency;
 - conflicts with existing entries.
 
+Identity fields must be non-empty strings, vectors must be arrays of finite numbers,
+and token counts must be non-negative integers. Malformed fields are rejected instead
+of being converted. Validation failure leaves the cache unchanged and identifies
+the invalid line.
+
 An identical existing entry is retained and touched. A different vector under an existing key fails instead of overwriting trusted state.
 
 ## Backup strategy

docs/guides/embedding-providers.md

@@ -80,7 +80,7 @@ Constraints:
 - text in missing chunks is sent to the provider;
 - current price, quota, rate limits, retention, and regional availability remain operator responsibilities.
 
-Custom base URLs are rejected. This prevents an untrusted repository configuration from forwarding documents and a chosen environment secret to another endpoint.
+Custom base URLs are rejected in project configuration, and `OPENAI_BASE_URL` cannot override the official endpoint. This prevents repository configuration or an inherited endpoint override from forwarding documents and the API key to another endpoint.
 
 ## Sentence Transformers provider
 

docs/guides/indexing.md

@@ -33,7 +33,7 @@ Each occurrence receives one operation:
 
 | Operation | Meaning | New embedding when identity is unchanged? |
 | --- | --- | --- |
-| `add` | New chunk occurrence. | Only on a cache miss for its chunk hash. |
+| `add` | New chunk occurrence. | Only when neither the active index nor cache holds its chunk hash. |
 | `keep` | Same hash and position. | No. |
 | `move` | Same chunk content at a different position or with changed source metadata. | No. |
 | `delete` | Previously active occurrence absent from the target. | No; the old record is tombstoned. |
@@ -42,6 +42,9 @@ A model or provider-parameter migration can re-embed kept and moved content beca
 
 Unknown provider prices remain unknown. Steadlith never fetches pricing or assumes that its word-based token count matches a provider billing tokenizer.
 
+Copied, repeated, and renamed chunks reuse the active vector even after its cache
+entry is pruned. This reuse does not count as a cache hit in the plan or apply result.
+
 ## Apply approval gates
 
 Steadlith requires explicit approval for three classes of effect.
@@ -78,10 +81,19 @@ An apply performs the following work:
 6. Write successful batches to the cache.
 7. Build all target index records in memory.
 8. Publish records, document state, manifest, root, embedding identity, and the next generation in one SQLite transaction.
-9. Write the diffable manifest mirror after the database commit.
+9. Mirror the latest committed manifest under a short SQLite writer reservation,
+   preventing an older apply from overwriting a newer generation's mirror.
 
 Queries never observe a partially published generation. If another writer committed after preparation, the apply fails instead of overwriting the newer state.
 
+Plan preparation, status, and database verification each read one consistent SQLite
+snapshot. An interrupted apply rolls back its uncommitted index changes; the same
+connection can be reused after the interruption.
+
+Readers remain available during mirror publication. A mirror write failure leaves
+the committed index usable and reports a repair instruction; rerunning `index`
+with the same configuration and source scope repairs the mirror.
+
 Provider-side charging cannot be strictly transactional with a local SQLite commit. A process failure after a remote provider accepts a request but before the cache records its response can lead to a repeated charge on retry.
 
 ## Idempotent routine

docs/guides/migrations.md

@@ -23,6 +23,8 @@ Preview is the default. `--dry-run` is accepted when an explicit marker is usefu
 
 Positional source paths are permitted only for exploratory previews. An apply always uses the persisted `[sources]` configuration so the published config immediately reproduces the committed corpus.
 
+Migration editing supports named `[chunker]` and `[embedding]` tables with single-line settings, including quoted names and whitespace inside table headers. It preserves unrelated comments and settings. If a configuration uses inline or dotted tables, or multiline values for settings being changed, rewrite those sections as named tables before migrating. The parsed target must match exactly the requested changes.
+
 ## Apply a reviewed migration
 
 Repeat the target arguments with `--apply` and required approvals:
@@ -49,6 +51,8 @@ Apply sequence:
 
 The receipt is checksummed for corruption detection. It is not an authenticated audit record.
 
+Both the current and target TOML must fit the 1 MiB migration limit. Oversized targets are rejected during preparation, before any provider work or index publication, so recovery can always read the staged configuration.
+
 Journal and receipt publication requires a filesystem that supports hard-link creation within a directory. Configuration publication requires atomic replacement within its directory. Steadlith creates each temporary file beside its destination; filesystems that do not provide these operations are unsupported and cause migration apply or recovery to fail with a storage error. Parent directories are not explicitly fsynced, so sudden-power-loss durability of directory entries depends on the operating system and filesystem.
 
 ## Recover an interrupted migration

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "steadlith"
-version = "1.0.0"
+version = "1.0.1"
 description = "Steadlith reuses unchanged RAG chunks with content-defined identities, cache-aware planning, and transactional indexing."
 readme = "README.md"
 requires-python = ">=3.10"

src/steadlith/__init__.py

@@ -12,6 +12,6 @@
 try:
     __version__ = version("steadlith")
 except PackageNotFoundError:  # source checkout without an installed distribution
-    __version__ = "1.0.0"
+    __version__ = "1.0.1"
 
 __all__ = ["CDCChunker", "CDCParams", "Cache", "Chunk", "__version__"]

src/steadlith/cli.py

@@ -22,9 +22,10 @@
     load_config,
     write_default_config,
 )
-from steadlith.errors import ConfigError, ExitCode, SteadlithError, VerificationError
+from steadlith.errors import BackendError, ConfigError, ExitCode, SteadlithError, VerificationError
 from steadlith.index.plan import OperationKind
 from steadlith.index.service import (
+    _protected_state,
     apply_prepared,
     compact_index,
     index_status,
@@ -386,6 +387,18 @@ def _cache_export(args: argparse.Namespace, console: Console) -> int:
     destination = args.destination.expanduser().resolve()
     if destination == config.resolve(config.index.database):
         raise ConfigError("Cache export destination cannot overwrite the configured index")
+    cache_path = config.resolve(config.store.cache)
+    if destination in {Path(f"{cache_path}{suffix}").resolve() for suffix in ("", "-wal", "-shm")}:
+        raise BackendError("Cache export destination cannot be the live cache or its sidecars")
+    protected_files, protected_directories = _protected_state(config)
+    if destination in {path.resolve() for path in protected_files} or any(
+        destination == directory.resolve() or directory.resolve() in destination.parents
+        for directory in protected_directories
+    ):
+        raise ConfigError(
+            f"Cache export destination cannot overwrite managed project state: {destination}. "
+            "Choose a separate export file, even when using --force."
+        )
     with Cache(config.resolve(config.store.cache), readonly=True) as cache:
         count = cache.export_jsonl(destination, force=args.force)
     if args.json:

src/steadlith/config.py

@@ -258,6 +258,26 @@
                 ) from exc
         if state_paths["store.cache"] == state_paths["index.database"]:
             raise ConfigError("store.cache and index.database must use different files")
+        # SQLite sidecars and derived index files share the state namespace.
+        # A collision can replace a live database when the manifest is published.
+        reserved = dict(state_paths)
+        for label, path in state_paths.items():
+            for suffix in ("-wal", "-shm", "-journal"):
+                reserved[f"{label} {suffix} sidecar"] = Path(f"{path}{suffix}").resolve()
+        database = state_paths["index.database"]
+        reserved["index manifest"] = Path(f"{database}.manifest.json").resolve()
+        reserved["migration receipts"] = Path(f"{database}.migrations").resolve()
+        if self.config_path is not None:
+            reserved["configuration"] = self.config_path.expanduser().resolve()
+            reserved["migration journal"] = pending_migration_path(self.config_path)
+        paths = list(reserved.items())
+        for index, (label, path) in enumerate(paths):
+            for other_label, other in paths[index + 1 :]:
+                if path == other or path in other.parents or other in path.parents:
+                    raise ConfigError(
+                        f"State paths overlap: {label} ({path}) and {other_label} ({other}). "
+                        "Configure separate paths outside reserved state files and directories."
+                    )
         return self
 
     def resolve(self, value: str) -> Path:
@@ -449,6 +469,8 @@
         raise ConfigError(
             f"Configuration not found: {config_path}. Run 'steadlith init' first."
         ) from exc
+    except UnicodeDecodeError as exc:
+        raise ConfigError(f"Configuration must be valid UTF-8: {config_path}") from exc
     except (OSError, tomllib.TOMLDecodeError) as exc:
         raise ConfigError(f"Could not read {config_path}: {exc}") from exc
     if _has_legacy_marker(raw, config_path):
@@ -572,6 +594,8 @@
             raw = tomllib.load(handle)
     except FileNotFoundError as exc:
         raise ConfigError(f"Configuration not found: {source_path}") from exc
+    except UnicodeDecodeError as exc:
+        raise ConfigError(f"Configuration must be valid UTF-8: {source_path}") from exc
     except (OSError, tomllib.TOMLDecodeError) as exc:
         raise ConfigError(f"Could not read {source_path}: {exc}") from exc
     config = _config_from_mapping(

src/steadlith/embed/providers/openai.py

@@ -33,13 +33,16 @@ def __init__(
         self.model = model
         self._dimensions = dimensions
         self._transient_error_types = (APIConnectionError, APITimeoutError)
+        endpoint = base_url or "https://api.openai.com/v1"
         try:
-            self._client: Any = OpenAI(api_key=api_key, base_url=base_url, max_retries=0)
+            # Passing None lets the SDK substitute OPENAI_BASE_URL, bypassing the
+            # configured endpoint restriction and disagreeing with cache identity.
+            self._client: Any = OpenAI(api_key=api_key, base_url=endpoint, max_retries=0)
         except Exception as exc:
             raise ProviderError(f"Could not initialize the OpenAI client: {exc}") from exc
         payload = json.dumps(
             {
-                "base_url": base_url or "https://api.openai.com/v1",
+                "base_url": endpoint,
                 "dimensions": dimensions,
                 "model": model,
                 "provider": "openai",