Add transactional derived-artifact materialization

.github/workflows/tests.yml

@@ -51,6 +51,7 @@ jobs:
         run: |
           python -m examples.basic_usage
           python -m examples.versioned_datasets
+          python -m examples.reference_dna_materialization
       - name: Publish coverage to Coveralls
         # Coverage upload is best-effort: parallel matrix jobs race to post to
         # the same Coveralls build, and the loser gets a "build already closed"

CHANGELOG.md

@@ -2,6 +2,18 @@
 
 ## 1.19.0
 
+- Add `materialize` and offline `inspect_materialization` for caller-built
+  multi-file artifacts with complete raw-input/transform/output dependency
+  receipts. Preserve completed inputs and HTTP resume state across failures,
+  publish immutable output generations atomically, and discard owned inputs
+  only after successful publication unless retention is requested. Local
+  sources stay caller-owned; scientific validation stays in the builder (#90).
+  `force=True` always rebuilds, copies local sources again while the original
+  exists, and repairs corrupt private input receipts.
+- Recover the newest complete generation when a bundle, archive or
+  materialization pointer is lost. Generation names are random, so recovery
+  previously chose an arbitrary matching generation and could roll back to
+  older content; it now orders by the creation time each receipt records.
 - Add a `store_path(name, version)` callback to `VersionedDatasetRegistry` for
   exact consumer-chosen managed-store locations. It is mutually exclusive with
   root configuration and does not run at construction; the first lookup

README.md

@@ -19,8 +19,9 @@ python -m pip install datacache
 python -m pip install "datacache[html]"
 ```
 
-tqdm is installed by default; progress displays are opt-in with
-`show_progress=True`. Normal use does not import tqdm or configure your
+tqdm is installed by default; existing download APIs enable displays with
+`show_progress=True`, while `materialize` enables them by default. Quiet use
+does not import tqdm or configure your
 application's logging. The old `datacache[progress]` install syntax remains
 supported as a compatibility alias.
 
@@ -61,6 +62,8 @@ guides; their old links land on the guide that replaced them.
 - [Progress and logging](https://github.com/openvax/datacache/blob/master/docs/progress.md)
 - <a id="versioned-datasets-and-large-downloads"></a>[Pinned bundles](https://github.com/openvax/datacache/blob/master/docs/bundles.md), [archive trees](https://github.com/openvax/datacache/blob/master/docs/archives.md) and the [legacy file registry](https://github.com/openvax/datacache/blob/master/docs/file_registry.md)
 - <a id="choose-the-right-api"></a><a id="guarantees-and-limits"></a>[Interface selection and guarantees](https://github.com/openvax/datacache/blob/master/docs/reference/choosing.md)
+- [Derived-artifact materialization](https://github.com/openvax/datacache/blob/master/docs/materialization.md): caller-owned
+  builders, atomic output generations, dependency receipts and resumable inputs.
 - [Complete API reference](https://github.com/openvax/datacache/blob/master/docs/api.md)
 - [Downstream integration](https://github.com/openvax/datacache/blob/master/docs/integration.md): contracts for consuming libraries.
 - [Release notes](https://github.com/openvax/datacache/blob/master/CHANGELOG.md) and [release procedure](https://github.com/openvax/datacache/blob/master/RELEASING.md).

datacache/__init__.py

@@ -38,6 +38,7 @@
 from .archives import (
     ArchiveInspection, VersionedArchiveRegistry, inspect_archive, install_archive)
 from .file_registry import VersionedFileRegistry
+from .materialization import MaterializationInspection, inspect_materialization, materialize
 from .version import __version__
 
 __all__ = [
@@ -46,6 +47,9 @@
     'fetch_bytes',
     'discard_partial',
     'BundleInspection',
+    'MaterializationInspection',
+    'inspect_materialization',
+    'materialize',
     'ArchiveInspection',
     'VersionedArchiveRegistry',
     'VersionedDatasetRegistry',

datacache/archives.py

@@ -26,7 +26,7 @@
 from filelock import FileLock
 
 from ._filesystem import open_regular, path_present, read_json, write_json
-from .bundles import _relative_name
+from .bundles import _newest_generations_first, _relative_name
 from .inspection import FileInspection
 from .integrity import FileValidationError, _validate_expectations
 from .provenance import redact_url
@@ -419,7 +419,8 @@ def _initialize(path):
 
 
 def _recover(path, definition):
-    for entry in sorted((path / "generations").iterdir(), reverse=True):
+    for name in _newest_generations_first(path, MANIFEST, "fetched_at"):
+        entry = path / "generations" / name
         try:
             candidate = _inspect_generation(path, entry.name, definition)
         except (OSError, ValueError, KeyError, TypeError, RecursionError):

datacache/bundles.py

@@ -88,10 +88,10 @@ def _directory(path):
         raise FileValidationError(path, 'expected a directory, not a link')
 
 
-def _store(path):
+def _store(path, *, marker=STORE):
     _directory(path)
-    if read_json(path / STORE) != {'format': FORMAT}:
-        raise FileValidationError(path, 'unrecognized bundle store')
+    if read_json(path / marker) != {'format': FORMAT}:
+        raise FileValidationError(path, 'unrecognized generation store')
     _directory(path / 'generations')
 
 
@@ -215,29 +215,29 @@ def _paths(inspection):
     return {name: value.path for name, value in inspection.files.items()}
 
 
-def _refuse_foreign(path):
-    """Raise if path is a populated directory that isn't a bundle store, which
-    neither installation nor force=True ever takes over."""
+def _refuse_foreign(path, *, marker=STORE):
+    """Raise if path is a populated directory that isn't a generation store,
+    which neither installation nor force=True ever takes over."""
     if (path_present(path) and stat.S_ISDIR(path.lstat().st_mode)
-            and not path_present(path / STORE) and any(path.iterdir())):
+            and not path_present(path / marker) and any(path.iterdir())):
         raise FileValidationError(
-            path, 'not a datacache bundle store; a populated directory is never taken over')
+            path, 'not a datacache generation store; a populated directory is never taken over')
 
 
-def _initialize(path):
+def _initialize(path, *, marker=STORE):
     existing_mode = None
     if path_present(path):
         _directory(path)
         if any(path.iterdir()):
-            _refuse_foreign(path)
-            _store(path)  # Never take over an arbitrary nonempty directory.
+            _refuse_foreign(path, marker=marker)
+            _store(path, marker=marker)  # Never adopt a foreign directory.
             return
         existing_mode = stat.S_IMODE(path.lstat().st_mode)
     # Publish a complete store skeleton at once: concurrent first-time readers
     # see absence or a recognized store, never a half-written ownership marker.
     staging = Path(tempfile.mkdtemp(prefix='.datacache-store-', dir=path.parent))
     try:
-        write_json(staging / STORE, {'format': FORMAT}, mode=_file_mode(path.parent))
+        write_json(staging / marker, {'format': FORMAT}, mode=_file_mode(path.parent))
         (staging / 'generations').mkdir()
         os.chmod(staging, existing_mode if existing_mode is not None else
                  stat.S_IMODE((staging / 'generations').stat().st_mode))
@@ -252,9 +252,28 @@ def _file_mode(path):
     return _normal_creation_mode(path)
 
 
+def _newest_generations_first(store, manifest, time_key):
+    """Generation names, newest first by the UTC time their receipt records
+    under time_key, else by directory modification time. Names are random
+    UUIDs, so their own order says nothing about age."""
+    def created(entry):
+        try:
+            recorded = datetime.fromisoformat(read_json(entry / manifest, limit=None)[time_key])
+            if recorded.tzinfo is None:
+                recorded = recorded.replace(tzinfo=timezone.utc)
+            return recorded.timestamp()
+        except (OSError, ValueError, KeyError, TypeError, RecursionError, OverflowError):
+            try:
+                return entry.lstat().st_mtime
+            except OSError:
+                return float('-inf')
+    entries = list((store / 'generations').iterdir())
+    return [entry.name for entry in sorted(entries, key=lambda e: (created(e), e.name), reverse=True)]
+
+
 def _recover(path, assets):
-    generations = path / 'generations'
-    for entry in sorted(generations.iterdir(), reverse=True):
+    for name in _newest_generations_first(path, MANIFEST, 'fetched_at'):
+        entry = path / 'generations' / name
         try:
             candidate = _inspect_generation(path, entry.name, assets)
         except (OSError, ValueError, KeyError, TypeError, RecursionError):