Develop

NEWS.md

@@ -1,5 +1,31 @@
 # 📰 Picopt News
 
+## v6.8.0
+
+### Fixes
+
+- Config files are written atomically. A crash or full disk during `-w`, `-W`,
+  or `--write-config-file` can no longer truncate a hand-written config.
+
+### Features
+
+- Timestamps survive picopt upgrades that record new options. Only a changed
+  option value invalidates them, so stamp files written before an option joined
+  the recorded set stay valid at its default. Timestamps from before 6.7.0 work
+  again.
+- Timestamp files now open with a comment explaining what they are, that they
+  are machine-written, and that deleting one just re-optimizes that tree.
+- Discarding timestamps because a sub-directory `.picopt.yaml` changed now says
+  so, instead of naming an internal key.
+- README documents `.picopt_treestamps.yaml`: what it records, that it is never
+  applied as configuration, and that it is safe to delete.
+
+### Changes
+
+- Requires treestamps 5.0.0. Timestamp files no longer duplicate `ignore` and
+  `symlinks` in a second block. Existing timestamp files still load, and
+  upgrading invalidates nothing.
+
 ## v6.7.0
 
 ### Features

README.md

@@ -312,8 +312,9 @@ Any config key is accepted and validated, but run-scoped keys — `dry_run`,
 `list_only`, `timestamps`, `after`, `jobs`, `memory_limit`, `fail_fast`,
 `fail_fast_container`, `verbose`, and `paths` — are governed by the run-level
 value; setting them in a directory file has no per-directory effect. When
-timestamps (`-t`) are enabled, editing, adding, or removing any `.picopt.yaml`
-re-processes its tree on the next run.
+timestamps (`-t`) are enabled, editing an option value in any `.picopt.yaml`, or
+adding or removing one, re-processes its tree on the next run. Comment and
+formatting edits do not.
 
 ### Writing config files
 
@@ -326,6 +327,39 @@ re-processes its tree on the next run.
 Existing keys and comments in the target file survive the merge. Run-mode flags
 (`-d`, `-L`, verbosity) and the target paths are never persisted.
 
+### The two files
+
+Picopt keeps two kinds of file in your directories, and they are opposites:
+
+`.picopt.yaml` is **your input**. You write it, picopt only reads it (or merges
+your invoked options into it when you ask with `-W`). Commit it.
+
+`.picopt_treestamps.yaml` — plus a `.picopt_treestamps.wal.yaml` while a run is
+in progress — is **picopt's memory**. It records when each file was last
+optimized, so already-optimized files are skipped on the next `-t` run. It is
+written by the program, never applied as configuration, and safe to delete: the
+tree is simply re-optimized on the next run. Keep it out of version control or
+commit it, whichever suits you; picopt does not care.
+
+Its header holds a snapshot of the options that produced those timestamps — not
+a copy of your `.picopt.yaml`, but of the _resolved_ options from every layer,
+including flags and environment variables that live in no file. When those
+options change, the recorded timestamps no longer describe what a run would do,
+so they are discarded and the tree is re-examined. Picopt prints which options
+differed. Options that don't affect which files are selected are not recorded,
+and adding or retiring a recorded option does not invalidate anything by itself:
+only a changed value does.
+
+Timestamp files you should know about:
+
+- Deleting one re-optimizes that tree.
+- Running from a higher directory absorbs and removes the stamp files beneath
+  it.
+- Stamp files inside archives are consumed and dropped when the archive is
+  repacked.
+- `-N` / `--timestamps-no-check-config` skips the option comparison for a run,
+  keeping timestamps that would otherwise be discarded.
+
 ## ⌨️ Use Examples
 
 Optimize all JPEG files in a directory:

package.json

@@ -65,7 +65,7 @@
     "@prettier/plugin-xml": "^3.4.2",
     "@stylistic/eslint-plugin": "^5.10.0",
     "eslint": "^10.8.0",
-    "eslint_d": "^15.0.2",
+    "eslint_d": "^15.0.3",
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-array-func": "^5.1.1",
     "eslint-plugin-compat": "^7.0.2",
@@ -87,7 +87,7 @@
     "eslint-plugin-sonarjs": "^4.2.0",
     "eslint-plugin-toml": "^1.5.0",
     "eslint-plugin-unicorn": "^72.0.0",
-    "eslint-plugin-yml": "^3.6.0",
+    "eslint-plugin-yml": "^3.7.0",
     "prettier": "^3.9.6",
     "prettier-plugin-nginx": "^1.0.3",
     "prettier-plugin-packagejson": "^3.0.2",

picopt/config/__init__.py

@@ -15,6 +15,7 @@
 import time
 from contextlib import suppress
 from pathlib import Path
+from tempfile import NamedTemporaryFile
 from typing import TYPE_CHECKING, Any, Final
 
 from confuse import Configuration, MappingTemplate
@@ -208,8 +209,10 @@ def merge_config_file(target: Path, base_path: Path, options: dict) -> None:
     Merge ``options`` into ``base_path``'s config section and write to ``target``.
 
     Round-trip loads ``base_path`` so existing keys and comments survive,
-    then writes owner-only. Raises ``YAMLError`` or ``OSError`` on
-    read/write failure so callers decide whether it is fatal.
+    then writes owner-only via a sibling temp file and an atomic replace, so
+    a crash mid-write cannot truncate a hand-authored config. Raises
+    ``YAMLError`` or ``OSError`` on read/write failure so callers decide
+    whether it is fatal.
     """
     yaml = YAML()
     data = yaml.load(base_path.read_text()) if base_path.is_file() else None
@@ -221,11 +224,25 @@ def merge_config_file(target: Path, base_path: Path, options: dict) -> None:
         data[PROGRAM_NAME] = section
     section.update(options)
     target.parent.mkdir(parents=True, exist_ok=True)
-    with target.open("w") as stream:
-        yaml.dump(data, stream)
-    # Best effort: filesystems without POSIX modes (e.g. FAT) just skip it.
-    with suppress(OSError):
-        target.chmod(0o600)
+    # A unique temp name, not a fixed sibling: two concurrent writers to the
+    # same target would otherwise unlink each other's temp mid-write. Same
+    # directory keeps the replace off a cross-device path.
+    tmp = NamedTemporaryFile(  # noqa: SIM115
+        "w", dir=target.parent, prefix=target.name + ".", delete=False
+    )
+    tmp_path = Path(tmp.name)
+    try:
+        with tmp:
+            yaml.dump(data, tmp)
+            tmp.flush()
+            os.fsync(tmp.fileno())
+        # Best effort: filesystems without POSIX modes (e.g. FAT) just skip
+        # it. Set before the replace so the target is never briefly wider.
+        with suppress(OSError):
+            tmp_path.chmod(0o600)
+        tmp_path.replace(target)
+    finally:
+        tmp_path.unlink(missing_ok=True)
 
 
 def _write_merged_config(

picopt/config/consts.py

@@ -8,7 +8,14 @@
 allowlist, which doesn't depend on the format registry at all.
 """
 
-from typing import Final
+from collections.abc import Mapping
+from importlib.resources import files
+from types import MappingProxyType
+from typing import Any, Final
+
+from ruamel.yaml import YAML
+
+from picopt import PROGRAM_NAME
 
 # Per-directory config files layer beneath env vars and CLI args but above
 # the user config. Named to be distinct from the timestamps file
@@ -35,3 +42,25 @@
         "timestamps_ignore_archive_entry_mtimes",
     }
 )
+
+
+def _packaged_defaults() -> Mapping[str, Any]:
+    """Load the packaged default config values."""
+    text = (files(PROGRAM_NAME) / "config_default.yaml").read_text()
+    return YAML(typ="safe").load(text)[PROGRAM_NAME]
+
+
+# Treestamps fills keys missing from either side of a config comparison
+# from these before diffing, so adding or retiring a recorded key does not
+# invalidate stamp files written before the change. A KeyError here is a
+# deliberate import-time guard: every recorded key needs a packaged default.
+TIMESTAMPS_CONFIG_DEFAULTS: Final[Mapping[str, Any]] = MappingProxyType(
+    {key: _packaged_defaults()[key] for key in TIMESTAMPS_CONFIG_KEYS}
+)
+
+# When a key leaves TIMESTAMPS_CONFIG_KEYS, move its last packaged default
+# here: stamp files that recorded it at its default stay valid, while ones
+# that recorded another value still invalidate. Keys *added* to
+# TIMESTAMPS_CONFIG_KEYS must default to behavior-preserving values,
+# because a key missing from an old stamp file reads as the current default.
+RETIRED_TIMESTAMPS_CONFIG_DEFAULTS: Final[Mapping[str, Any]] = MappingProxyType({})