diff --git a/core/activity.py b/core/activity.py
index c7a6c7e..a5165a4 100644
--- a/core/activity.py
+++ b/core/activity.py
@@ -123,6 +123,10 @@ class FileActivity:
# (treated as legacy, bucketed by 15-min time windows by activity_grouping).
run_id: Optional[str] = None
run_source: str = "legacy" # "scheduled" | "web" | "cli" | "maintenance" | "legacy"
+ # True when this row is a header gathering a title's associated files and the
+ # video itself did not move. Without it the action badge sits beside a .mkv
+ # name and reads as though the video was copied.
+ sidecars_only: bool = False
def to_dict(self, time_format: Optional[str] = None) -> dict:
"""Serialize for the dashboard/API.
@@ -166,6 +170,8 @@ def to_dict(self, time_format: Optional[str] = None) -> dict:
}
if self.associated_files:
result["associated_files"] = self.associated_files
+ if self.sidecars_only:
+ result["sidecars_only"] = True
return result
def _format_size(self, size_bytes: int) -> str:
@@ -205,6 +211,7 @@ def _load_activity_unlocked() -> List[FileActivity]:
associated_files=item.get('associated_files', []),
run_id=item.get('run_id'),
run_source=item.get('run_source', 'legacy'),
+ sidecars_only=item.get('sidecars_only', False),
))
except (KeyError, ValueError):
continue # Skip malformed entries
@@ -243,6 +250,8 @@ def _save_activity_unlocked(activities: List[FileActivity]) -> None:
entry['run_id'] = activity.run_id
if activity.run_source and activity.run_source != "legacy":
entry['run_source'] = activity.run_source
+ if activity.sidecars_only:
+ entry['sidecars_only'] = True
data.append(entry)
save_json_atomically(str(ACTIVITY_FILE), data, label="activity")
diff --git a/tests/test_operation_eta.py b/tests/test_operation_eta.py
new file mode 100644
index 0000000..fb365e7
--- /dev/null
+++ b/tests/test_operation_eta.py
@@ -0,0 +1,138 @@
+"""The remaining-time estimate prices files as well as bytes.
+
+Measured against a real run (2026-08-12, 199 files / 287.53 GB): at 14m 11s of
+copying, 171.16 GB was done across 43 files. The old pure-byte model reported
+9m 38s left; the run actually needed 11m 27s. It was optimistic because the 43
+completed files averaged ~4 GB while the 156 remaining averaged ~0.75 GB, many
+of them artwork and NFOs whose cost is per-file rather than per-byte.
+"""
+
+import sys
+from unittest.mock import MagicMock
+
+import pytest
+
+sys.modules.setdefault('fcntl', MagicMock())
+for _mod_name in ['plexapi', 'plexapi.server', 'plexapi.video', 'plexapi.myplex',
+ 'plexapi.exceptions', 'requests', 'apscheduler',
+ 'apscheduler.schedulers', 'apscheduler.schedulers.background',
+ 'apscheduler.triggers', 'apscheduler.triggers.cron',
+ 'apscheduler.triggers.interval']:
+ sys.modules.setdefault(_mod_name, MagicMock())
+
+GB = 1024 ** 3
+
+
+def _estimate(**kw):
+ from web.services.operation_runner import OperationRunner
+ return OperationRunner._estimate_remaining_seconds(**kw)
+
+
+class TestTheRealRun:
+ """Numbers taken from the 08:30 run on 2026-08-12."""
+
+ ELAPSED = 14 * 60 + 11 # copying began 08:50:09, sampled 09:04:20
+ DONE = int(171.16 * GB)
+ REMAINING = int((287.53 - 171.16) * GB)
+ FILES_DONE = 43
+ FILES_LEFT = 156
+ ACTUAL_LEFT = 11 * 60 + 27 # finished 09:15:47
+
+ def test_pure_byte_rate_is_the_optimistic_baseline(self):
+ """What the old model produced, kept as the thing to beat."""
+ rate = self.DONE / self.ELAPSED
+ naive = self.REMAINING / rate
+
+ assert 560 < naive < 600, naive # ~9m 38s
+ assert naive < self.ACTUAL_LEFT * 0.9 # meaningfully under
+
+ def test_per_file_term_closes_most_of_the_gap(self):
+ # A sustained rate slightly above the batch average, as a peak sample
+ # taken during large-file transfer would be.
+ peak = (self.DONE / self.ELAPSED) * 1.08
+
+ eta = _estimate(elapsed=self.ELAPSED, bytes_done=self.DONE,
+ bytes_remaining=self.REMAINING, files_done=self.FILES_DONE,
+ files_remaining=self.FILES_LEFT, peak_rate=peak)
+
+ naive = self.REMAINING / (self.DONE / self.ELAPSED)
+ assert eta > naive, "must be less optimistic than the pure byte model"
+ assert abs(eta - self.ACTUAL_LEFT) < abs(naive - self.ACTUAL_LEFT), (
+ f"estimate {eta:.0f}s should beat naive {naive:.0f}s "
+ f"against actual {self.ACTUAL_LEFT}s"
+ )
+
+
+class TestBehaviour:
+
+ def test_a_tail_of_tiny_files_is_not_free(self):
+ """500 sidecars totalling almost nothing still take time."""
+ eta = _estimate(elapsed=600, bytes_done=100 * GB, bytes_remaining=1024,
+ files_done=20, files_remaining=500, peak_rate=200 * 1024 ** 2)
+
+ assert eta > 30, f"a 500-file tail priced at {eta:.1f}s is not credible"
+
+ def test_matches_the_byte_model_when_no_per_file_cost_shows(self):
+ """A perfectly byte-bound run should not be inflated."""
+ rate = 100 * 1024 ** 2
+ eta = _estimate(elapsed=100, bytes_done=100 * rate,
+ bytes_remaining=50 * rate, files_done=1,
+ files_remaining=1, peak_rate=rate)
+
+ assert eta == pytest.approx(50, rel=0.02)
+
+ def test_peak_rate_is_neutral_when_the_tail_mirrors_the_head(self):
+ """Splitting one measured elapsed into two terms must not invent time.
+
+ With the remaining work shaped like the completed work, any split of
+ cost between bytes and files reproduces the same total. That the peak
+ rate changes nothing here is the model behaving.
+ """
+ kw = dict(elapsed=100, bytes_done=10 * GB, bytes_remaining=10 * GB,
+ files_done=5, files_remaining=5)
+
+ assert _estimate(peak_rate=0, **kw) == pytest.approx(
+ _estimate(peak_rate=(10 * GB / 100) * 2, **kw))
+
+ def test_peak_rate_is_what_prices_a_small_file_tail(self):
+ """Without it the per-file cost is invisible and the tail reads as free.
+
+ This is the case that motivated the change: bytes nearly exhausted,
+ hundreds of sidecars left. On the batch average alone the residual is
+ zero, so the estimate collapses to nothing.
+ """
+ kw = dict(elapsed=100, bytes_done=10 * GB, bytes_remaining=1024,
+ files_done=5, files_remaining=500)
+
+ assert _estimate(peak_rate=0, **kw) == pytest.approx(0, abs=1)
+ assert _estimate(peak_rate=(10 * GB / 100) * 2, **kw) > 60
+
+ @pytest.mark.parametrize("kw", [
+ {"elapsed": 0, "bytes_done": 1024},
+ {"elapsed": 10, "bytes_done": 0},
+ ])
+ def test_returns_none_with_nothing_to_measure(self, kw):
+ assert _estimate(bytes_remaining=1024, files_done=0, files_remaining=1,
+ peak_rate=0, **kw) is None
+
+ def test_zero_remaining_is_zero(self):
+ eta = _estimate(elapsed=100, bytes_done=10 * GB, bytes_remaining=0,
+ files_done=10, files_remaining=0, peak_rate=0)
+
+ assert eta == 0
+
+
+class TestDisplayDenominatorsAgree:
+
+ def test_banner_leads_with_bytes_when_available(self):
+ """The bar and ETA are byte-based; the headline number must match."""
+ import pathlib
+ html = (pathlib.Path(__file__).resolve().parent.parent / "web" / "templates" /
+ "components" / "global_operation_banner.html").read_text(encoding="utf-8")
+
+ # There are two progress-meta blocks; pick the operation one.
+ block = html[html.index("status.has_byte_progress"):]
+ block = block[:block.index("")]
+ assert block.index("bytes_display") < block.index("completed_files"), (
+ "file count still leads, which is what made a correct ETA look wrong"
+ )
diff --git a/tests/test_orphan_sidecar_grouping.py b/tests/test_orphan_sidecar_grouping.py
new file mode 100644
index 0000000..49e36c9
--- /dev/null
+++ b/tests/test_orphan_sidecar_grouping.py
@@ -0,0 +1,197 @@
+"""Sidecars copied without their video still get grouped under it.
+
+Artwork and NFOs are frequently copied on their own, because the video is
+already on cache and only its metadata changed. The sibling merge indexed
+parents from the current run only, so those sidecars had nothing to fold into
+and rendered as several unattributed rows — four "…-poster.jpg" lines with
+nothing naming the film.
+
+The parent is known regardless (sibling_map covers every video with siblings,
+not just the ones this run touched), so the group header can name it. The
+header carries no size: the video did not move, and a size there would say it
+did.
+"""
+
+import sys
+from datetime import datetime
+from unittest.mock import MagicMock
+
+import pytest
+
+sys.modules.setdefault('fcntl', MagicMock())
+for _mod_name in ['plexapi', 'plexapi.server', 'plexapi.video', 'plexapi.myplex',
+ 'plexapi.exceptions', 'requests', 'apscheduler',
+ 'apscheduler.schedulers', 'apscheduler.schedulers.background',
+ 'apscheduler.triggers', 'apscheduler.triggers.cron',
+ 'apscheduler.triggers.interval']:
+ sys.modules.setdefault(_mod_name, MagicMock())
+
+MOVIE = "Half Baked (1998) - [BLURAY-1080P][EAC3 5.1][X264][8Bit]-J3RICO.mkv"
+STEM = "Half Baked (1998) - [BLURAY-1080P][EAC3 5.1][X264][8Bit]-J3RICO"
+
+_COMPATIBLE = {
+ "Restored": ("Restored", "Moved"),
+ "Moved": ("Restored", "Moved"),
+ "Cached": ("Cached",),
+}
+
+
+def _runner(files):
+ from web.services.operation_runner import OperationRunner
+ r = object.__new__(OperationRunner)
+ r._current_run_files = files
+ return r
+
+
+def _sidecar(name, size="1.00 MB", action="Cached"):
+ return {"action": action, "filename": name, "size": size, "size_bytes": 1024}
+
+
+def _parent_map(*sidecar_names, parent=MOVIE):
+ return {s: parent for s in sidecar_names}
+
+
+class TestSidecarsWithoutTheirVideo:
+
+ def test_grouped_under_a_header_naming_the_film(self):
+ files = [_sidecar(f"{STEM}{ext}") for ext in
+ ("-poster.jpg", "-fanart.jpg", "-clearlogo.png", ".nfo")]
+ runner = _runner(list(files))
+
+ runner._merge_run_files(_parent_map(*[f["filename"] for f in files]), _COMPATIBLE)
+
+ assert len(runner._current_run_files) == 1, runner._current_run_files
+ header = runner._current_run_files[0]
+ assert header["filename"] == MOVIE
+ assert len(header["associated_files"]) == 4
+ assert header["sidecars_only"] is True
+
+ def test_header_carries_no_size(self):
+ """The video did not move; a size here would claim it did."""
+ files = [_sidecar(f"{STEM}-poster.jpg"), _sidecar(f"{STEM}.nfo")]
+ runner = _runner(list(files))
+
+ runner._merge_run_files(_parent_map(*[f["filename"] for f in files]), _COMPATIBLE)
+
+ header = runner._current_run_files[0]
+ assert header["size"] == ""
+ assert header["size_bytes"] == 0
+
+ def test_a_lone_sidecar_is_left_alone(self):
+ """Its own filename already names the film; a header adds a row, not information."""
+ files = [_sidecar(f"{STEM}.nfo")]
+ runner = _runner(list(files))
+
+ runner._merge_run_files(_parent_map(f"{STEM}.nfo"), _COMPATIBLE)
+
+ assert len(runner._current_run_files) == 1
+ assert runner._current_run_files[0]["filename"] == f"{STEM}.nfo"
+ assert "associated_files" not in runner._current_run_files[0]
+
+ def test_real_parent_still_wins_over_a_synthetic_one(self):
+ """When the video IS in the run, nothing changes."""
+ video = {"action": "Cached", "filename": MOVIE, "size": "4.00 GB",
+ "size_bytes": 4 * 1024 ** 3}
+ files = [video] + [_sidecar(f"{STEM}-poster.jpg"), _sidecar(f"{STEM}.nfo")]
+ runner = _runner(list(files))
+
+ runner._merge_run_files(
+ _parent_map(f"{STEM}-poster.jpg", f"{STEM}.nfo"), _COMPATIBLE)
+
+ assert len(runner._current_run_files) == 1
+ header = runner._current_run_files[0]
+ assert header["size_bytes"] == 4 * 1024 ** 3, "the real video row must survive"
+ assert "sidecars_only" not in header
+ assert len(header["associated_files"]) == 2
+
+ def test_two_films_do_not_merge_into_one_group(self):
+ other = "Saltburn (2023) - [WEBDL-1080P][EAC3 ATMOS 5.1][H264][8Bit]-RE.mkv"
+ other_stem = other[:-4]
+ files = [_sidecar(f"{STEM}-poster.jpg"), _sidecar(f"{STEM}.nfo"),
+ _sidecar(f"{other_stem}-poster.jpg"), _sidecar(f"{other_stem}.nfo")]
+ mapping = {f"{STEM}-poster.jpg": MOVIE, f"{STEM}.nfo": MOVIE,
+ f"{other_stem}-poster.jpg": other, f"{other_stem}.nfo": other}
+ runner = _runner(list(files))
+
+ runner._merge_run_files(mapping, _COMPATIBLE)
+
+ names = sorted(f["filename"] for f in runner._current_run_files)
+ assert names == sorted([MOVIE, other]), names
+
+ def test_unrelated_rows_are_untouched(self):
+ unrelated = {"action": "Cached", "filename": "Troy (2004).mkv",
+ "size": "14.00 GB", "size_bytes": 14 * 1024 ** 3}
+ files = [unrelated, _sidecar(f"{STEM}-poster.jpg"), _sidecar(f"{STEM}.nfo")]
+ runner = _runner(list(files))
+
+ runner._merge_run_files(
+ _parent_map(f"{STEM}-poster.jpg", f"{STEM}.nfo"), _COMPATIBLE)
+
+ assert unrelated in runner._current_run_files
+ assert len(runner._current_run_files) == 2
+
+ def test_actions_are_not_mixed_into_one_group(self):
+ """A cached sidecar and a restored one describe different events."""
+ files = [_sidecar(f"{STEM}-poster.jpg", action="Cached"),
+ _sidecar(f"{STEM}.nfo", action="Cached"),
+ _sidecar(f"{STEM}-fanart.jpg", action="Restored")]
+ runner = _runner(list(files))
+
+ runner._merge_run_files(
+ _parent_map(f"{STEM}-poster.jpg", f"{STEM}.nfo", f"{STEM}-fanart.jpg"),
+ _COMPATIBLE)
+
+ actions = sorted(f["action"] for f in runner._current_run_files)
+ assert actions == ["Cached", "Restored"], actions
+
+
+class TestTheHeaderDoesNotClaimTheVideoMoved:
+ """The header names a .mkv that was never copied — the badge must say so."""
+
+ def test_run_file_header_is_flagged(self):
+ files = [_sidecar(f"{STEM}-poster.jpg"), _sidecar(f"{STEM}.nfo")]
+ runner = _runner(list(files))
+
+ runner._merge_run_files(_parent_map(*[f["filename"] for f in files]), _COMPATIBLE)
+
+ assert runner._current_run_files[0]["sidecars_only"] is True
+
+ def test_activity_entries_round_trip_the_flag(self):
+ """It is persisted, so a reload must not turn it back into a Cached row."""
+ from core.activity import FileActivity
+
+ entry = FileActivity(timestamp=datetime.now(), action="Cached",
+ filename=MOVIE, size_bytes=0, sidecars_only=True)
+
+ assert entry.to_dict().get("sidecars_only") is True
+
+ def test_normal_rows_carry_no_flag(self):
+ from core.activity import FileActivity
+
+ entry = FileActivity(timestamp=datetime.now(), action="Cached",
+ filename=MOVIE, size_bytes=1024)
+
+ assert "sidecars_only" not in entry.to_dict()
+
+ def test_banner_renders_a_neutral_tag(self):
+ """Not the green CACHED tag, which would claim the video was copied."""
+ import pathlib, re
+ html = (pathlib.Path(__file__).resolve().parent.parent / "web" / "templates" /
+ "components" / "global_operation_banner.html").read_text(encoding="utf-8")
+
+ assert "f.sidecars_only" in html
+ # Only the truthy branch — the else branch legitimately styles real rows.
+ branch = html[html.index("f.sidecars_only"):]
+ branch = branch[:branch.index("{% else %}")]
+ assert "EXTRAS" in branch
+ assert "di-action-tag--cached" not in branch, (
+ "the sidecars-only header must not use the cached styling"
+ )
+
+ def test_activity_feed_renders_a_neutral_badge(self):
+ import pathlib
+ html = (pathlib.Path(__file__).resolve().parent.parent / "web" / "templates" /
+ "components" / "recent_activity.html").read_text(encoding="utf-8")
+
+ assert "entry.sidecars_only" in html
+ assert "Extras only" in html
diff --git a/web/services/operation_runner.py b/web/services/operation_runner.py
index 1549806..24a22c3 100644
--- a/web/services/operation_runner.py
+++ b/web/services/operation_runner.py
@@ -42,6 +42,11 @@
SETTINGS_FILE = CONFIG_SETTINGS_FILE
+# Sidecars whose parent video is not in the run are gathered under a header row
+# naming that video. Two is the threshold: grouping a lone sidecar would turn one
+# row into two without telling the reader anything its own filename does not.
+_ORPHAN_GROUP_MIN = 2
+
class OperationState(str, Enum):
"""Operation states"""
@@ -86,6 +91,14 @@ class OperationResult:
cumulative_bytes_copied: int = 0
cumulative_bytes_total: int = 0
_prev_batch_cumulative: int = 0 # internal: snapshot at batch start
+ # Best sustained transfer rate seen this run, in bytes/sec. Copy time is
+ # bytes/rate PLUS a fixed cost per file (open, verify, chown, tracker
+ # write), so the batch average conflates the two and reads high while big
+ # files dominate. Tracking the peak separates them: the residual between
+ # elapsed and bytes/peak_rate is what the per-file cost has to explain.
+ peak_byte_rate: float = 0.0
+ _rate_last_time: Optional[float] = None
+ _rate_last_bytes: int = 0
class WebLogHandler(logging.Handler):
@@ -1033,15 +1046,29 @@ def _run_operation(self, dry_run: bool, verbose: bool = False):
def _bytes_cb(bytes_copied: int, bytes_total: int):
with self._lock:
r = self._current_result
+ now = time.time()
if bytes_copied == 0:
# New batch starting — snapshot cumulative progress
- r.batch_copy_start_time = time.time()
+ r.batch_copy_start_time = now
r._prev_batch_cumulative = r.cumulative_bytes_copied
r.cumulative_bytes_total = r._prev_batch_cumulative + bytes_total
+ r._rate_last_time = now
+ r._rate_last_bytes = 0
r.batch_bytes_copied = bytes_copied
r.batch_bytes_total = bytes_total
r.cumulative_bytes_copied = r._prev_batch_cumulative + bytes_copied
+ # Sample the sustained rate over >=2s windows. Shorter ones
+ # are dominated by scheduling noise and would inflate the
+ # peak, which then hides the per-file cost in the ETA.
+ if r._rate_last_time is not None:
+ dt = now - r._rate_last_time
+ db = bytes_copied - r._rate_last_bytes
+ if dt >= 2.0 and db > 0:
+ r.peak_byte_rate = max(r.peak_byte_rate, db / dt)
+ r._rate_last_time = now
+ r._rate_last_bytes = bytes_copied
+
# Create and run the app
app = PlexCacheApp(
config_file=config_file,
@@ -1187,25 +1214,68 @@ def _merge_sibling_activities(self, sibling_map: Dict[str, list]) -> None:
parent_index[key] = i
merged_indices: set = set()
+ orphans: Dict[tuple, list] = {}
+ orphan_key_at: Dict[int, tuple] = {}
+
for i, act in enumerate(activities):
- if act.filename in sibling_to_parent:
- parent_basename = sibling_to_parent[act.filename]
- # Try compatible actions (e.g. Moved sibling → Restored parent)
- compatible = _COMPATIBLE_ACTIONS.get(act.action, (act.action,))
- for try_action in compatible:
- parent_key = (parent_basename, try_action)
- if parent_key in parent_index:
- parent_idx = parent_index[parent_key]
- parent_act = activities[parent_idx]
- parent_act.associated_files.append({
- "filename": act.filename,
- "size": format_bytes(act.size_bytes) if act.size_bytes > 0 else "",
- })
- merged_indices.add(i)
- break
+ if act.filename not in sibling_to_parent:
+ continue
+ parent_basename = sibling_to_parent[act.filename]
+ # Try compatible actions (e.g. Moved sibling → Restored parent)
+ compatible = _COMPATIBLE_ACTIONS.get(act.action, (act.action,))
+ merged = False
+ for try_action in compatible:
+ parent_key = (parent_basename, try_action)
+ if parent_key in parent_index:
+ parent_idx = parent_index[parent_key]
+ parent_act = activities[parent_idx]
+ parent_act.associated_files.append({
+ "filename": act.filename,
+ "size": format_bytes(act.size_bytes) if act.size_bytes > 0 else "",
+ })
+ merged_indices.add(i)
+ merged = True
+ break
+ if not merged:
+ key = (parent_basename, act.action)
+ orphans.setdefault(key, []).append(i)
+ orphan_key_at.setdefault(key, i)
+
+ # Sidecars copied without their video (already on cache) have no parent
+ # to fold into. Give them one, so the feed names the film instead of
+ # listing loose artwork.
+ orphans = {k: v for k, v in orphans.items() if len(v) >= _ORPHAN_GROUP_MIN}
+ for idxs in orphans.values():
+ merged_indices.update(idxs)
if merged_indices:
- activities = [a for i, a in enumerate(activities) if i not in merged_indices]
+ rebuilt: list = []
+ emitted: set = set()
+ for i, act in enumerate(activities):
+ if i not in merged_indices:
+ rebuilt.append(act)
+ continue
+ for key, first_i in orphan_key_at.items():
+ if key in orphans and first_i == i and key not in emitted:
+ emitted.add(key)
+ first = activities[i]
+ rebuilt.append(FileActivity(
+ timestamp=first.timestamp,
+ action=key[1],
+ filename=key[0],
+ size_bytes=0, # the video did not move
+ users=list(first.users),
+ run_id=first.run_id,
+ run_source=first.run_source,
+ sidecars_only=True,
+ associated_files=[{
+ "filename": activities[j].filename,
+ "size": (format_bytes(activities[j].size_bytes)
+ if activities[j].size_bytes > 0 else ""),
+ } for j in orphans[key]],
+ ))
+ break
+ activities = rebuilt
_save_activity_unlocked(activities)
# Update in-memory list and merge _current_run_files for banner pill
@@ -1229,26 +1299,124 @@ def _merge_run_files(self, sibling_to_parent: Dict[str, str], compatible_actions
parent_index[key] = i
merged_indices: set = set()
+ # Sidecars whose parent video is not in this run — see _group_orphan_sidecars.
+ orphans: Dict[tuple, list] = {}
+ orphan_key_at: Dict[int, tuple] = {}
+
for i, f in enumerate(self._current_run_files):
- if f["filename"] in sibling_to_parent:
- parent_basename = sibling_to_parent[f["filename"]]
- compatible = compatible_actions.get(f["action"], (f["action"],))
- for try_action in compatible:
- parent_key = (parent_basename, try_action)
- if parent_key in parent_index:
- parent_idx = parent_index[parent_key]
- parent_entry = self._current_run_files[parent_idx]
- if "associated_files" not in parent_entry:
- parent_entry["associated_files"] = []
- parent_entry["associated_files"].append({
- "filename": f["filename"],
- "size": f.get("size", ""),
- })
- merged_indices.add(i)
- break
+ if f["filename"] not in sibling_to_parent:
+ continue
+ parent_basename = sibling_to_parent[f["filename"]]
+ compatible = compatible_actions.get(f["action"], (f["action"],))
+ merged = False
+ for try_action in compatible:
+ parent_key = (parent_basename, try_action)
+ if parent_key in parent_index:
+ parent_idx = parent_index[parent_key]
+ parent_entry = self._current_run_files[parent_idx]
+ if "associated_files" not in parent_entry:
+ parent_entry["associated_files"] = []
+ parent_entry["associated_files"].append({
+ "filename": f["filename"],
+ "size": f.get("size", ""),
+ })
+ merged_indices.add(i)
+ merged = True
+ break
+ if not merged:
+ key = (parent_basename, f["action"])
+ orphans.setdefault(key, []).append(f)
+ orphan_key_at.setdefault(key, i)
+
+ orphans = {k: v for k, v in orphans.items() if len(v) >= _ORPHAN_GROUP_MIN}
+ for key, entries in orphans.items():
+ for f in entries:
+ merged_indices.add(self._current_run_files.index(f))
+
+ if not merged_indices:
+ return
+
+ rebuilt: list = []
+ emitted: set = set()
+ for i, f in enumerate(self._current_run_files):
+ if i not in merged_indices:
+ rebuilt.append(f)
+ continue
+ for key, first_i in orphan_key_at.items():
+ if key in orphans and first_i == i and key not in emitted:
+ emitted.add(key)
+ rebuilt.append(self._synthetic_sidecar_parent(key, orphans[key]))
+ break
+ self._current_run_files = rebuilt
+
+ @staticmethod
+ def _estimate_remaining_seconds(*, elapsed: float, bytes_done: int,
+ bytes_remaining: int, files_done: int,
+ files_remaining: int,
+ peak_rate: float) -> Optional[float]:
+ """Seconds left, priced per byte and per file.
+
+ A pure bytes/second model assumes every remaining byte costs the same
+ as the ones already copied. That holds while the queue is video files
+ and breaks at the tail, where artwork and NFOs are thousands of times
+ smaller but still cost a copy, a size verify, a chown and a tracker
+ write each. The estimate then reads far too low exactly when a user is
+ watching it, and the run appears stuck near the end.
+
+ Two terms instead:
+
+ remaining = bytes_remaining / rate + files_remaining * per_file
+
+ ``rate`` is the best sustained rate seen, which is the throughput of
+ bulk transfer with per-file costs mostly excluded. Whatever elapsed
+ time that does not explain is attributed to the files completed so
+ far, giving ``per_file``. Both are measured from this run, so a slow
+ array or a busy mover is reflected without any tuning constant.
+
+ Falls back to the plain byte rate when there is not enough to fit,
+ and returns None when there is nothing to estimate from.
+ """
+ if elapsed <= 0 or bytes_done <= 0:
+ return None
- if merged_indices:
- self._current_run_files = [f for i, f in enumerate(self._current_run_files) if i not in merged_indices]
+ average_rate = bytes_done / elapsed
+ rate = peak_rate if peak_rate > average_rate else average_rate
+ if rate <= 0:
+ return None
+
+ # Time the bytes alone cannot account for is per-file cost.
+ per_file = 0.0
+ if files_done > 0:
+ unexplained = elapsed - (bytes_done / rate)
+ if unexplained > 0:
+ per_file = unexplained / files_done
+
+ return (bytes_remaining / rate) + (files_remaining * per_file)
+
+ @staticmethod
+ def _synthetic_sidecar_parent(key: tuple, entries: list) -> dict:
+ """A header row for sidecars whose parent video was not part of this run.
+
+ Artwork and NFOs are often copied on their own, because the video is
+ already on cache. Without a parent to fold into they rendered as
+ several unattributed rows — four "…-poster.jpg" lines with nothing
+ naming the film. This carries the parent's filename so the group reads
+ as what it is.
+
+ No size on the header: the video did not move, and showing the sidecar
+ total there would imply it did.
+ """
+ parent_basename, action = key
+ return {
+ "action": action,
+ "filename": parent_basename,
+ "size": "",
+ "size_bytes": 0,
+ "sidecars_only": True,
+ "associated_files": [
+ {"filename": e["filename"], "size": e.get("size", "")} for e in entries
+ ],
+ }
# Show-episode grouping helper lives in core.activity (shared with the
# Recent Activity dashboard grouping service). Imported as
@@ -1357,14 +1525,22 @@ def get_status_dict(self) -> dict:
status["progress_percent"] = min(int(cumul_copied / cumul_total * 100), 100)
status["bytes_display"] = f"{self._format_bytes(cumul_copied)} / {self._format_bytes(cumul_total)}"
- # ETA from current batch byte rate
+ status["has_byte_progress"] = True
+
+ # ETA from the batch, priced per byte AND per file.
if result.batch_bytes_copied > 0 and result.batch_copy_start_time:
copy_elapsed = time.time() - result.batch_copy_start_time
if copy_elapsed > 0:
- rate = result.batch_bytes_copied / copy_elapsed
- remaining = cumul_total - cumul_copied
- if rate > 0:
- status["eta_display"] = self._format_duration(remaining / rate)
+ eta = self._estimate_remaining_seconds(
+ elapsed=copy_elapsed,
+ bytes_done=result.batch_bytes_copied,
+ bytes_remaining=max(0, cumul_total - cumul_copied),
+ files_done=completed_files,
+ files_remaining=max(0, total_files - completed_files),
+ peak_rate=result.peak_byte_rate,
+ )
+ if eta is not None:
+ status["eta_display"] = self._format_duration(eta)
# Recent log messages (last 5) for hover mini-log
# Files completed so far in this run for detail panel
diff --git a/web/templates/components/global_operation_banner.html b/web/templates/components/global_operation_banner.html
index 35b74cf..aa9aeff 100644
--- a/web/templates/components/global_operation_banner.html
+++ b/web/templates/components/global_operation_banner.html
@@ -829,11 +829,17 @@
{% if status.total_files is defined and status.total_files > 0 %}
+ {#- The bar and the ETA are both byte-based, so bytes lead. Showing the
+ file count first made them disagree: 43/199 files reads as a fifth
+ done while the same moment was 60% of the bytes, which made a
+ correct "9m left" look impossible. -#}
- {{ status.completed_files }}/{{ status.total_files }} file{{ 's' if status.total_files != 1 }}
+
+ {%- if status.has_byte_progress and status.bytes_display %}{{ status.bytes_display }}
+ {%- else %}{{ status.completed_files }}/{{ status.total_files }} file{{ 's' if status.total_files != 1 }}{% endif %}
{%- if status.last_completed_file %} — {{ status.last_completed_file|truncate(35, True, '...') }}{% endif %}
- {%- if status.bytes_display %}{{ status.bytes_display }} · {% endif %}
+ {%- if status.has_byte_progress %}{{ status.completed_files }}/{{ status.total_files }} file{{ 's' if status.total_files != 1 }} · {% endif %}
{{- status.elapsed_display }}
{%- if status.eta_display %} | ~{{ status.eta_display }} left{% endif %}
@@ -860,7 +866,14 @@
{%- for f in status.recent_files %}
+ {#- A sidecars-only header names the video so its extras can be
+ attributed, but the video did not move. Its own action badge
+ would say otherwise, so it gets a neutral one instead. -#}
+ {% if f.sidecars_only is defined and f.sidecars_only %}
+ EXTRAS
+ {% else %}
{{ f.action|upper }}
+ {% endif %}
{{ f.filename|truncate_filename(55) }}
{% if f.size and f.size != '-' %}{{ f.size }}{% endif %}
{% if f.associated_files is defined and f.associated_files %}+{{ f.associated_files|length }}{% endif %}
diff --git a/web/templates/components/recent_activity.html b/web/templates/components/recent_activity.html
index 12dd56c..9dda660 100644
--- a/web/templates/components/recent_activity.html
+++ b/web/templates/components/recent_activity.html
@@ -124,7 +124,7 @@
data-action="{{ entry.action }}"
onclick="ActivityRuns.toggleShow(this)">
{{ entry.time_display }} |
- {{ action_badge(entry.action) }} |
+ {% if entry.sidecars_only %}Extras only{% else %}{{ action_badge(entry.action) }}{% endif %} |
›{{ entry.show_name }}
{{ entry.episode_count }} eps
@@ -157,7 +157,7 @@
data-action="{{ entry.action }}"
{% if entry.associated_files %}{{ af_expand_onclick() }}{% endif %}>
| {{ entry.time_display }} |
- {{ action_badge(entry.action) }} |
+ {% if entry.sidecars_only %}Extras only{% else %}{{ action_badge(entry.action) }}{% endif %} |
{{ entry.filename }} |
{% if entry.users and entry.users|length > 0 %}
|