+ {#- A sidecars-only header names the video so its extras can be
+ attributed, but the video did not move. Its own action badge
+ would say otherwise, so it gets a neutral one instead. -#}
+ {% if f.sidecars_only is defined and f.sidecars_only %}
+
EXTRAS
+ {% else %}
{{ f.action|upper }}
+ {% endif %}
{{ f.filename|truncate_filename(55) }}
{% if f.size and f.size != '-' %}
{{ f.size }}{% endif %}
{% if f.associated_files is defined and f.associated_files %}
+{{ f.associated_files|length }}{% endif %}
diff --git a/web/templates/components/recent_activity.html b/web/templates/components/recent_activity.html
index 12dd56c..9dda660 100644
--- a/web/templates/components/recent_activity.html
+++ b/web/templates/components/recent_activity.html
@@ -124,7 +124,7 @@
data-action="{{ entry.action }}"
onclick="ActivityRuns.toggleShow(this)">
{{ entry.time_display }} |
-
{{ action_badge(entry.action) }} |
+
{% if entry.sidecars_only %}Extras only{% else %}{{ action_badge(entry.action) }}{% endif %} |
›{{ entry.show_name }}
{{ entry.episode_count }} eps
@@ -157,7 +157,7 @@
data-action="{{ entry.action }}"
{% if entry.associated_files %}{{ af_expand_onclick() }}{% endif %}>
| {{ entry.time_display }} |
-
{{ action_badge(entry.action) }} |
+
{% if entry.sidecars_only %}Extras only{% else %}{{ action_badge(entry.action) }}{% endif %} |
{{ entry.filename }} |
{% if entry.users and entry.users|length > 0 %}
From 5a56553aa1c575ca1409e1940c76febb4aea185c Mon Sep 17 00:00:00 2001
From: Brandon Haney <121782102+Brandon-Haney@users.noreply.github.com>
Date: Wed, 12 Aug 2026 09:49:02 -0500
Subject: [PATCH 3/3] Price files as well as bytes in the remaining-time
estimate
Copy time is bytes/rate plus a fixed cost per file: open, size verify, chown,
tracker write. The estimate modelled only the first, so a queue that ends in
artwork and NFOs read as nearly free. On a 199-file run it reported 9m 38s
left when 11m 27s remained, because the 43 completed files averaged ~4 GB and
the 156 still queued averaged ~0.75 GB.
_estimate_remaining_seconds() splits the two. The best sustained rate seen is
bulk throughput with per-file costs largely excluded; whatever elapsed time
that cannot explain is attributed to the files completed so far. Both are
measured from the run in progress, so a slow array or a competing mover is
reflected without a tuning constant. Sampled over 2s windows, since shorter
ones are scheduling noise that would inflate the peak and hide the per-file
term.
The banner also led with the file count while its bar and ETA were byte-based.
At the moment above that read "43/199 files" beside a correct "9m left",
which is a fifth against three fifths and makes the estimate look impossible.
Bytes lead when byte progress is available; the file count moves alongside
elapsed.
---
tests/test_operation_eta.py | 138 ++++++++++++++++++
web/services/operation_runner.py | 86 ++++++++++-
.../components/global_operation_banner.html | 10 +-
3 files changed, 226 insertions(+), 8 deletions(-)
create mode 100644 tests/test_operation_eta.py
diff --git a/tests/test_operation_eta.py b/tests/test_operation_eta.py
new file mode 100644
index 0000000..fb365e7
--- /dev/null
+++ b/tests/test_operation_eta.py
@@ -0,0 +1,138 @@
+"""The remaining-time estimate prices files as well as bytes.
+
+Measured against a real run (2026-08-12, 199 files / 287.53 GB): at 14m 11s of
+copying, 171.16 GB was done across 43 files. The old pure-byte model reported
+9m 38s left; the run actually needed 11m 27s. It was optimistic because the 43
+completed files averaged ~4 GB while the 156 remaining averaged ~0.75 GB, many
+of them artwork and NFOs whose cost is per-file rather than per-byte.
+"""
+
+import sys
+from unittest.mock import MagicMock
+
+import pytest
+
+sys.modules.setdefault('fcntl', MagicMock())
+for _mod_name in ['plexapi', 'plexapi.server', 'plexapi.video', 'plexapi.myplex',
+ 'plexapi.exceptions', 'requests', 'apscheduler',
+ 'apscheduler.schedulers', 'apscheduler.schedulers.background',
+ 'apscheduler.triggers', 'apscheduler.triggers.cron',
+ 'apscheduler.triggers.interval']:
+ sys.modules.setdefault(_mod_name, MagicMock())
+
+GB = 1024 ** 3
+
+
+def _estimate(**kw):
+ from web.services.operation_runner import OperationRunner
+ return OperationRunner._estimate_remaining_seconds(**kw)
+
+
+class TestTheRealRun:
+ """Numbers taken from the 08:30 run on 2026-08-12."""
+
+ ELAPSED = 14 * 60 + 11 # copying began 08:50:09, sampled 09:04:20
+ DONE = int(171.16 * GB)
+ REMAINING = int((287.53 - 171.16) * GB)
+ FILES_DONE = 43
+ FILES_LEFT = 156
+ ACTUAL_LEFT = 11 * 60 + 27 # finished 09:15:47
+
+ def test_pure_byte_rate_is_the_optimistic_baseline(self):
+ """What the old model produced, kept as the thing to beat."""
+ rate = self.DONE / self.ELAPSED
+ naive = self.REMAINING / rate
+
+ assert 560 < naive < 600, naive # ~9m 38s
+ assert naive < self.ACTUAL_LEFT * 0.9 # meaningfully under
+
+ def test_per_file_term_closes_most_of_the_gap(self):
+ # A sustained rate slightly above the batch average, as a peak sample
+ # taken during large-file transfer would be.
+ peak = (self.DONE / self.ELAPSED) * 1.08
+
+ eta = _estimate(elapsed=self.ELAPSED, bytes_done=self.DONE,
+ bytes_remaining=self.REMAINING, files_done=self.FILES_DONE,
+ files_remaining=self.FILES_LEFT, peak_rate=peak)
+
+ naive = self.REMAINING / (self.DONE / self.ELAPSED)
+ assert eta > naive, "must be less optimistic than the pure byte model"
+ assert abs(eta - self.ACTUAL_LEFT) < abs(naive - self.ACTUAL_LEFT), (
+ f"estimate {eta:.0f}s should beat naive {naive:.0f}s "
+ f"against actual {self.ACTUAL_LEFT}s"
+ )
+
+
+class TestBehaviour:
+
+ def test_a_tail_of_tiny_files_is_not_free(self):
+ """500 sidecars totalling almost nothing still take time."""
+ eta = _estimate(elapsed=600, bytes_done=100 * GB, bytes_remaining=1024,
+ files_done=20, files_remaining=500, peak_rate=200 * 1024 ** 2)
+
+ assert eta > 30, f"a 500-file tail priced at {eta:.1f}s is not credible"
+
+ def test_matches_the_byte_model_when_no_per_file_cost_shows(self):
+ """A perfectly byte-bound run should not be inflated."""
+ rate = 100 * 1024 ** 2
+ eta = _estimate(elapsed=100, bytes_done=100 * rate,
+ bytes_remaining=50 * rate, files_done=1,
+ files_remaining=1, peak_rate=rate)
+
+ assert eta == pytest.approx(50, rel=0.02)
+
+ def test_peak_rate_is_neutral_when_the_tail_mirrors_the_head(self):
+ """Splitting one measured elapsed into two terms must not invent time.
+
+ With the remaining work shaped like the completed work, any split of
+ cost between bytes and files reproduces the same total. That the peak
+ rate changes nothing here is the model behaving.
+ """
+ kw = dict(elapsed=100, bytes_done=10 * GB, bytes_remaining=10 * GB,
+ files_done=5, files_remaining=5)
+
+ assert _estimate(peak_rate=0, **kw) == pytest.approx(
+ _estimate(peak_rate=(10 * GB / 100) * 2, **kw))
+
+ def test_peak_rate_is_what_prices_a_small_file_tail(self):
+ """Without it the per-file cost is invisible and the tail reads as free.
+
+ This is the case that motivated the change: bytes nearly exhausted,
+ hundreds of sidecars left. On the batch average alone the residual is
+ zero, so the estimate collapses to nothing.
+ """
+ kw = dict(elapsed=100, bytes_done=10 * GB, bytes_remaining=1024,
+ files_done=5, files_remaining=500)
+
+ assert _estimate(peak_rate=0, **kw) == pytest.approx(0, abs=1)
+ assert _estimate(peak_rate=(10 * GB / 100) * 2, **kw) > 60
+
+ @pytest.mark.parametrize("kw", [
+ {"elapsed": 0, "bytes_done": 1024},
+ {"elapsed": 10, "bytes_done": 0},
+ ])
+ def test_returns_none_with_nothing_to_measure(self, kw):
+ assert _estimate(bytes_remaining=1024, files_done=0, files_remaining=1,
+ peak_rate=0, **kw) is None
+
+ def test_zero_remaining_is_zero(self):
+ eta = _estimate(elapsed=100, bytes_done=10 * GB, bytes_remaining=0,
+ files_done=10, files_remaining=0, peak_rate=0)
+
+ assert eta == 0
+
+
+class TestDisplayDenominatorsAgree:
+
+ def test_banner_leads_with_bytes_when_available(self):
+ """The bar and ETA are byte-based; the headline number must match."""
+ import pathlib
+ html = (pathlib.Path(__file__).resolve().parent.parent / "web" / "templates" /
+ "components" / "global_operation_banner.html").read_text(encoding="utf-8")
+
+ # There are two progress-meta blocks; pick the operation one.
+ block = html[html.index("status.has_byte_progress"):]
+ block = block[:block.index("")]
+ assert block.index("bytes_display") < block.index("completed_files"), (
+ "file count still leads, which is what made a correct ETA look wrong"
+ )
diff --git a/web/services/operation_runner.py b/web/services/operation_runner.py
index ebdf936..24a22c3 100644
--- a/web/services/operation_runner.py
+++ b/web/services/operation_runner.py
@@ -91,6 +91,14 @@ class OperationResult:
cumulative_bytes_copied: int = 0
cumulative_bytes_total: int = 0
_prev_batch_cumulative: int = 0 # internal: snapshot at batch start
+ # Best sustained transfer rate seen this run, in bytes/sec. Copy time is
+ # bytes/rate PLUS a fixed cost per file (open, verify, chown, tracker
+ # write), so the batch average conflates the two and reads high while big
+ # files dominate. Tracking the peak separates them: the residual between
+ # elapsed and bytes/peak_rate is what the per-file cost has to explain.
+ peak_byte_rate: float = 0.0
+ _rate_last_time: Optional[float] = None
+ _rate_last_bytes: int = 0
class WebLogHandler(logging.Handler):
@@ -1038,15 +1046,29 @@ def _run_operation(self, dry_run: bool, verbose: bool = False):
def _bytes_cb(bytes_copied: int, bytes_total: int):
with self._lock:
r = self._current_result
+ now = time.time()
if bytes_copied == 0:
# New batch starting — snapshot cumulative progress
- r.batch_copy_start_time = time.time()
+ r.batch_copy_start_time = now
r._prev_batch_cumulative = r.cumulative_bytes_copied
r.cumulative_bytes_total = r._prev_batch_cumulative + bytes_total
+ r._rate_last_time = now
+ r._rate_last_bytes = 0
r.batch_bytes_copied = bytes_copied
r.batch_bytes_total = bytes_total
r.cumulative_bytes_copied = r._prev_batch_cumulative + bytes_copied
+ # Sample the sustained rate over >=2s windows. Shorter ones
+ # are dominated by scheduling noise and would inflate the
+ # peak, which then hides the per-file cost in the ETA.
+ if r._rate_last_time is not None:
+ dt = now - r._rate_last_time
+ db = bytes_copied - r._rate_last_bytes
+ if dt >= 2.0 and db > 0:
+ r.peak_byte_rate = max(r.peak_byte_rate, db / dt)
+ r._rate_last_time = now
+ r._rate_last_bytes = bytes_copied
+
# Create and run the app
app = PlexCacheApp(
config_file=config_file,
@@ -1327,6 +1349,50 @@ def _merge_run_files(self, sibling_to_parent: Dict[str, str], compatible_actions
break
self._current_run_files = rebuilt
+ @staticmethod
+ def _estimate_remaining_seconds(*, elapsed: float, bytes_done: int,
+ bytes_remaining: int, files_done: int,
+ files_remaining: int,
+ peak_rate: float) -> Optional[float]:
+ """Seconds left, priced per byte and per file.
+
+ A pure bytes/second model assumes every remaining byte costs the same
+ as the ones already copied. That holds while the queue is video files
+ and breaks at the tail, where artwork and NFOs are thousands of times
+ smaller but still cost a copy, a size verify, a chown and a tracker
+ write each. The estimate then reads far too low exactly when a user is
+ watching it, and the run appears stuck near the end.
+
+ Two terms instead:
+
+ remaining = bytes_remaining / rate + files_remaining * per_file
+
+ ``rate`` is the best sustained rate seen, which is the throughput of
+ bulk transfer with per-file costs mostly excluded. Whatever elapsed
+ time that does not explain is attributed to the files completed so
+ far, giving ``per_file``. Both are measured from this run, so a slow
+ array or a busy mover is reflected without any tuning constant.
+
+ Falls back to the plain byte rate when there is not enough to fit,
+ and returns None when there is nothing to estimate from.
+ """
+ if elapsed <= 0 or bytes_done <= 0:
+ return None
+
+ average_rate = bytes_done / elapsed
+ rate = peak_rate if peak_rate > average_rate else average_rate
+ if rate <= 0:
+ return None
+
+ # Time the bytes alone cannot account for is per-file cost.
+ per_file = 0.0
+ if files_done > 0:
+ unexplained = elapsed - (bytes_done / rate)
+ if unexplained > 0:
+ per_file = unexplained / files_done
+
+ return (bytes_remaining / rate) + (files_remaining * per_file)
+
@staticmethod
def _synthetic_sidecar_parent(key: tuple, entries: list) -> dict:
"""A header row for sidecars whose parent video was not part of this run.
@@ -1459,14 +1525,22 @@ def get_status_dict(self) -> dict:
status["progress_percent"] = min(int(cumul_copied / cumul_total * 100), 100)
status["bytes_display"] = f"{self._format_bytes(cumul_copied)} / {self._format_bytes(cumul_total)}"
- # ETA from current batch byte rate
+ status["has_byte_progress"] = True
+
+ # ETA from the batch, priced per byte AND per file.
if result.batch_bytes_copied > 0 and result.batch_copy_start_time:
copy_elapsed = time.time() - result.batch_copy_start_time
if copy_elapsed > 0:
- rate = result.batch_bytes_copied / copy_elapsed
- remaining = cumul_total - cumul_copied
- if rate > 0:
- status["eta_display"] = self._format_duration(remaining / rate)
+ eta = self._estimate_remaining_seconds(
+ elapsed=copy_elapsed,
+ bytes_done=result.batch_bytes_copied,
+ bytes_remaining=max(0, cumul_total - cumul_copied),
+ files_done=completed_files,
+ files_remaining=max(0, total_files - completed_files),
+ peak_rate=result.peak_byte_rate,
+ )
+ if eta is not None:
+ status["eta_display"] = self._format_duration(eta)
# Recent log messages (last 5) for hover mini-log
# Files completed so far in this run for detail panel
diff --git a/web/templates/components/global_operation_banner.html b/web/templates/components/global_operation_banner.html
index 52cf3c8..aa9aeff 100644
--- a/web/templates/components/global_operation_banner.html
+++ b/web/templates/components/global_operation_banner.html
@@ -829,11 +829,17 @@
{% if status.total_files is defined and status.total_files > 0 %}
+ {#- The bar and the ETA are both byte-based, so bytes lead. Showing the
+ file count first made them disagree: 43/199 files reads as a fifth
+ done while the same moment was 60% of the bytes, which made a
+ correct "9m left" look impossible. -#}
- {{ status.completed_files }}/{{ status.total_files }} file{{ 's' if status.total_files != 1 }}
+
+ {%- if status.has_byte_progress and status.bytes_display %}{{ status.bytes_display }}
+ {%- else %}{{ status.completed_files }}/{{ status.total_files }} file{{ 's' if status.total_files != 1 }}{% endif %}
{%- if status.last_completed_file %} — {{ status.last_completed_file|truncate(35, True, '...') }}{% endif %}
- {%- if status.bytes_display %}{{ status.bytes_display }} · {% endif %}
+ {%- if status.has_byte_progress %}{{ status.completed_files }}/{{ status.total_files }} file{{ 's' if status.total_files != 1 }} · {% endif %}
{{- status.elapsed_display }}
{%- if status.eta_display %} | ~{{ status.eta_display }} left{% endif %}
|