From 050bce809401d0986c154008546e7f82b979f02b Mon Sep 17 00:00:00 2001 From: Agi-Asi <206806952+Agi-Asi@users.noreply.github.com> Date: Thu, 27 Aug 2026 09:33:01 +0000 Subject: [PATCH 01/21] Truncate generated filenames to the filesystem byte limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Filesystems limit each path component to 255 bytes, not characters. Multi-byte titles (CJK, emoji) hit that with far fewer characters — a 98-character Japanese title is already 225+ bytes before TubeSync adds prefixes and yt-dlp adds '.fNNN.mp4.part(-FragNN)' suffixes — and the download then fails permanently with: ERROR: unable to open for writing: [Errno 36] File name too long (#522). The task retries and fails forever with no way out short of renaming the source format. Media.filename now clamps the final name component to a 200-byte budget (leaving headroom for the suffixes appended during download): - only the name component is shortened; directories from the format string are preserved untouched - bytes are removed from the middle of the stem, keeping its start (the beginning of the title) and its end (the unique {key}/{format} suffixes of the default media format), joined by '_..._' — so uniqueness of generated names survives truncation - truncation never splits a multi-byte character (partial sequences are dropped when decoding) - a warning is logged when a name is shortened Names within the limit are returned byte-identical, so existing installs see no change unless they were already broken. Fixes #522 --- tubesync/common/utils.py | 40 ++++++++++++++++++++++++++++ tubesync/sync/models/media.py | 18 ++++++++++++- tubesync/sync/tests/test_filepath.py | 28 +++++++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index b0236e7c7..2ae09153b 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -305,6 +305,46 @@ def clean_emoji(s): return emoji.replace_emoji(s) +def truncate_filename_bytes(filename, /, max_bytes=200, encoding='utf-8'): + ''' + Shortens a filename to fit within `max_bytes` bytes (not characters) + while keeping its extension intact. Filesystems limit name length in + bytes (commonly 255), so multi-byte titles can exceed the limit with + far fewer characters (see issue #522). The default budget leaves + headroom for prefixes/suffixes appended later (e.g. `.fNNN`, + `.part-FragNN` fragments written by yt-dlp during downloads). + + Bytes are removed from the middle of the stem, keeping its start + (usually the beginning of the title) and its end (usually unique + suffixes such as the media key and format details, e.g. + `{title_full}_{key}_{format}` from the default media format), joined + by `_..._`. Truncation never splits a multi-byte character: partial + trailing/leading sequences are dropped when decoding. + ''' + if not isinstance(filename, str): + raise ValueError(f'filename must be a str, got {type(filename)}') + if len(filename.encode(encoding)) <= max_bytes: + return filename + name, dot, ext = filename.rpartition('.') + if not dot: + name, ext_bytes = filename, b'' + else: + ext_bytes = (dot + ext).encode(encoding) + if len(ext_bytes) >= max_bytes: + # Pathological extension; fall back to a plain byte cut + name, ext_bytes = filename, b'' + marker = '_..._' + stem_budget = max_bytes - len(ext_bytes) - len(marker.encode(encoding)) + stem_bytes = name.encode(encoding) + # Keep the unique suffixes at the end of the stem intact (up to half of + # the budget), then fill the rest from the front. + tail_keep = min(stem_budget // 2, 64) + head_keep = stem_budget - tail_keep + head = stem_bytes[:head_keep].decode(encoding, errors='ignore').rstrip() + tail = stem_bytes[-tail_keep:].decode(encoding, errors='ignore').lstrip() + return head + marker + tail + ext_bytes.decode(encoding) + + def seconds_to_timestr(seconds): seconds = seconds % (24 * 3600) hour = seconds // 3600 diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 1c0c0acee..d6184b62b 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -19,6 +19,7 @@ from common.utils import ( clean_filename, clean_emoji, directory_and_stem, glob_quote, mkdir_p, seconds_to_timestr, + truncate_filename_bytes, ) from ..youtube import ( get_media_info as get_youtube_media_info, @@ -841,7 +842,22 @@ def filename(self): media_format = str(self.source.media_format) media_details = self.format_dict result = media_format.format(**media_details) - return '.' + result if '/' == result[0] else result + result = '.' + result if '/' == result[0] else result + # Filesystems limit each path component to 255 bytes (not + # characters), and multi-byte titles can blow past that with far + # fewer characters — downloads then fail with: + # [Errno 36] File name too long + # (issue #522). Only the final component (the name) is shortened; + # any directories in the format string are preserved. The budget + # leaves headroom for suffixes appended during download + # (`.fNNN.ext.part-FragNNN.part` and thumbnail/subtitle siblings). + head, _, tail = result.rpartition('/') + truncated = truncate_filename_bytes(tail) + if truncated != tail: + log.warning(f'Media filename exceeded the filesystem byte limit ' + f'and was shortened: {self!r}') + return f'{head}/{truncated}' if head else truncated + return result @property def directory_path(self): diff --git a/tubesync/sync/tests/test_filepath.py b/tubesync/sync/tests/test_filepath.py index a843e31fd..22fee8672 100644 --- a/tubesync/sync/tests/test_filepath.py +++ b/tubesync/sync/tests/test_filepath.py @@ -158,6 +158,34 @@ def test_media_filename(self): ('no-fancy-stuff-title_test_720p-720x1280-opus' '-vp9-30fps-hdr.mkv')) + def test_media_filename_truncates_to_filesystem_byte_limit(self): + # Filesystems limit each name component to 255 bytes (issue #522). + # Multi-byte titles reach that with far fewer characters, and the + # download then fails with '[Errno 36] File name too long'. + import json + long_metadata = json.loads(metadata) + long_metadata['title'] = '耳' * 120 # 3 bytes per char = 360 bytes + long_title_media = Media.objects.create( + key='longkey', + source=self.source, + metadata=json.dumps(long_metadata), + ) + self.source.media_format = '{yyyy}/{title_full}_{key}.{ext}' + filename = long_title_media.filename + directory, _, name = filename.rpartition('/') + # Directories from the format string survive untouched + self.assertEqual('2017', directory) + # The name component fits in the byte budget... + self.assertLessEqual(len(name.encode('utf-8')), 200) + # ... keeps its extension and key suffix material intact ... + self.assertTrue(name.endswith('_longkey.mkv')) + # ... and was not cut mid multi-byte character (encodes cleanly) + name.encode('utf-8').decode('utf-8') + + def test_media_filename_unchanged_when_within_limit(self): + self.source.media_format = '{yyyy}/{key}.{ext}' + self.assertEqual(self.media.filename, '2017/mediakey.mkv') + def test_directory_prefix(self): # Confirm the setting exists and is valid self.assertTrue(hasattr(settings, 'SOURCE_DOWNLOAD_DIRECTORY_PREFIX')) From b88d2456d4b00b17fcf07af14261edfc71f0f13e Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Aug 2026 06:20:09 -0400 Subject: [PATCH 02/21] chore: move the import --- tubesync/sync/tests/test_filepath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/sync/tests/test_filepath.py b/tubesync/sync/tests/test_filepath.py index 22fee8672..d126cc9cb 100644 --- a/tubesync/sync/tests/test_filepath.py +++ b/tubesync/sync/tests/test_filepath.py @@ -1,3 +1,4 @@ +import json import logging from pathlib import Path from django.conf import settings @@ -162,7 +163,6 @@ def test_media_filename_truncates_to_filesystem_byte_limit(self): # Filesystems limit each name component to 255 bytes (issue #522). # Multi-byte titles reach that with far fewer characters, and the # download then fails with '[Errno 36] File name too long'. - import json long_metadata = json.loads(metadata) long_metadata['title'] = '耳' * 120 # 3 bytes per char = 360 bytes long_title_media = Media.objects.create( From d9f28d4e31c55346ca35ab69eb55723daf647433 Mon Sep 17 00:00:00 2001 From: Agi-Asi <206806952+Agi-Asi@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:33:06 +0000 Subject: [PATCH 03/21] review: parse names with pathlib, raise TypeError, add encoding-edge tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses review feedback: - truncate_filename_bytes parses stem/suffix with PurePosixPath instead of str.rpartition; Media.filename splits directory/name with PurePosixPath.with_name instead of string slicing - non-str input now raises TypeError (was ValueError) to match the type-error semantics - new tests cover bytes known to cause encoding/decoding trouble: cuts landing inside 4-byte emoji, combining sequences, ZWJ families, mixed-width boundaries, RTL text, dotfiles, no-extension and oversized-extension names — asserting byte budget, UTF-8 round-trip validity, and non-empty results in every case --- tubesync/common/utils.py | 16 +++++----- tubesync/sync/models/media.py | 10 +++---- tubesync/sync/tests/test_filepath.py | 45 ++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 14 deletions(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index 2ae09153b..952555b24 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -12,7 +12,7 @@ from functools import partial from itertools import chain from operator import attrgetter, itemgetter -from pathlib import Path +from pathlib import Path, PurePosixPath from urllib.parse import urlunsplit, urlencode, urlparse from .errors import DatabaseConnectionError, QuerySetEmptyError @@ -322,17 +322,15 @@ def truncate_filename_bytes(filename, /, max_bytes=200, encoding='utf-8'): trailing/leading sequences are dropped when decoding. ''' if not isinstance(filename, str): - raise ValueError(f'filename must be a str, got {type(filename)}') + raise TypeError(f'filename must be a str, got {type(filename)}') if len(filename.encode(encoding)) <= max_bytes: return filename - name, dot, ext = filename.rpartition('.') - if not dot: + path = PurePosixPath(filename) + name, ext = path.stem, path.suffix + ext_bytes = ext.encode(encoding) + if len(ext_bytes) >= max_bytes: + # Pathological extension; fall back to a plain byte cut name, ext_bytes = filename, b'' - else: - ext_bytes = (dot + ext).encode(encoding) - if len(ext_bytes) >= max_bytes: - # Pathological extension; fall back to a plain byte cut - name, ext_bytes = filename, b'' marker = '_..._' stem_budget = max_bytes - len(ext_bytes) - len(marker.encode(encoding)) stem_bytes = name.encode(encoding) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index d6184b62b..c950110a8 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -4,7 +4,7 @@ from collections import OrderedDict from copy import deepcopy from datetime import datetime, timedelta, timezone as tz -from pathlib import Path +from pathlib import Path, PurePosixPath from xml.etree import ElementTree from django.conf import settings from django.db import models @@ -851,12 +851,12 @@ def filename(self): # any directories in the format string are preserved. The budget # leaves headroom for suffixes appended during download # (`.fNNN.ext.part-FragNNN.part` and thumbnail/subtitle siblings). - head, _, tail = result.rpartition('/') - truncated = truncate_filename_bytes(tail) - if truncated != tail: + path = PurePosixPath(result) + truncated = truncate_filename_bytes(path.name) + if truncated != path.name: log.warning(f'Media filename exceeded the filesystem byte limit ' f'and was shortened: {self!r}') - return f'{head}/{truncated}' if head else truncated + return str(path.with_name(truncated)) return result @property diff --git a/tubesync/sync/tests/test_filepath.py b/tubesync/sync/tests/test_filepath.py index d126cc9cb..ae92218e7 100644 --- a/tubesync/sync/tests/test_filepath.py +++ b/tubesync/sync/tests/test_filepath.py @@ -186,6 +186,51 @@ def test_media_filename_unchanged_when_within_limit(self): self.source.media_format = '{yyyy}/{key}.{ext}' self.assertEqual(self.media.filename, '2017/mediakey.mkv') + def test_truncate_filename_bytes_encoding_edge_cases(self): + # Bytes known to cause encoding/decoding trouble must never produce + # an invalid or over-budget name: the cut points land inside + # multi-byte sequences on purpose here. + from common.utils import truncate_filename_bytes + + cases = [ + # 4-byte astral plane (emoji): cut lands mid-sequence + '🍣' * 100 + '_key.mkv', + # combining characters (é as e + U+0301) + ('e\u0301' * 150) + '_key.mkv', + # zero-width joiner sequences (family emoji) + ('👨\u200d👩\u200d👧\u200d👦' * 30) + '_key.mkv', + # mixed 1-byte/3-byte at every boundary parity + ('a耳' * 120) + '_key.mkv', + # right-to-left text + ('שלום' * 60) + '_key.mkv', + # no extension at all + '⽕' * 200, + # dotfile-style name (suffix is empty for PurePosixPath) + '.' + ('h' * 300), + # very long "extension" exceeding the whole budget + 'name.' + ('x' * 300), + ] + for original in cases: + with self.subTest(original=original[:24]): + result = truncate_filename_bytes(original) + # fits the byte budget + self.assertLessEqual(len(result.encode('utf-8')), 200) + # still valid UTF-8 round-trip (no partial sequences kept) + self.assertEqual( + result, + result.encode('utf-8').decode('utf-8'), + ) + # never empty + self.assertTrue(result) + + def test_truncate_filename_bytes_rejects_non_str(self): + from common.utils import truncate_filename_bytes + + for bad in (None, 42, b'bytes.mkv', Path('p.mkv')): + with self.subTest(bad=bad): + with self.assertRaises(TypeError): + truncate_filename_bytes(bad) + def test_directory_prefix(self): # Confirm the setting exists and is valid self.assertTrue(hasattr(settings, 'SOURCE_DOWNLOAD_DIRECTORY_PREFIX')) From af0230f088e554d1bd81a892bf0b07afc9823c90 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Aug 2026 09:24:49 -0400 Subject: [PATCH 04/21] chore: move some imports --- tubesync/sync/models/media.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index c950110a8..87ee1cd66 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -647,7 +647,6 @@ def save_to_metadata(self, key, value, /): migrated['_using_table'] = True self.metadata = self.metadata_dumps(arg_dict=migrated) self.save() - from common.logger import log log.debug(f'Saved to metadata: {self.key} / {self.uuid}: {key=}: {value}') @@ -679,10 +678,8 @@ def reduce_data(self): filtered_data['_reduce_data_ran_at'] = round((now - self.posix_epoch).total_seconds()) filtered_json = self.metadata_dumps(arg_dict=filtered_data) except Exception as e: - from common.logger import log log.exception('reduce_data: %s', e) else: - from common.logger import log # log the results of filtering / compacting on metadata size new_mdl = len(compact_json) if old_mdl > new_mdl: @@ -1038,10 +1035,11 @@ def get_download_state(self, task=None): if self.downloaded: return Val(MediaState.DOWNLOADED) if task: + # Avoid the circular import `ImportError` from using this at the top of the file. + from ..tasks import get_media_download_task def running(arg_task, /): if hasattr(arg_task, 'locked_by_pid_running'): return arg_task.locked_by_pid_running() - from ..tasks import get_media_download_task return get_media_download_task(str(self.pk)) if running(task): return Val(MediaState.DOWNLOADING) From f8d8da4df352df173edef90e8d1ddedfa7621dcf Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Aug 2026 09:25:32 -0400 Subject: [PATCH 05/21] fix(lint): address RUF012 --- tubesync/sync/models/media.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 87ee1cd66..07e0c838c 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -5,6 +5,7 @@ from copy import deepcopy from datetime import datetime, timedelta, timezone as tz from pathlib import Path, PurePosixPath +from typing import ClassVar from xml.etree import ElementTree from django.conf import settings from django.db import models @@ -57,14 +58,14 @@ class Media(models.Model): posix_epoch = datetime(1970, 1, 1, tzinfo=tz.utc) # Format to use to display a URL for the media - URLS = _srctype_dict('https://www.youtube.com/watch?v={key}') + URLS: ClassVar[dict[str, str]] = _srctype_dict('https://www.youtube.com/watch?v={key}') # Callback functions to get a list of media from the source - INDEXERS = _srctype_dict(get_youtube_media_info) + INDEXERS: ClassVar[dict[str, type(get_youtube_media_info)]] = _srctype_dict(get_youtube_media_info) # Maps standardised names to names used in source metdata _same_name = lambda n, k=None: {k or n: _srctype_dict(n) } - METADATA_FIELDS = { + METADATA_FIELDS: ClassVar[dict[str, dict[str, str]]] = { **(_same_name('upload_date')), **(_same_name('timestamp')), **(_same_name('title')), @@ -81,7 +82,7 @@ class Media(models.Model): **(_same_name('playlist_title')), } - STATE_ICONS = dict(zip( + STATE_ICONS: ClassVar[dict[str, str]] = dict(zip( MediaState.values, ( '', From ab269c3702e7b0951a65ba093ed7d049ebdf4c2d Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Aug 2026 09:25:35 -0400 Subject: [PATCH 06/21] fix(lint): address FURB188 --- tubesync/common/utils.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index 952555b24..84721af56 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -209,7 +209,7 @@ def parse_database_connection_string(database_connection_string): f'as a database connection string: {e}') from e driver = parts.scheme user_pass_host_port = parts.netloc - database = parts.path + database = parts.path.removeprefix('/') if driver not in valid_drivers: raise DatabaseConnectionError(f'Database connection string ' f'"{database_connection_string}" specified an ' @@ -244,8 +244,6 @@ def parse_database_connection_string(database_connection_string): # Malformed raise DatabaseConnectionError('Database connection host must be a hostname or ' 'a hostname:port combination') - if database.startswith('/'): - database = database[1:] if not database: raise DatabaseConnectionError('Database connection string path must be a ' 'string in the format of /databasename') From 137e65ae01078d0782249dd63f24dadcba2c37f6 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Aug 2026 09:25:38 -0400 Subject: [PATCH 07/21] fix(lint): address UP032 --- tubesync/common/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index 84721af56..a8eaf30da 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -347,7 +347,7 @@ def seconds_to_timestr(seconds): seconds %= 3600 minutes = seconds // 60 seconds %= 60 - return '{:02d}:{:02d}:{:02d}'.format(hour, minutes, seconds) + return f'{hour:02d}:{minutes:02d}:{seconds:02d}' def time_func(func): From 45237a2cd4cccf0d4b63460ed4ee818feed32e4e Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Aug 2026 09:25:41 -0400 Subject: [PATCH 08/21] chore(lint): ignore RUF059 --- tubesync/common/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index a8eaf30da..27e330453 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -220,6 +220,8 @@ def parse_database_connection_string(database_connection_string): if len(host_parts) != 2 or len(user_pass_parts) != 2: raise DatabaseConnectionError('Database connection string netloc must be in ' 'the format of user:pass@host') + # user_pass never used + # ruff: ignore[RUF059] user_pass, host_port = host_parts username, password = user_pass_parts host_port_parts = host_port.split(':') From 2be3b9f23e05ec73b6c7fb39516f4b3b9b14ed4d Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Aug 2026 09:25:45 -0400 Subject: [PATCH 09/21] fix(lint): address TRY004 --- tubesync/common/utils.py | 10 +++++----- tubesync/sync/tests/test_filepath.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index 27e330453..e6cf66922 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -284,9 +284,9 @@ def append_uri_params(uri, params): return urlunsplit(('', '', uri, qs, '')) -def clean_filename(filename): +def clean_filename(filename: str) -> str: if not isinstance(filename, str): - raise ValueError(f'filename must be a str, got {type(filename)}') + raise TypeError(f'filename must be a str, got {type(filename)}') to_scrub = r'<>\/:*?"|%' for char in list(to_scrub): filename = filename.replace(char, '') @@ -299,13 +299,13 @@ def clean_filename(filename): return clean_filename.strip() -def clean_emoji(s): +def clean_emoji(s: str) -> str: if not isinstance(s, str): - raise ValueError(f'parameter must be a str, got {type(s)}') + raise TypeError(f'parameter must be a str, got {type(s)}') return emoji.replace_emoji(s) -def truncate_filename_bytes(filename, /, max_bytes=200, encoding='utf-8'): +def truncate_filename_bytes(filename: str, *, max_bytes=216, encoding='utf-8') -> str: ''' Shortens a filename to fit within `max_bytes` bytes (not characters) while keeping its extension intact. Filesystems limit name length in diff --git a/tubesync/sync/tests/test_filepath.py b/tubesync/sync/tests/test_filepath.py index ae92218e7..510490a31 100644 --- a/tubesync/sync/tests/test_filepath.py +++ b/tubesync/sync/tests/test_filepath.py @@ -176,7 +176,7 @@ def test_media_filename_truncates_to_filesystem_byte_limit(self): # Directories from the format string survive untouched self.assertEqual('2017', directory) # The name component fits in the byte budget... - self.assertLessEqual(len(name.encode('utf-8')), 200) + self.assertLessEqual(len(name.encode('utf-8')), 216) # ... keeps its extension and key suffix material intact ... self.assertTrue(name.endswith('_longkey.mkv')) # ... and was not cut mid multi-byte character (encodes cleanly) From ee5416309b9aa4e8f31eb65ffb0cc81c98f5d322 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Aug 2026 09:30:47 -0400 Subject: [PATCH 10/21] fix(test): specify max_bytes for truncate_filename_bytes --- tubesync/sync/tests/test_filepath.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/tests/test_filepath.py b/tubesync/sync/tests/test_filepath.py index 510490a31..1394d96b4 100644 --- a/tubesync/sync/tests/test_filepath.py +++ b/tubesync/sync/tests/test_filepath.py @@ -212,9 +212,9 @@ def test_truncate_filename_bytes_encoding_edge_cases(self): ] for original in cases: with self.subTest(original=original[:24]): - result = truncate_filename_bytes(original) + result = truncate_filename_bytes(original, max_bytes=208) # fits the byte budget - self.assertLessEqual(len(result.encode('utf-8')), 200) + self.assertLessEqual(len(result.encode('utf-8')), 208) # still valid UTF-8 round-trip (no partial sequences kept) self.assertEqual( result, From e520f9c25489ec03962b14e66e3279d16dbc086f Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Aug 2026 10:39:35 -0400 Subject: [PATCH 11/21] fix(lint): address B006 --- tubesync/sync/models/media.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 07e0c838c..08fdc4a86 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -598,14 +598,17 @@ def metadata_clear(self, /, *, save=False): self.save() - def metadata_dumps(self, arg_dict=dict()): + def metadata_dumps(self, arg_dict=None): fallback = dict() try: fallback.update(self.new_metadata.with_formats) except ObjectDoesNotExist: pass - data = arg_dict or fallback - return json.dumps(data, separators=(',', ':'), cls=JSONEncoder) + return json.dumps( + arg_dict or fallback, + separators=(',', ':'), + cls=JSONEncoder, + ) def metadata_loads(self, arg_str='{}'): From 964025fa85ff414aeb076f4e5d7ea43e931fe805 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Aug 2026 10:39:37 -0400 Subject: [PATCH 12/21] fix(lint): address PIE790 --- tubesync/sync/models/media.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 08fdc4a86..6512437ac 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -744,7 +744,6 @@ def ts_to_dt(self, /, timestamp): timestamp_float = float(timestamp) except (TypeError, ValueError,) as e: log.warn(f'Could not compute published from timestamp for: {self.source} / {self} with "{e}"') - pass else: return self.posix_epoch + timedelta(seconds=timestamp_float) return None @@ -784,7 +783,6 @@ def upload_date(self): return datetime.strptime(upload_date_str, '%Y%m%d') except (AttributeError, ValueError) as e: log.debug(f'Media.upload_date: {self.source} / {self}: strptime: {e}') - pass return None @property From 7e62a082f9d2d96f0e543d69eb8a00a823f849b2 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Aug 2026 10:39:40 -0400 Subject: [PATCH 13/21] fix(lint): address DTZ007 --- tubesync/sync/models/media.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 6512437ac..611a491a5 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -776,11 +776,16 @@ def name(self): @property def upload_date(self): + ts = self.get_metadata_first_value('timestamp') + dt = self.ts_to_dt(ts) if ts else None + if dt and dt > self.posix_epoch: + return dt + upload_date_str = self.get_metadata_first_value('upload_date') if not upload_date_str: return None try: - return datetime.strptime(upload_date_str, '%Y%m%d') + return datetime.strptime(upload_date_str, '%Y%m%d').replace(tzinfo=tz.utc) except (AttributeError, ValueError) as e: log.debug(f'Media.upload_date: {self.source} / {self}: strptime: {e}') return None From e76b05ebd3527061f81485dec599517bba48a6ca Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Aug 2026 10:53:21 -0400 Subject: [PATCH 14/21] chore(lint): ignore TRY002 & TRY004 --- tubesync/sync/models/media.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 611a491a5..55be381cb 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -1084,6 +1084,7 @@ def index_metadata(self): ''' indexer = self.INDEXERS.get(self.source.source_type, None) if not callable(indexer): + # ruff: ignore[TRY002,TRY004] raise Exception(f'Media with source type f"{self.source.source_type}" ' f'has no indexer') response = indexer(self.url) From c77a82feb6b4bcd2b5c6edeceb271e45d7b5e2ef Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Aug 2026 13:25:31 -0400 Subject: [PATCH 15/21] chore(lint): ignore BLE001 --- tubesync/sync/models/media.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 55be381cb..db077660b 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -681,8 +681,9 @@ def reduce_data(self): filtered_data = filter_response(data, True) filtered_data['_reduce_data_ran_at'] = round((now - self.posix_epoch).total_seconds()) filtered_json = self.metadata_dumps(arg_dict=filtered_data) - except Exception as e: - log.exception('reduce_data: %s', e) + # ruff: ignore[BLE001] + except Exception: + log.exception(f'Media.reduce_data: {self.pk}') else: # log the results of filtering / compacting on metadata size new_mdl = len(compact_json) @@ -722,6 +723,7 @@ def loaded_metadata(self): pass setattr(self, '_cached_metadata_dict', data) return data + # ruff: ignore[BLE001] except Exception: return {} From 661f928e3c5dc10f998c1fd44d93ce0a6a34ca8a Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Aug 2026 13:25:33 -0400 Subject: [PATCH 16/21] chore(lint): ignore B010,RUF059,SIM118 --- tubesync/sync/models/media.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index db077660b..7c0b67ccc 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -47,6 +47,8 @@ ) from .source import Source +# ruff: file-ignore[B010,RUF059,SIM118] + class Media(models.Model): ''' From a7e2e192e998fe892d12b17d9d253df76ae330a5 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Aug 2026 13:45:35 -0400 Subject: [PATCH 17/21] chore: rename truncate_filename_bytes to truncate_filename --- tubesync/common/utils.py | 2 +- tubesync/sync/models/media.py | 4 ++-- tubesync/sync/tests/test_filepath.py | 7 +++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index e6cf66922..e5ca7815a 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -305,7 +305,7 @@ def clean_emoji(s: str) -> str: return emoji.replace_emoji(s) -def truncate_filename_bytes(filename: str, *, max_bytes=216, encoding='utf-8') -> str: +def truncate_filename(filename: str, *, max_bytes=216, encoding='utf-8') -> str: ''' Shortens a filename to fit within `max_bytes` bytes (not characters) while keeping its extension intact. Filesystems limit name length in diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 7c0b67ccc..40fb32d17 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -20,7 +20,7 @@ from common.utils import ( clean_filename, clean_emoji, directory_and_stem, glob_quote, mkdir_p, seconds_to_timestr, - truncate_filename_bytes, + truncate_filename, ) from ..youtube import ( get_media_info as get_youtube_media_info, @@ -860,7 +860,7 @@ def filename(self): # leaves headroom for suffixes appended during download # (`.fNNN.ext.part-FragNNN.part` and thumbnail/subtitle siblings). path = PurePosixPath(result) - truncated = truncate_filename_bytes(path.name) + truncated = truncate_filename(path.name) if truncated != path.name: log.warning(f'Media filename exceeded the filesystem byte limit ' f'and was shortened: {self!r}') diff --git a/tubesync/sync/tests/test_filepath.py b/tubesync/sync/tests/test_filepath.py index 1394d96b4..feebb8dd8 100644 --- a/tubesync/sync/tests/test_filepath.py +++ b/tubesync/sync/tests/test_filepath.py @@ -4,6 +4,7 @@ from django.conf import settings from django.test import TestCase from django.utils import timezone +from common.utils import truncate_filename from sync.models import Source, Media from sync.choices import ( Val, Fallback, SourceResolution, @@ -190,7 +191,6 @@ def test_truncate_filename_bytes_encoding_edge_cases(self): # Bytes known to cause encoding/decoding trouble must never produce # an invalid or over-budget name: the cut points land inside # multi-byte sequences on purpose here. - from common.utils import truncate_filename_bytes cases = [ # 4-byte astral plane (emoji): cut lands mid-sequence @@ -212,7 +212,7 @@ def test_truncate_filename_bytes_encoding_edge_cases(self): ] for original in cases: with self.subTest(original=original[:24]): - result = truncate_filename_bytes(original, max_bytes=208) + result = truncate_filename(original, max_bytes=208) # fits the byte budget self.assertLessEqual(len(result.encode('utf-8')), 208) # still valid UTF-8 round-trip (no partial sequences kept) @@ -224,12 +224,11 @@ def test_truncate_filename_bytes_encoding_edge_cases(self): self.assertTrue(result) def test_truncate_filename_bytes_rejects_non_str(self): - from common.utils import truncate_filename_bytes for bad in (None, 42, b'bytes.mkv', Path('p.mkv')): with self.subTest(bad=bad): with self.assertRaises(TypeError): - truncate_filename_bytes(bad) + truncate_filename(bad) def test_directory_prefix(self): # Confirm the setting exists and is valid From e13df5412703ae7352e835d06196a5b3ba9fe520 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Aug 2026 14:52:22 -0400 Subject: [PATCH 18/21] fix: clamp max_bytes to realistic values --- tubesync/common/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index e5ca7815a..839be0753 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -321,6 +321,7 @@ def truncate_filename(filename: str, *, max_bytes=216, encoding='utf-8') -> str: by `_..._`. Truncation never splits a multi-byte character: partial trailing/leading sequences are dropped when decoding. ''' + max_bytes = max(96, min(232, max_bytes)) if not isinstance(filename, str): raise TypeError(f'filename must be a str, got {type(filename)}') if len(filename.encode(encoding)) <= max_bytes: @@ -334,9 +335,9 @@ def truncate_filename(filename: str, *, max_bytes=216, encoding='utf-8') -> str: marker = '_..._' stem_budget = max_bytes - len(ext_bytes) - len(marker.encode(encoding)) stem_bytes = name.encode(encoding) - # Keep the unique suffixes at the end of the stem intact (up to half of - # the budget), then fill the rest from the front. - tail_keep = min(stem_budget // 2, 64) + # Keep the unique suffixes at the end of the stem intact (up to a third of + # the bytes limit), then fill the rest from the front. + tail_keep = min(stem_budget // 2, max_bytes // 3) head_keep = stem_budget - tail_keep head = stem_bytes[:head_keep].decode(encoding, errors='ignore').rstrip() tail = stem_bytes[-tail_keep:].decode(encoding, errors='ignore').lstrip() From 1ad1a94fc5275de778291d44e5ca46ad8622199c Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Aug 2026 15:06:24 -0400 Subject: [PATCH 19/21] fix: clean before trucate --- tubesync/common/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tubesync/common/utils.py b/tubesync/common/utils.py index 839be0753..141b65a93 100644 --- a/tubesync/common/utils.py +++ b/tubesync/common/utils.py @@ -12,7 +12,7 @@ from functools import partial from itertools import chain from operator import attrgetter, itemgetter -from pathlib import Path, PurePosixPath +from pathlib import Path from urllib.parse import urlunsplit, urlencode, urlparse from .errors import DatabaseConnectionError, QuerySetEmptyError @@ -326,8 +326,8 @@ def truncate_filename(filename: str, *, max_bytes=216, encoding='utf-8') -> str: raise TypeError(f'filename must be a str, got {type(filename)}') if len(filename.encode(encoding)) <= max_bytes: return filename - path = PurePosixPath(filename) - name, ext = path.stem, path.suffix + path = Path(filename) + name, ext = clean_filename(path.stem), clean_filename(path.suffix) ext_bytes = ext.encode(encoding) if len(ext_bytes) >= max_bytes: # Pathological extension; fall back to a plain byte cut From 1d588b617fe8a943973e8e42067c81122fc6cf02 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Aug 2026 15:15:51 -0400 Subject: [PATCH 20/21] chore: use Path instead of PurePosixPath --- tubesync/sync/models/media.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 40fb32d17..26f78dce2 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -4,7 +4,7 @@ from collections import OrderedDict from copy import deepcopy from datetime import datetime, timedelta, timezone as tz -from pathlib import Path, PurePosixPath +from pathlib import Path from typing import ClassVar from xml.etree import ElementTree from django.conf import settings @@ -859,7 +859,7 @@ def filename(self): # any directories in the format string are preserved. The budget # leaves headroom for suffixes appended during download # (`.fNNN.ext.part-FragNNN.part` and thumbnail/subtitle siblings). - path = PurePosixPath(result) + path = Path(result) truncated = truncate_filename(path.name) if truncated != path.name: log.warning(f'Media filename exceeded the filesystem byte limit ' From b703703c50c716f1197239b015e56d4363503185 Mon Sep 17 00:00:00 2001 From: tcely Date: Thu, 27 Aug 2026 15:29:39 -0400 Subject: [PATCH 21/21] chore(lint): ignore SIM117 --- tubesync/sync/tests/test_filepath.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tubesync/sync/tests/test_filepath.py b/tubesync/sync/tests/test_filepath.py index feebb8dd8..ed841b9b7 100644 --- a/tubesync/sync/tests/test_filepath.py +++ b/tubesync/sync/tests/test_filepath.py @@ -226,6 +226,7 @@ def test_truncate_filename_bytes_encoding_edge_cases(self): def test_truncate_filename_bytes_rejects_non_str(self): for bad in (None, 42, b'bytes.mkv', Path('p.mkv')): + # ruff: ignore[SIM117] with self.subTest(bad=bad): with self.assertRaises(TypeError): truncate_filename(bad)