From 09c0e4c70c5d0e89b2f8dbaf52b108da7b46dcc0 Mon Sep 17 00:00:00 2001 From: Sazpaimon Date: Wed, 14 Aug 2024 21:47:35 -0400 Subject: [PATCH 1/2] Common: Remove Pathlab iso accessor It's abandoned and doesn't work on python > 3.8 --- Pipfile | 3 +- common/factory.py | 12 ++--- common/iso_path_reader/methods/pathlab.py | 52 ------------------ common/processor.py | 3 +- iso_accessor.py | 65 ----------------------- 5 files changed, 6 insertions(+), 129 deletions(-) delete mode 100644 common/iso_path_reader/methods/pathlab.py delete mode 100644 iso_accessor.py diff --git a/Pipfile b/Pipfile index 7573275..24d8989 100755 --- a/Pipfile +++ b/Pipfile @@ -7,7 +7,6 @@ verify_ssl = true [packages] pycdlib = "*" -pathlab = "*" xxhash = "*" numpy = "*" libarchive-c = "*" @@ -24,4 +23,4 @@ inflate64 = "*" bitarray = "*" [requires] -python_version = "3.8" +python_version = ">=3.8" diff --git a/common/factory.py b/common/factory.py index c3b7b48..d42fb98 100644 --- a/common/factory.py +++ b/common/factory.py @@ -11,14 +11,12 @@ from cdi.path_reader import CdiPathReader from common.iso_path_reader.methods.compressed import CompressedPathReader from common.iso_path_reader.methods.hfs import HfsPathReader -from common.iso_path_reader.methods.pathlab import PathlabPathReader from common.iso_path_reader.methods.pycdlib import PyCdLibPathReader from common.processor import GenericIsoProcessor from common.udf.pycdlib_udf import PyCdlibUdf from dreamcast.processor import DreamcastIsoProcessor from gamecube.path_reader import GamecubePathReader from gamecube.processor import GamecubeIsoProcessor -from iso_accessor import IsoAccessor from machfs import Volume, Folder, File from cdi.processor import CdiIsoProcessor @@ -296,6 +294,8 @@ def get_iso_path_readers(fp, file_name, parent_container, pbar): try: iso.open_fp(wrapper) except Exception as e: + exceptions["iso9660"] = e + # pycdlib may fail on reading the directory contents of an iso, but it should still correctly parse the PVD if not hasattr(iso, "pvd") and hasattr(iso, "pvds") and iso.pvds: iso.pvd = iso.pvds[0] @@ -326,12 +326,8 @@ def get_iso_path_readers(fp, file_name, parent_container, pbar): iso.open_fp(wrapper) path_readers.append(path_reader_class(iso, wrapper, parent_container, volume_type="udf")) - if not iso._initialized and not iso._has_udf: - try: - iso_accessor = IsoAccessor(wrapper, ignore_susp=True) - path_readers.append(PathlabPathReader(iso_accessor, wrapper, parent_container, pvd=iso.pvd)) - except Exception as e: - exceptions[PathlabPathReader.volume_type] = e + if (iso._initialized or iso._has_udf) and "iso9660" in exceptions: + exceptions.pop("iso9660") return path_readers, exceptions diff --git a/common/iso_path_reader/methods/pathlab.py b/common/iso_path_reader/methods/pathlab.py deleted file mode 100644 index 9645f34..0000000 --- a/common/iso_path_reader/methods/pathlab.py +++ /dev/null @@ -1,52 +0,0 @@ -from common.iso_path_reader.methods.base import IsoPathReader -from common.iso_path_reader.methods.chunked_hash_trait import ChunkedHashTrait - - -class PathlabPathReader(ChunkedHashTrait, IsoPathReader): - volume_type = "iso9660_fallback" - - def __init__(self, *args, pvd, **kwargs): - self.pvd = pvd - super().__init__(*args, **kwargs) - - def get_root_dir(self): - return self.iso.IsoPath("/") - - def iso_iterator(self, base_dir, recursive=False, include_dirs=False): - if recursive: - method = base_dir.rglob - else: - method = base_dir.glob - - for file in method("*"): - if file.is_dir() and not include_dirs: - continue - - yield file - - def get_file_path(self, file): - return file.path - - def get_file_date(self, file): - return file.stat().create_time - - def get_file(self, path): - if path[0] != "/": - path = "/" + path - - return self.iso.IsoPath(path.replace(";1", "")) - - def open_file(self, file): - return file.open(mode='rb') - - def get_file_size(self, file): - return file.stat().size - - def get_file_sector(self, file): - return self.iso._load_record(file)['sector'] - - def is_directory(self, file): - return file.is_dir() - - def get_pvd(self): - return self.pvd diff --git a/common/processor.py b/common/processor.py index 7dd2ac8..0c2b64c 100644 --- a/common/processor.py +++ b/common/processor.py @@ -8,7 +8,6 @@ from cdi.path_reader import CdiPathReader from common.iso_path_reader.methods.compressed import CompressedPathReader -from common.iso_path_reader.methods.pathlab import PathlabPathReader from common.iso_path_reader.methods.pycdlib import PyCdLibPathReader from utils.common import format_bar_desc from utils.hash_progress_wrapper import HashProgressWrapper @@ -76,7 +75,7 @@ def get_system_type(iso_path_reader): if fp.read(5) == b'CD-I ': return "cdi" - if isinstance(iso_path_reader, (XboxPathReader, CompressedPathReader, PyCdLibPathReader, PathlabPathReader)): + if isinstance(iso_path_reader, (XboxPathReader, CompressedPathReader, PyCdLibPathReader)): for sys_type, exe_type, expected_header in (("xbox360", ".xex", b"XEX"), ("xbox", ".xbe", b"XBE")): for file in iso_path_reader.iso_iterator(iso_path_reader.get_root_dir(), recursive=False): file_path = iso_path_reader.get_file_path(file) diff --git a/iso_accessor.py b/iso_accessor.py deleted file mode 100644 index 8044eb1..0000000 --- a/iso_accessor.py +++ /dev/null @@ -1,65 +0,0 @@ -import datetime - -from pathlab import IsoAccessor as _IsoAccessor -from pathlab.iso import SECTOR - -class _ScandirIter: - """ - For compatibility with different python versions. - Pathlib: - - prior 3.8 - Use it as an iterator - - 3.8 - Use it as an context manager - """ - - def __init__(self, iterator): - self.iterator = iterator - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - pass - - def __iter__(self): - return self.iterator - - -class IsoAccessor(_IsoAccessor): - def _load_children(self, record): - assert record['type'] == 'dir' - start = SECTOR * record['sector'] - end = start + record['size'] - self.fileobj.seek(start) - while self.fileobj.tell() < end: - if not self.fileobj.peek(1)[0]: - self._unpack_to_boundary() - continue - try: - yield self._unpack_record() - except AssertionError: - self._unpack_to_boundary() - continue - - def _unpack_time(self, long=False): - if long: - s = self.fileobj.read(16).decode('ascii') - t = datetime.datetime.strptime(s, '%Y%m%d%H%M%S%f') - else: - y, m, d, H, M, S = self.fileobj.read(6) - - # For whatever reason, despite the ISO spec, many discs use 0-68 to to - # represent years between 2000 and 2068, and 69-99 to represent 1969-1999 - if y < 70: - y += 100 - - try: - t = datetime.datetime(1900 + y, m, d, H, M, S) - except ValueError: - t = datetime.datetime.min - - offset = int.from_bytes(self.fileobj.read(1), byteorder="little", signed=True) - t = t.replace(tzinfo=datetime.timezone(datetime.timedelta(minutes=15 * offset))) - return t - - def scandir(self, path): - return _ScandirIter((self.factory(path, name) for name in self.listdir(path))) \ No newline at end of file From 2fd965bf0cfa3395354215b0b6cc68009daaa2c7 Mon Sep 17 00:00:00 2001 From: Sazpaimon Date: Wed, 14 Aug 2024 22:26:47 -0400 Subject: [PATCH 2/2] Fix recent python incompatibility --- Pipfile | 2 +- common/processor.py | 4 ++-- patches.py | 4 ++-- post_psx/processor.py | 6 +++--- ps3/processor.py | 4 ++-- ps3/self_parser.py | 14 +++++++------- psp/processor.py | 2 +- utils/archives.py | 18 +++++++++++++----- xbox/processor.py | 16 ++++++++-------- 9 files changed, 39 insertions(+), 31 deletions(-) diff --git a/Pipfile b/Pipfile index 24d8989..87bf922 100755 --- a/Pipfile +++ b/Pipfile @@ -23,4 +23,4 @@ inflate64 = "*" bitarray = "*" [requires] -python_version = ">=3.8" +python_version = "3" diff --git a/common/processor.py b/common/processor.py index 0c2b64c..83e6f40 100644 --- a/common/processor.py +++ b/common/processor.py @@ -17,8 +17,8 @@ class BaseIsoProcessor: globally_ignored_paths = [ - re.compile(".*\.nfo$", re.IGNORECASE), - re.compile(".*\.diz$", re.IGNORECASE), + re.compile(r".*\.nfo$", re.IGNORECASE), + re.compile(r".*\.diz$", re.IGNORECASE), ] ignored_paths = [] _hash_bar_fmt = ' Hashing {file_name} {desc_pad}{percentage:3.0f}%|{bar}| ' \ diff --git a/patches.py b/patches.py index 9ca904c..7dedf38 100644 --- a/patches.py +++ b/patches.py @@ -321,9 +321,9 @@ def check_int_header(retcode, func, args): import zipfile orig_decodeExtra = zipfile.ZipInfo._decodeExtra - def _decodeExtra(self): + def _decodeExtra(*args): try: - orig_decodeExtra(self) + orig_decodeExtra(*args) except zipfile.BadZipfile: pass zipfile.ZipInfo._decodeExtra = _decodeExtra diff --git a/post_psx/processor.py b/post_psx/processor.py index 1400547..4698c37 100644 --- a/post_psx/processor.py +++ b/post_psx/processor.py @@ -10,9 +10,9 @@ class PostPsxIsoProcessor(BaseIsoProcessor): exe_patterns = [ - re.compile(".*/EBOOT\.BIN$", re.IGNORECASE), - re.compile(".*\.self$", re.IGNORECASE), - re.compile(".*\.sprx$", re.IGNORECASE) + re.compile(r".*/EBOOT\.BIN$", re.IGNORECASE), + re.compile(r".*\.self$", re.IGNORECASE), + re.compile(r".*\.sprx$", re.IGNORECASE) ] SFO_HEADER_BYTES = 20 diff --git a/ps3/processor.py b/ps3/processor.py index 7d94217..79ab246 100644 --- a/ps3/processor.py +++ b/ps3/processor.py @@ -11,13 +11,13 @@ class Ps3IsoProcessor(PostPsxIsoProcessor): - update_folder = re.compile(".*PS3_UPDATE$", re.IGNORECASE) + update_folder = re.compile(r".*PS3_UPDATE$", re.IGNORECASE) sfo_path = "/PS3_GAME/PARAM.SFO" @property def ignored_paths(self): paths = self.exe_patterns + [self.update_folder] - paths.append(re.compile("(?!^\/PS3_GAME\/USRDIR\/)", re.IGNORECASE)) + paths.append(re.compile(r"(?!^\/PS3_GAME\/USRDIR\/)", re.IGNORECASE)) return paths def get_disc_type(self): diff --git a/ps3/self_parser.py b/ps3/self_parser.py index 78fcec2..d3032c7 100644 --- a/ps3/self_parser.py +++ b/ps3/self_parser.py @@ -2,7 +2,7 @@ import io import struct import zlib -from dataclasses import dataclass +from dataclasses import dataclass, field from Crypto.Cipher import AES from Crypto.Util import Counter @@ -126,18 +126,18 @@ class SegmentCertHeader(Struct): @dataclass class Attributes(Struct): - key: bytearray = bytearray(0x10) - iv: bytearray = bytearray(0x10) + key: bytearray = field(default_factory=lambda: bytearray(0x10)) + iv: bytearray = field(default_factory=lambda: bytearray(0x10)) struct = struct.Struct(">16s16s") @dataclass class EncryptionRootHeader(Struct): - key: bytearray = bytearray(0x10) - key_pad: bytearray = bytearray(0x10) - iv: bytearray = bytearray(0x10) - iv_pad: bytearray = bytearray(0x10) + key: bytearray = field(default_factory=lambda: bytearray(0x10)) + key_pad: bytearray = field(default_factory=lambda: bytearray(0x10)) + iv: bytearray = field(default_factory=lambda: bytearray(0x10)) + iv_pad: bytearray = field(default_factory=lambda: bytearray(0x10)) struct = struct.Struct(">16s16s16s16s") diff --git a/psp/processor.py b/psp/processor.py index 2522719..9f74aaf 100644 --- a/psp/processor.py +++ b/psp/processor.py @@ -11,7 +11,7 @@ class PspIsoProcessor(PostPsxIsoProcessor): - update_folder = re.compile(".*/PSP_GAME/SYSDIR/UPDATE/$", re.IGNORECASE) + update_folder = re.compile(r".*/PSP_GAME/SYSDIR/UPDATE/$", re.IGNORECASE) sfo_path = "/PSP_GAME/PARAM.SFO" def __init__(self, iso_path_reader, iso_filename, system, progress_manager): diff --git a/utils/archives.py b/utils/archives.py index 10165b6..e77e4cf 100644 --- a/utils/archives.py +++ b/utils/archives.py @@ -349,10 +349,13 @@ def _open_unrar(rar, inf, pwd=None, tmpfile=None, force_file=False): if self.ctx.filelist: flags = struct.unpack("