From c7029baed425b9f16911f806da3189ed8a0d5963 Mon Sep 17 00:00:00 2001 From: Sazpaimon Date: Mon, 26 Aug 2024 12:45:37 -0400 Subject: [PATCH 1/8] PSX: Add support for PKG files and EDAT files. --- Pipfile | 1 + common/factory.py | 16 +- common/iso_path_reader/methods/pycdlib.py | 4 +- post_psx/npdrm/__init__.py | 0 post_psx/npdrm/path_reader.py | 145 ++++++ post_psx/npdrm/pkg.py | 244 +++++++++ post_psx/types.py | 204 ++++++++ post_psx/utils/__init__.py | 0 post_psx/utils/edat.py | 475 ++++++++++++++++++ ps3/path_reader.py | 71 +-- utils/archives.py | 8 +- utils/files.py | 58 +-- utils/in_memory_rollover_temp_file.py | 11 + .../pycdlib.py => utils/pycdlib/__init__.py | 8 +- utils/pycdlib/decrypted_file_io.py | 101 ++++ 15 files changed, 1260 insertions(+), 86 deletions(-) create mode 100644 post_psx/npdrm/__init__.py create mode 100644 post_psx/npdrm/path_reader.py create mode 100644 post_psx/npdrm/pkg.py create mode 100644 post_psx/types.py create mode 100644 post_psx/utils/__init__.py create mode 100644 post_psx/utils/edat.py rename common/pycdlib.py => utils/pycdlib/__init__.py (99%) create mode 100644 utils/pycdlib/decrypted_file_io.py diff --git a/Pipfile b/Pipfile index 59f43ac..37cc796 100755 --- a/Pipfile +++ b/Pipfile @@ -22,6 +22,7 @@ psutil = "*" stream-inflate = "*" inflate64 = "*" bitarray = "*" +xor-cipher = "*" [requires] python_version = "3.8" diff --git a/common/factory.py b/common/factory.py index c445d82..4758329 100644 --- a/common/factory.py +++ b/common/factory.py @@ -5,7 +5,9 @@ import rarfile -from common import pycdlib +from post_psx.npdrm.path_reader import NPDRMPathReader +from post_psx.npdrm.pkg import Pkg +from utils import pycdlib from pyisotools.iso import GamecubeISO from cdi.path_reader import CdiPathReader @@ -126,7 +128,7 @@ def get_iso_path_readers(fp, file_name, parent_container, pbar): exceptions[CompressedPathReader.volume_type] = e fp.seek(0) - if fp.read(4) == b"LIVE": + if fp.read(4) == b"LIVE" and parent_container.get_file_size(parent_container.get_file(fp.name)) >= 0x971A: stfs = STFS(filename=None, fd=fp) if stfs.content_type == 0x7000: # Not an XBLA archive, actually a GOD game. Process like a concatenated ISO @@ -191,6 +193,16 @@ def get_iso_path_readers(fp, file_name, parent_container, pbar): else: wrapper = fp + wrapper.seek(0) + if wrapper.peek(4) == b"\x7FPKG": + try: + reader = Pkg(wrapper) + reader.parse_header() + reader.parse_metadata() + path_readers.append(NPDRMPathReader(reader, wrapper, parent_container)) + except Exception as e: + exceptions[NPDRMPathReader.volume_type] = e + wrapper.seek(0) if wrapper.peek(7) == b"\x01\x5A\x5A\x5A\x5A\x5A\x01" and file_name != "Disc label": try: diff --git a/common/iso_path_reader/methods/pycdlib.py b/common/iso_path_reader/methods/pycdlib.py index a72eb71..edd93ec 100644 --- a/common/iso_path_reader/methods/pycdlib.py +++ b/common/iso_path_reader/methods/pycdlib.py @@ -44,9 +44,9 @@ def iso_iterator(self, base_dir, recursive=False, include_dirs=False): continue if file.is_dir(): + if include_dirs: + yield file if recursive: - if include_dirs: - yield file yield from self.iso_iterator(file, recursive, include_dirs) continue diff --git a/post_psx/npdrm/__init__.py b/post_psx/npdrm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/post_psx/npdrm/path_reader.py b/post_psx/npdrm/path_reader.py new file mode 100644 index 0000000..a33fd28 --- /dev/null +++ b/post_psx/npdrm/path_reader.py @@ -0,0 +1,145 @@ +import io +import logging +import os +import pathlib + +from Crypto.Cipher import AES +from Crypto.Util import Counter + +from common.iso_path_reader.methods.base import IsoPathReader +from common.iso_path_reader.methods.chunked_hash_trait import ChunkedHashTrait +from post_psx.utils.edat import EdatFile +from utils import pycdlib +from utils.files import OffsetFile +from utils.pycdlib.decrypted_file_io import DecryptedFileIO + + +LOGGER = logging.getLogger(__name__) + + +class DecryptedFileReader(DecryptedFileIO): + def __init__(self, ino, logical_block_size, pkg, entry): + super().__init__(ino, logical_block_size, buffer_blocks=65536 // logical_block_size) + self.pkg = pkg + self.entry = entry + self.pkg.init_cipher(ino.extent_location(), entry.key) + + def seek(self, offset, whence=0): + super().seek(offset, whence) + self.pkg.init_cipher( + self._ctxt.ino.extent_location() + (self._offset // self.logical_block_size), self.entry.key + ) + + def decrypt_blocks(self, blocks): + return self.pkg.decrypt(blocks) + + +class NPDRMPathReader(ChunkedHashTrait, IsoPathReader): + volume_type = "npdrm" + + def __init__(self, iso, fp, parent_container): + super().__init__(iso, fp, parent_container) + pkg_header = self.iso.pkg_header + self.fp = OffsetFile(self.fp, pkg_header.data_offset, pkg_header.data_offset + pkg_header.data_size) + self.edat_key = self.get_edat_key() + + def get_edat_key(self): + edat_file = next((file for file in self.iso_iterator(self.get_root_dir(), recursive=True) if + self.get_file_path(file).lower().endswith(".edat")), None) + if not edat_file: + return + + edat_key = None + file_dir = pathlib.Path(self.fp.name).parent + try: + title_id = self.iso.pkg_header.title_id.decode("utf-8") + # Check for a rap file in the dir + try: + with self.parent_container.open_file(self.parent_container.get_file(str((file_dir / title_id).with_suffix(".rap")))) as f: + edat_key = EdatFile.rap_to_rif(f.read()) + except FileNotFoundError: + pass + except UnicodeDecodeError: + pass + + if edat_key: + if self.test_key(edat_key, edat_file): + return edat_key + else: + test_edat_key = bytes.fromhex("0" * 32) + if self.test_key(test_edat_key, edat_file): + return test_edat_key + + def test_key(self, key, file): + file_path = self.get_file_path(file) + inode = pycdlib.inode.Inode() + inode.parse(file.file_offset // 16, file.file_size, self.fp, 16) + with DecryptedFileReader(inode, 16, self.iso, file) as f, \ + EdatFile(f, os.path.basename(file_path), key) as f: + return f.decrypt_block(0, key) != -1 + + def get_root_dir(self): + return None + + def iso_iterator(self, base_dir, recursive=False, include_dirs=False): + # always recursive + for file in self.iso.entries(): + if self.is_directory(file) and not include_dirs: + continue + yield file + + def get_file(self, path): + try: + return next(file for file in self.iso.entries() if file.name_decoded == path) + except StopIteration: + raise FileNotFoundError + + def get_file_date(self, file): + return None + + def get_file_path(self, file): + return file.name_decoded + + def open_file(self, file): + inode = pycdlib.inode.Inode() + inode.parse(file.file_offset // 16, file.file_size, self.fp, 16) + f = DecryptedFileReader(inode, 16, self.iso, file) + file_path = self.get_file_path(file) + if file_path.lower().endswith("edat"): + f.__enter__() + edat_file = EdatFile(f, os.path.basename(file_path), self.edat_key) + if not edat_file.validate_npd_hashes(os.path.basename(file_path), edat_file.hash_key): + LOGGER.warning("Could not validate header hashes for %s", file_path) + if edat_file.edat_header.file_size == 0: + f.__exit__() + f = io.BytesIO(b"") + elif not self.edat_key: + LOGGER.warning("Could not locate decryption key for %s. File will not be decrypted", file_path) + else: + f = edat_file + + f.name = self.get_file_path(file) + return f + + def get_file_sector(self, file): + raise NotImplementedError + + def get_file_size(self, file): + file_path = self.get_file_path(file) + if file_path.lower().endswith("edat"): + inode = pycdlib.inode.Inode() + inode.parse(file.file_offset // 16, file.file_size, self.fp, 16) + with DecryptedFileReader(inode, 16, self.iso, file) as f: + edat_size = EdatFile(f, os.path.basename(file_path), self.edat_key).edat_header.file_size + if edat_size == 0 or self.edat_key: + return edat_size + return file.file_size + + def is_directory(self, file): + return file.is_dir + + def get_pvd(self): + return None + + def get_pvd_info(self): + return {} diff --git a/post_psx/npdrm/pkg.py b/post_psx/npdrm/pkg.py new file mode 100644 index 0000000..55ea358 --- /dev/null +++ b/post_psx/npdrm/pkg.py @@ -0,0 +1,244 @@ +import hashlib +import struct + +from Crypto.Cipher import AES +from Crypto.Hash import SHA1 +from Crypto.Util import Counter, strxor +from Crypto.Util.number import long_to_bytes + +from post_psx.types import PKGMetaData, PKGHeader, PKGExtHeader, PKGEntry +from utils.files import get_file_size +from xor_cipher import cyclic_xor + +import logging + +LOGGER = logging.getLogger(__name__) + + +class DebugCTR: + def __init__(self, pkg_header_digest, block_num): + self.key = bytearray(pkg_header_digest[:8] * 2 + pkg_header_digest[8:16] * 2 + bytes(32)) + self.counter = block_num + self.block = bytes(16) + self.block_index = 16 # Force initial update + + def update_block(self): + self.key[56:64] = long_to_bytes(self.counter, 8) + self.block = hashlib.sha1(self.key).digest()[:16] + self.block_index = 0 + self.counter += 1 + + def crypt(self, data): + data_len = len(data) + result = bytearray(data_len) + i = 0 + while i < data_len: + if self.block_index == 16: + self.update_block() + chunk_size = min(16 - self.block_index, data_len - i) + result[i:i + chunk_size] = cyclic_xor( + data[i:i + chunk_size], + self.block[self.block_index:self.block_index + chunk_size] + ) + self.block_index += chunk_size + i += chunk_size + return bytes(result) + + encrypt = decrypt = crypt + + +class Pkg: + PSP_AES_KEY = bytes([0x07, 0xF2, 0xC6, 0x82, 0x90, 0xB5, 0x0D, 0x2C, + 0x33, 0x81, 0x8D, 0x70, 0x9B, 0x60, 0xE6, 0x2B]) + + PS3_AES_KEY = bytes([0x2E, 0x7B, 0x71, 0xD7, 0xC9, 0xC9, 0xA1, 0x4E, + 0xA3, 0x22, 0x1F, 0x18, 0x88, 0x28, 0xB8, 0xF8]) + + PKG_AES_KEY_VITA_1 = bytes([0xE3, 0x1A, 0x70, 0xC9, 0xCE, 0x1D, 0xD7, 0x2B, + 0xF3, 0xC0, 0x62, 0x29, 0x63, 0xF2, 0xEC, 0xCB]) + + PKG_AES_KEY_VITA_2 = bytes([0x42, 0x3A, 0xCA, 0x3A, 0x2B, 0xD5, 0x64, 0x9F, + 0x96, 0x86, 0xAB, 0xAD, 0x6F, 0xD8, 0x80, 0x1F]) + + PKG_AES_KEY_VITA_3 = bytes([0xAF, 0x07, 0xFD, 0x59, 0x65, 0x25, 0x27, 0xBA, + 0xF1, 0x33, 0x89, 0x66, 0x8B, 0x17, 0xD9, 0xEA]) + + PKG_PLATFORM_TYPE_PS3 = 0x0001 + PKG_PLATFORM_TYPE_PSP_PSVITA = 0x0002 + + PKG_FILE_ENTRY_PSP = 0x10000000 + + PKG_DEBUG_TYPE = 0x0000 + PKG_RETAIL_TYPE = 0x8000 + + def __init__(self, fp): + self.fp = fp + self.pkg_header = None + self.pkg_ext_header = None + self.metadata = PKGMetaData() + self._entries = {} + self.pkg_key = None + self.aes_context = None + + def parse_header(self): + self.fp.seek(0) + self.pkg_header = PKGHeader.unpack(self.fp.read(PKGHeader.struct.size)) + + LOGGER.debug("Header: pkg_magic = 0x%x = %s", + int.from_bytes(self.pkg_header.pkg_magic, byteorder="little"), + self.pkg_header.pkg_magic) + LOGGER.debug("Header: pkg_type = 0x%x = %d", *[self.pkg_header.pkg_type] * 2) + LOGGER.debug("Header: pkg_platform = 0x%x = %d", *[self.pkg_header.pkg_type] * 2) + LOGGER.debug("Header: meta_offset = 0x%x = %d", *[self.pkg_header.meta_offset] * 2) + LOGGER.debug("Header: meta_count = 0x%x = %d", *[self.pkg_header.meta_count] * 2) + LOGGER.debug("Header: meta_size = 0x%x = %d", *[self.pkg_header.meta_size] * 2) + LOGGER.debug("Header: file_count = 0x%x = %d", *[self.pkg_header.file_count] * 2) + LOGGER.debug("Header: pkg_size = 0x%x = %d", *[self.pkg_header.pkg_size] * 2) + LOGGER.debug("Header: data_offset = 0x%x = %d", *[self.pkg_header.data_offset] * 2) + LOGGER.debug("Header: data_size = 0x%x = %d", *[self.pkg_header.data_size] * 2) + LOGGER.debug("Header: title_id = %s", self.pkg_header.title_id) + LOGGER.debug("Header: qa_digest = 0x%x", int.from_bytes(self.pkg_header.digest, byteorder="little")) + + if not self.pkg_header.check_magic(): + LOGGER.error("Not a PKG file!") + return False + + if self.pkg_header.pkg_platform == self.PKG_PLATFORM_TYPE_PS3: + self.pkg_key = self.PS3_AES_KEY + elif self.pkg_header.pkg_platform == self.PKG_PLATFORM_TYPE_PSP_PSVITA: + self.pkg_key = self.PSP_AES_KEY + else: + LOGGER.error("PKG type not supported") + return False + + if self.pkg_header.pkg_platform == self.PKG_PLATFORM_TYPE_PSP_PSVITA: + # Parse extended header for PSP/Vita packages + self.fp.seek(PKGHeader.struct.size) + self.pkg_ext_header = PKGExtHeader.unpack(self.fp.read(PKGExtHeader.struct.size)) + if not self.pkg_ext_header.check_magic(): + LOGGER.error("PKG extended header corrupt") + return False + + LOGGER.debug("Extended header: magic = 0x%x = %s", + int.from_bytes(self.pkg_ext_header.magic, byteorder="little"), + self.pkg_ext_header.magic) + LOGGER.debug("Extended header: unknown_1 = 0x%x = %d", *[self.pkg_ext_header.unknown_1] * 2) + LOGGER.debug("Extended header: ext_hdr_size = 0x%x = %d", *[self.pkg_ext_header.ext_hdr_size] * 2) + LOGGER.debug("Extended header: ext_data_size = 0x%x = %d", *[self.pkg_ext_header.ext_data_size] * 2) + LOGGER.debug("Extended header: main_and_self.pkg_ext_headers_hmac_offset = 0x%x = %d", + *[self.pkg_ext_header.main_and_ext_headers_hmac_offset] * 2) + LOGGER.debug("Extended header: metadata_header_hmac_offset = 0x%x = %d", + *[self.pkg_ext_header.metadata_header_hmac_offset] * 2) + LOGGER.debug("Extended header: tail_offset = 0x%x = %d", *[self.pkg_ext_header.tail_offset] * 2) + LOGGER.debug("Extended header: pkg_key_id = 0x%x = %d", *[self.pkg_ext_header.pkg_key_id] * 2) + LOGGER.debug("Extended header: full_header_hmac_offset = 0x%x = %d", + *[self.pkg_ext_header.full_header_hmac_offset] * 2) + + if self.pkg_header.pkg_type not in [self.PKG_DEBUG_TYPE, self.PKG_RETAIL_TYPE]: + LOGGER.error(f"Unknown PKG type (0x{self.pkg_header.pkg_type:x})") + return False + + # TODO: Split packages + if self.pkg_header.pkg_size > get_file_size(self.fp): + LOGGER.error("Split package not support") + return False + + if self.pkg_header.data_size + self.pkg_header.data_offset > self.pkg_header.pkg_size: + LOGGER.error(f"PKG data size mismatch (" + f"data_size=0x{self.pkg_header.data_size:x}, " + f"data_offset=0x{self.pkg_header.data_size:x}, " + f"file_size=0x{self.pkg_header.data_size:x})") + return False + + return True + + def parse_metadata(self): + self.fp.seek(self.pkg_header.meta_offset) + for i in range(0, self.pkg_header.meta_count): + metadata_id, entry_size = struct.unpack(">II", self.fp.read(8)) + metadata = self.fp.read(entry_size) + if metadata_id == 0x1: + self.metadata.drm_type, = struct.unpack(">I", metadata) + elif metadata_id == 0x2: + self.metadata.content_type, = struct.unpack(">I", metadata) + elif metadata_id == 0x3: + self.metadata.package_type, = struct.unpack(">I", metadata) + elif metadata_id == 0x4: + self.metadata.package_size, = struct.unpack(">Q", metadata) + elif metadata_id == 0x5: + self.metadata.package_revision = self.metadata.package_revision.unpack(metadata) + elif metadata_id == 0x6: + self.metadata.title_id = metadata + elif metadata_id == 0x7: + self.metadata.qa_digest = metadata + elif metadata_id == 0x8: + self.metadata.software_revision = self.metadata.software_revision.unpack(metadata) + elif metadata_id == 0x9: + self.metadata.unk_0x9, = struct.unpack(">Q", metadata) + elif metadata_id == 0xA: + self.metadata.install_dir = metadata + elif metadata_id == 0xB: + self.metadata.unk_0xB, = struct.unpack(">Q", metadata) + elif metadata_id == 0xC: + continue + elif metadata_id == 0xD: + self.metadata.item_info = self.metadata.item_info.unpack(metadata) + elif metadata_id == 0xE: + self.metadata.sfo_info = self.metadata.sfo_info.unpack(metadata) + elif metadata_id == 0xF: + self.metadata.unknown_data_info = self.metadata.unknown_data_info.unpack(metadata) + elif metadata_id == 0x10: + self.metadata.entirety_info = self.metadata.entirety_info.unpack(metadata) + elif metadata_id == 0x11: + self.metadata.version_info = self.metadata.version_info.unpack(metadata) + elif metadata_id == 0x12: + self.metadata.self_info = self.metadata.self_info.unpack(metadata) + else: + LOGGER.error("Unknown metadata type %d", metadata_id) + continue + + if self.pkg_header.pkg_platform == self.PKG_PLATFORM_TYPE_PSP_PSVITA and 0x15 <= self.metadata.content_type <= 0x17: + if self.metadata.content_type == 0x15: + key = self.PKG_AES_KEY_VITA_1 + elif self.metadata.content_type == 0x16: + key = self.PKG_AES_KEY_VITA_2 + else: + key = self.PKG_AES_KEY_VITA_3 + cipher = AES.new(key, AES.MODE_ECB) + self.pkg_key = cipher.encrypt(bytes(self.pkg_header.pkg_data_riv)) + + def init_cipher(self, block_num, key): + if self.pkg_header.pkg_type == self.PKG_RETAIL_TYPE: + ctr = Counter.new(128, initial_value=( + int.from_bytes(self.pkg_header.pkg_data_riv, byteorder='big') + block_num + )) + self.aes_context = AES.new(key, AES.MODE_CTR, counter=ctr) + else: + # Debug mode + self.aes_context = DebugCTR(self.pkg_header.digest, block_num) + + def entries(self): + if len(self._entries) == self.pkg_header.file_count: + yield from iter(self._entries.values()) + + for block in range(0, self.pkg_header.file_count * 2, 2): + self.init_cipher(block, self.pkg_key) + self.fp.seek(self.pkg_header.data_offset + (16 * block)) + enc = self.fp.read(32) + entry_data = self.decrypt(enc) + entry = PKGEntry.unpack(entry_data) + + name_block_start = entry.name_offset // 16 + self.fp.seek(self.pkg_header.data_offset + (16 * name_block_start)) + if entry.type & self.PKG_FILE_ENTRY_PSP: + entry.key = self.PSP_AES_KEY + else: + entry.key = self.PS3_AES_KEY + self.init_cipher(name_block_start, entry.key) + name = self.decrypt(self.fp.read(entry.name_size)) + entry.name_decoded = name.rstrip(b"\x00").decode("utf-8", errors="replace") + self._entries[entry.name_decoded] = entry + yield entry + + def decrypt(self, blocks): + return self.aes_context.encrypt(blocks) diff --git a/post_psx/types.py b/post_psx/types.py new file mode 100644 index 0000000..a576e31 --- /dev/null +++ b/post_psx/types.py @@ -0,0 +1,204 @@ +import struct + +from dataclasses import dataclass, field +from typing import List + +from ps3.self_parser import Struct + + +@dataclass +class PKGHeader(Struct): + pkg_magic: bytes # Magic (0x7f504b47) (" PKG") + pkg_type: int # Release type (Retail:0x8000, Debug:0x0000) + pkg_platform: int # Platform type (PS3:0x0001, PSP:0x0002) + meta_offset: int # Metadata offset. Usually 0xC0 for PS3, usually 0x280 for PSP and PSVita + meta_count: int # Metadata item count + meta_size: int # Metadata size. + file_count: int # Number of files + pkg_size: int # PKG size in bytes + data_offset: int # Encrypted data offset + data_size: int # Encrypted data size in bytes + title_id: bytes = field(default_factory=lambda: b'\x00' * 0x24) # Title ID + digest: bytearray = field(default_factory=lambda: bytearray(0x10)) # hash of "files + attribs" + pkg_data_riv: bytearray = field(default_factory=lambda: bytearray(0x10)) + pkg_header_digest: bytearray = field(default_factory=lambda: bytearray(0x40)) + + struct = struct.Struct(">4sHHIIIIQQQ36s12x16s16s64s") + + def check_magic(self): + return self.pkg_magic == b"\x7FPKG" + + +@dataclass +class PKGExtHeader(Struct): + magic: bytes # 0x7F657874 (" ext") + unknown_1: int # Maybe version. always 1 + ext_hdr_size: int # Extended header size. ex: 0x40 + ext_data_size: int # ex: 0x180 + main_and_ext_headers_hmac_offset: int # ex: 0x100 + metadata_header_hmac_offset: int # ex: 0x360, 0x390, 0x490 + tail_offset: int # tail size seams to be always 0x1A0 + padding1: int # Padding + pkg_key_id: int # ID of the AES key used for decryption. PSP = 0x1, PSVita = 0xC0000002, PSM = 0xC0000004 + full_header_hmac_offset: int # ex: none (old pkg): 0, 0x930 + padding2: bytes = field(default_factory=lambda: b'\x00' * 20) + + struct = struct.Struct(">4sIIIIIQIII20s") + + def check_magic(self): + return self.magic == b"\x7Fext" + + +@dataclass +class PKGEntry(Struct): + name_offset: int # File name offset + name_size: int # File name size + file_offset: int # File offset + file_size: int # File size + type: int # File type + pad: int # Padding (zeros) + name_decoded: str = "" + key: bytes = b"" + + struct = struct.Struct(">IIQQII") + + @property + def is_dir(self): + return self.type & 0xFF == 0x04 and not self.file_size + + +def to_hex_string(buf: bytes, dotpos: int = None) -> str: + hex_str = ''.join(f'{b:02x}' for b in buf) + if dotpos is not None and len(hex_str) > dotpos: + hex_str = hex_str[:dotpos] + '.' + hex_str[dotpos:] + return hex_str + + +@dataclass +class PackageRevision(Struct): + make_package_npdrm_ver: bytes = field(default_factory=lambda: b'\x00' * 2) + version: bytes = field(default_factory=lambda: b'\x00' * 2) + + struct = struct.Struct(">HH") + + +@dataclass +class SoftwareRevision(Struct): + unk: bytes = field(default_factory=lambda: b'\x00') + firmware_version: bytes = field(default_factory=lambda: b'\x00' * 3) + version: bytes = field(default_factory=lambda: b'\x00' * 2) + app_version: bytes = field(default_factory=lambda: b'\x00' * 2) + + struct = struct.Struct(">s3s2s2s") + + +@dataclass +class VitaItemInfo(Struct): + offset: int = 0 + size: int = 0 + sha256: bytes = field(default_factory=lambda: b'\x00' * 32) + + struct = struct.Struct(">II32s") + + +@dataclass +class VitaSfoInfo(Struct): + param_offset: int = 0 + param_size: int = 0 + unk_1: int = 0 + psp2_system_ver: int = 0 + unk_2: bytes = field(default_factory=lambda: b'\x00' * 8) + param_digest: bytes = field(default_factory=lambda: b'\x00' * 32) + + struct = struct.Struct(">IIII8s32s") + + +@dataclass +class VitaUnknownDataInfo(Struct): + unknown_data_offset: int = 0 + unknown_data_size: int = 0 + unk: bytes = field(default_factory=lambda: b'\x00' * 32) + unknown_data_sha256: bytes = field(default_factory=lambda: b'\x00' * 32) + + struct = struct.Struct(">IH32s32s2x") + + +@dataclass +class VitaEntiretyInfo(Struct): + entirety_data_offset: int = 0 + entirety_data_size: int = 0 + flags: int = 0 + unk_1: int = 0 + unk_2: int = 0 + unk_3: bytes = field(default_factory=lambda: b'\x00' * 8) + entirety_digest: bytes = field(default_factory=lambda: b'\x00' * 32) + + struct = struct.Struct(">IIHHI8s32s") + + +@dataclass +class VitaVersionInfo(Struct): + publishing_tools_version: int = 0 + psf_builder_version: int = 0 + padding: bytes = field(default_factory=lambda: b'\x00' * 32) + + struct = struct.Struct(">II32s") + + +@dataclass +class VitaSelfInfo(Struct): + self_info_offset: int = 0 + self_info_size: int = 0 + unk: bytes = field(default_factory=lambda: b'\x00' * 16) + self_sha256: bytes = field(default_factory=lambda: b'\x00' * 32) + + struct = struct.Struct(">II16s32s") + + +@dataclass +class PKGMetaData: + drm_type: int = 0 + content_type: int = 0 + package_type: int = 0 + package_size: int = 0 + qa_digest: bytes = field(default_factory=lambda: b'\x00' * 24) + unk_0x9: int = 0 + unk_0xB: int = 0 + package_revision: PackageRevision = field(default_factory=PackageRevision) + software_revision: SoftwareRevision = field(default_factory=SoftwareRevision) + title_id: bytes = b"" + install_dir: bytes = b"" + item_info: VitaItemInfo = field(default_factory=VitaItemInfo) + sfo_info: VitaSfoInfo = field(default_factory=VitaSfoInfo) + unknown_data_info: VitaUnknownDataInfo = field(default_factory=VitaUnknownDataInfo) + entirety_info: VitaEntiretyInfo = field(default_factory=VitaEntiretyInfo) + version_info: VitaVersionInfo = field(default_factory=VitaVersionInfo) + self_info: VitaSelfInfo = field(default_factory=VitaSelfInfo) + + +@dataclass +class NPDHeader(Struct): + magic: bytes + version: int + license: int + app_type: int + content_id: List[int] = field(default_factory=lambda: [0] * 0x30) + digest: List[int] = field(default_factory=lambda: [0] * 0x10) + title_hash: List[int] = field(default_factory=lambda: [0] * 0x10) + dev_hash: List[int] = field(default_factory=lambda: [0] * 0x10) + activate_time: int = 0 + expire_time: int = 0 + + struct = struct.Struct(">Iiii48s16s16s16sqq") + + def check_magic(self): + return self.magic == b"NPD\0" + + +@dataclass +class EDATHeader(Struct): + flags: int + block_size: int + file_size: int + + struct = struct.Struct(">iiQ") diff --git a/post_psx/utils/__init__.py b/post_psx/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/post_psx/utils/edat.py b/post_psx/utils/edat.py new file mode 100644 index 0000000..5ea2318 --- /dev/null +++ b/post_psx/utils/edat.py @@ -0,0 +1,475 @@ +import io +import struct + +from Crypto.Cipher import AES +from Crypto.Hash import CMAC, SHA1, HMAC + +from post_psx.types import NPDHeader, EDATHeader + + +class EdatFile(io.RawIOBase): + SDAT_FLAG = 0x01000000 + EDAT_COMPRESSED_FLAG = 0x00000001 + EDAT_FLAG_0x02 = 0x00000002 + EDAT_ENCRYPTED_KEY_FLAG = 0x00000008 + EDAT_FLAG_0x10 = 0x00000010 + EDAT_FLAG_0x20 = 0x00000020 + EDAT_DEBUG_DATA_FLAG = 0x80000000 + + SDAT_KEY = bytearray([0x0D, 0x65, 0x5E, 0xF8, 0xE6, 0x74, 0xA9, 0x8A, + 0xB8, 0x50, 0x5C, 0xFA, 0x7D, 0x01, 0x29, 0x33]) + NP_OMAC_KEY_2 = bytearray([0x6B, 0xA5, 0x29, 0x76, 0xEF, 0xDA, 0x16, 0xEF, + 0x3C, 0x33, 0x9F, 0xB2, 0x97, 0x1E, 0x25, 0x6B]) + NP_OMAC_KEY_3 = bytearray([0x9B, 0x51, 0x5F, 0xEA, 0xCF, 0x75, 0x06, 0x49, + 0x81, 0xAA, 0x60, 0x4D, 0x91, 0xA5, 0x4E, 0x97]) + RAP_KEY = bytearray([0x86, 0x9F, 0x77, 0x45, 0xC1, 0x3F, 0xD8, 0x90, + 0xCC, 0xF2, 0x91, 0x88, 0xE3, 0xCC, 0x3E, 0xDF]) + RAP_PBOX = bytearray([0x0C, 0x03, 0x06, 0x04, 0x01, 0x0B, 0x0F, 0x08, + 0x02, 0x07, 0x00, 0x05, 0x0A, 0x0E, 0x0D, 0x09]) + RAP_E1 = bytearray([0xA9, 0x3E, 0x1F, 0xD6, 0x7C, 0x55, 0xA3, 0x29, + 0xB7, 0x5F, 0xDD, 0xA6, 0x2A, 0x95, 0xC7, 0xA5]) + RAP_E2 = bytearray([0x67, 0xD4, 0x5D, 0xA3, 0x29, 0x6D, 0x00, 0x6A, + 0x4E, 0x7C, 0x53, 0x7B, 0xF5, 0x53, 0x8C, 0x74]) + EDAT_KEY_0 = bytearray([0xBE, 0x95, 0x9C, 0xA8, 0x30, 0x8D, 0xEF, 0xA2, + 0xE5, 0xE1, 0x80, 0xC6, 0x37, 0x12, 0xA9, 0xAE]) + EDAT_HASH_0 = bytearray([0xEF, 0xFE, 0x5B, 0xD1, 0x65, 0x2E, 0xEB, 0xC1, + 0x19, 0x18, 0xCF, 0x7C, 0x04, 0xD4, 0xF0, 0x11]) + EDAT_KEY_1 = bytearray([0x4C, 0xA9, 0xC1, 0x4B, 0x01, 0xC9, 0x53, 0x09, + 0x96, 0x9B, 0xEC, 0x68, 0xAA, 0x0B, 0xC0, 0x81]) + EDAT_HASH_1 = bytearray([0x3D, 0x92, 0x69, 0x9B, 0x70, 0x5B, 0x07, 0x38, + 0x54, 0xD8, 0xFC, 0xC6, 0xC7, 0x67, 0x27, 0x47]) + EDAT_IV = bytearray(0x10) + + def __init__(self, fp, file_name, edat_key): + self.fp = fp + self.fp.seek(0) + self.npd_header = NPDHeader.unpack(fp.read(NPDHeader.struct.size)) + self.fp.seek(0x80) + edh = self.fp.read(EDATHeader.struct.size) + self.edat_header = EDATHeader.unpack(edh) + self.edat_key = edat_key + self._pos = 0 + self._size = self.edat_header.file_size + self.total_blocks = -(self._size // -self.edat_header.block_size) + + self.hash_key = int.from_bytes(self.npd_header.dev_hash, byteorder="little") + if self.edat_header.flags & self.SDAT_FLAG: + self.hash_key ^= int.from_bytes(self.SDAT_KEY, byteorder="little") + + @classmethod + def rap_to_rif(cls, rap): + key = bytearray(0x10) + iv = bytearray(0x10) + + # Initial decrypt. + aes = AES.new(cls.RAP_KEY, AES.MODE_CBC, iv) + key[:] = aes.decrypt(rap) + + # rap2rifkey round. + for _ in range(5): + for i in range(16): + p = cls.RAP_PBOX[i] + key[p] ^= cls.RAP_E1[p] + + for i in range(15, 0, -1): + p = cls.RAP_PBOX[i] + pp = cls.RAP_PBOX[i - 1] + key[p] ^= key[pp] + + o = 0 + for i in range(16): + p = cls.RAP_PBOX[i] + kc = (key[p] - o) & 0xFF # Ensure the subtraction is done modulo 256 + ec2 = cls.RAP_E2[p] + if o != 1 or kc != 0xFF: + o = 1 if kc < ec2 else 0 + key[p] = (kc - ec2) & 0xFF + else: + key[p] = (kc - ec2) & 0xFF + + return bytes(key) + + def validate_npd_hashes(self, file_name, dec_key): + if self.edat_header.flags & self.EDAT_DEBUG_DATA_FLAG: + return True + + if not self.validate_dev_klic(dec_key): + return False + + # Build the title buffer (content_id + file_name). + buf = bytearray(self.npd_header.content_id[:0x30] + file_name.encode('utf-8')) + buf_lower = bytearray(buf) + buf_upper = bytearray(buf) + + dot_index = file_name.rfind('.') + if dot_index != -1: + buf_lower = buf[0:-dot_index] + buf[-dot_index:].lower() + buf_upper = buf[0:-dot_index] + buf[-dot_index:].lower() + + # Hash with NPDRM_OMAC_KEY_3 and compare with title_hash. + # Try to ignore case sensitivity with file extension + def cmac_compare(buffer): + c = CMAC.new(self.NP_OMAC_KEY_3, ciphermod=AES) + c.update(buffer) + return c.digest() == self.npd_header.title_hash + + title_hash_result = ( + cmac_compare(buf) or + cmac_compare(buf_lower) or + cmac_compare(buf_upper) + ) + + return title_hash_result + + def validate_dev_klic2(self, dec_key): + if (self.npd_header.license & 0x3) != 0x3: + return True + + dev = bytearray(0x60) + + # Build the dev buffer (first 0x60 bytes of NPD header in big-endian). + dev[:0x60] = self.npd_header.pack()[:0x60] + + # Fix endianness. + version = struct.unpack('>i', struct.pack('i', struct.pack('i', struct.pack('i', version) + dev[0x8:0xC] = struct.pack('>i', license) + dev[0xC:0x10] = struct.pack('>i', app_type) + + # Check for an empty dev_hash (can't validate if devklic is NULL). + klic = dec_key + + # Generate klicensee xor key. + key = klic ^ int.from_bytes(self.NP_OMAC_KEY_2, byteorder='big') + + # Hash with generated key and compare with dev_hash. + key_bytes = key.to_bytes(16, byteorder='big') + c = CMAC.new(key_bytes, ciphermod=AES) + c.update(dev[:0x60]) + generated_hash = c.digest() + + return generated_hash == self.npd_header.dev_hash + + def validate_dev_klic(self, dec_key): + if (self.npd_header.license & 0x3) != 0x3: + return True + + dev = bytearray(0x60) + + # Build the dev buffer (first 0x60 bytes of NPD header in big-endian) + dev[:0x60] = self.npd_header.pack()[:0x60] + + # Fix endianness + version = struct.pack('>I', self.npd_header.version) + license = struct.pack('>I', self.npd_header.license) + app_type = struct.pack('>I', self.npd_header.app_type) + dev[0x4:0x8] = version + dev[0x8:0xC] = license + dev[0xC:0x10] = app_type + + # Check for an empty dev_hash (can't validate if devklic is NULL) + klic = dec_key.to_bytes(length=16, byteorder="little") + + # Generate klicensee xor key + key = bytes(a ^ b for a, b in zip(klic, self.NP_OMAC_KEY_2)) + + # Hash with generated key and compare with dev_hash. + key_bytes = key + c = CMAC.new(key_bytes, ciphermod=AES) + c.update(dev[:0x60]) + generated_hash = c.digest() + + # Hash with generated key and compare with dev_hash + return generated_hash == self.npd_header.dev_hash + + @staticmethod + def dec_section(metadata: bytes): + dec = bytearray(16) + dec[0x00] = metadata[0xC] ^ metadata[0x8] ^ metadata[0x10] + dec[0x01] = metadata[0xD] ^ metadata[0x9] ^ metadata[0x11] + dec[0x02] = metadata[0xE] ^ metadata[0xA] ^ metadata[0x12] + dec[0x03] = metadata[0xF] ^ metadata[0xB] ^ metadata[0x13] + dec[0x04] = metadata[0x4] ^ metadata[0x8] ^ metadata[0x14] + dec[0x05] = metadata[0x5] ^ metadata[0x9] ^ metadata[0x15] + dec[0x06] = metadata[0x6] ^ metadata[0xA] ^ metadata[0x16] + dec[0x07] = metadata[0x7] ^ metadata[0xB] ^ metadata[0x17] + dec[0x08] = metadata[0xC] ^ metadata[0x0] ^ metadata[0x18] + dec[0x09] = metadata[0xD] ^ metadata[0x1] ^ metadata[0x19] + dec[0x0A] = metadata[0xE] ^ metadata[0x2] ^ metadata[0x1A] + dec[0x0B] = metadata[0xF] ^ metadata[0x3] ^ metadata[0x1B] + dec[0x0C] = metadata[0x4] ^ metadata[0x0] ^ metadata[0x1C] + dec[0x0D] = metadata[0x5] ^ metadata[0x1] ^ metadata[0x1D] + dec[0x0E] = metadata[0x6] ^ metadata[0x2] ^ metadata[0x1E] + dec[0x0F] = metadata[0x7] ^ metadata[0x3] ^ metadata[0x1F] + + # Convert the 'dec' array to the appropriate types + offset = struct.unpack('>Q', dec[0:8])[0] + length = struct.unpack('>i', dec[8:12])[0] + compression_end = struct.unpack('>i', dec[12:16])[0] + + return offset, length, compression_end + + def decrypt_block(self, block_num, decrypt_key): + if (self.edat_header.flags & self.EDAT_COMPRESSED_FLAG != 0) or (self.edat_header.flags & self.EDAT_FLAG_0x20 != 0): + metadata_section_size = 0x20 + else: + metadata_section_size = 0x10 + metadata_offset = 0x100 + + # Initialize buffers + hash_ = bytearray(0x10) + hash_result = bytearray(0x14) + empty_iv = bytes(0x10) + + compression_end = None + + # Decrypt the metadata + if self.edat_header.flags & self.EDAT_COMPRESSED_FLAG: + metadata_sec_offset = metadata_offset + block_num * metadata_section_size + self.fp.seek(metadata_sec_offset) + metadata = self.fp.read(0x20) + + # If the data is compressed, decrypt the metadata. + # NOTE: For NPD version 1 the metadata is not encrypted. + if self.npd_header.version <= 1: + offset, length, compression_end = struct.unpack('>Qii', metadata[0x10:0x20]) + else: + offset, length, compression_end = self.dec_section(metadata) + + hash_result[:0x10] = metadata[:0x10] + elif self.edat_header.flags & self.EDAT_FLAG_0x20: + # If FLAG 0x20, the metadata precedes each data block. + metadata_sec_offset = metadata_offset + block_num * (metadata_section_size + self.edat_header.block_size) + self.fp.seek(metadata_sec_offset) + metadata = self.fp.read(0x20) + hash_result[:] = metadata[:0x14] + + # Apply custom XOR if FLAG 0x20 is set + for j in range(0x10): + hash_result[j] ^= metadata[j + 0x10] + + offset = metadata_sec_offset + 0x20 + length = self.edat_header.block_size + + if block_num == self.total_blocks - 1 and self.edat_header.file_size % self.edat_header.block_size: + length = self.edat_header.file_size % self.edat_header.block_size + + else: + metadata_sec_offset = metadata_offset + block_num * metadata_section_size + self.fp.seek(metadata_sec_offset) + hash_result[:0x10] = self.fp.read(0x10) + + offset = metadata_offset + block_num * self.edat_header.block_size + self.total_blocks * metadata_section_size + length = self.edat_header.block_size + + if block_num == self.total_blocks - 1 and self.edat_header.file_size % self.edat_header.block_size: + length = self.edat_header.file_size % self.edat_header.block_size + + pad_length = length + length = (length + 0x10 - 1) // 0x10 * 0x10 + + # Prepare decryption buffers + self.fp.seek(offset) + enc_data = self.fp.read(length) + dec_data = io.BytesIO() + + b_key = self.get_block_key(block_num) + cipher = AES.new(decrypt_key, AES.MODE_ECB) + key_result = cipher.encrypt(b_key) + + if self.edat_header.flags & self.EDAT_FLAG_0x10: + hash_[:0x10] = cipher.encrypt(key_result) + else: + hash_[:0x10] = key_result + + crypto_mode = 0x2 if not self.edat_header.flags & self.EDAT_FLAG_0x02 else 0x1 + hash_mode = 0x02 if not self.edat_header.flags & self.EDAT_FLAG_0x10 else ( + 0x04 if not self.edat_header.flags & self.EDAT_FLAG_0x20 else 0x01) + + if self.edat_header.flags & self.EDAT_ENCRYPTED_KEY_FLAG: + crypto_mode |= 0x10000000 + hash_mode |= 0x10000000 + + should_decompress = self.edat_header.flags & self.EDAT_COMPRESSED_FLAG and compression_end + + if self.edat_header.flags & self.EDAT_DEBUG_DATA_FLAG: + # Already decrypted, just copy the data + dec_data = io.BytesIO(enc_data if not should_decompress else bytearray(enc_data)) + else: + # Setup IV and perform decryption + iv = empty_iv if self.npd_header.version <= 1 else self.npd_header.digest + if not self.decrypt( + hash_mode, + crypto_mode, + self.npd_header.version == 4, + enc_data, + dec_data, + length, + key_result, + iv, + hash_, + hash_result + ): + return -1 + + # Handle decompression if needed + size_left = self.edat_header.file_size + if should_decompress: + res = decompress(dec_data, self.edat_header.block_size) + size_left -= res + + if size_left == 0 and res < 0: + return -1 + + return res + + # Copy the decrypted data to output buffer if necessary + return dec_data.getvalue() + + def get_block_key(self, block): + empty_key = bytes(0x10) + src_key = empty_key if self.npd_header.version <= 1 else self.npd_header.dev_hash + dest_key = bytearray(0x10) + + # Copy the first 12 bytes from src_key to dest_key + dest_key[:0xC] = src_key[:0xC] + + # Convert block number to big-endian and store it in the last 4 bytes of dest_key + swapped_block = struct.pack('>i', block) + dest_key[0xC:0x10] = swapped_block + + return dest_key + + def decrypt(self, hash_mode, crypto_mode, version, in_data, out_data, length, key, iv, enc_hash, test_hash): + # Setup buffers for key, iv, and hash. + key_final = bytearray(0x10) + iv_final = bytearray(0x10) + + # Generate the crypto key + mode = crypto_mode & 0xF0000000 + if mode == 0x10000000: + # Encrypted ERK + temp_iv = bytearray(self.EDAT_IV) + cipher = AES.new(self.EDAT_KEY_1 if version else self.EDAT_KEY_0, AES.MODE_CBC, temp_iv) + key_final[:] = cipher.decrypt(key) + iv_final[:] = iv + elif mode == 0x20000000: + # Default ERK + key_final[:] = self.EDAT_KEY_1 if version else self.EDAT_KEY_0 + iv_final[:] = self.EDAT_IV + elif mode == 0x00000000: + # Unencrypted ERK + key_final[:] = key + iv_final[:] = iv + + # Generate the hash + mode = hash_mode & 0xF0000000 + hash_len = 0x14 if hash_mode & 0xFF == 0x01 else 0x10 + hash_final = bytearray(hash_len) + if mode == 0x10000000: + # Encrypted HASH + temp_iv = bytearray(self.EDAT_IV) + cipher = AES.new(self.EDAT_KEY_1 if version else self.EDAT_KEY_0, AES.MODE_CBC, temp_iv) + hash_final[:] = cipher.decrypt(enc_hash) + elif mode == 0x20000000: + # Default HASH + hash_final[:] = self.EDAT_HASH_1 if version else self.EDAT_HASH_0 + elif mode == 0x00000000: + # Unencrypted HASH + hash_final[:] = enc_hash + + # Perform decryption or copying + if (crypto_mode & 0xFF) == 0x01: + out_data.write(in_data[:length]) + elif (crypto_mode & 0xFF) == 0x02: + cipher = AES.new(key_final, AES.MODE_CBC, iv_final) + out_data.write(cipher.decrypt(in_data[:length])) + else: + print("Unknown crypto algorithm!") + return False + + # Verify the hash + if (hash_mode & 0xFF) == 0x01: # 0x14 SHA1-HMAC + h = HMAC.new(hash_final, digestmod=SHA1) + h.update(in_data) + return h.digest() == test_hash[:hash_len] + elif (hash_mode & 0xFF) == 0x02: # 0x10 AES-CMAC + c = CMAC.new(hash_final, ciphermod=AES) + c.update(in_data) + return c.digest() == test_hash[:hash_len] + elif (hash_mode & 0xFF) == 0x04: # 0x10 SHA1-HMAC + h = HMAC.new(hash_final, digestmod=SHA1) + h.update(in_data) + return h.digest() == test_hash[:hash_len] + else: + print("Unknown hashing algorithm!") + return False + + def readinto(self, b): + # Determine how many bytes to read + bytes_to_read = len(b) + if self._pos + bytes_to_read > self._size: + bytes_to_read = self._size - self._pos + + if bytes_to_read <= 0: + return 0 + + # Read the data + data = self.read(bytes_to_read) + + # Copy the data into the provided buffer + b[:len(data)] = data + + return len(data) + + def read(self, size=-1): + if size < 0: + size = self._size - self._pos + + data = bytearray() + bytes_read = 0 + while bytes_read < size and self._pos < self._size: + block_num = self._pos // self.edat_header.block_size + block_offset = self._pos % self.edat_header.block_size + block_data = self.decrypt_block(block_num, self.edat_key) + + if block_data == -1: + break + + remaining_in_block = min(len(block_data) - block_offset, self._size - self._pos) + remaining_to_read = min(size - bytes_read, remaining_in_block) + + chunk = block_data[block_offset:block_offset + remaining_to_read] + data.extend(chunk) + self._pos += len(chunk) + bytes_read += len(chunk) + + if len(chunk) < remaining_to_read: + break + + return bytes(data) + + def seek(self, offset, whence=io.SEEK_SET): + if whence == io.SEEK_SET: + self._pos = offset + elif whence == io.SEEK_CUR: + self._pos += offset + elif whence == io.SEEK_END: + self._pos = self._size + offset + self._pos = max(0, min(self._pos, self._size)) + return self._pos + + def tell(self): + return self._pos + + def readable(self): + return True + + def writable(self): + return False + + def seekable(self): + return True diff --git a/ps3/path_reader.py b/ps3/path_reader.py index 1376230..aa987b9 100644 --- a/ps3/path_reader.py +++ b/ps3/path_reader.py @@ -1,70 +1,44 @@ import collections import functools -import io import logging -import math -import os import pathlib import sqlite3 import struct from Crypto.Cipher import AES -from pycdlib.pycdlibio import PyCdlibIO from common.iso_path_reader.methods.pycdlib import PyCdLibPathReader +from utils.pycdlib.decrypted_file_io import DecryptedFileIO LOGGER = logging.getLogger(__name__) -class DecryptedFileReader(PyCdlibIO): +class DecryptedFileReader(DecryptedFileIO): def __init__(self, ino, logical_block_size, disc_key): super().__init__(ino, logical_block_size) self.disc_key = disc_key - def read(self, size=None): - if self._offset >= self._length: - return super().read(size) - - read_size = min(size or (self._length - self._offset), self._length - self._offset) - orig_pos = self._fp.tell() - orig_offset = self._offset - read_offset = self._fp.tell() % 2048 - if read_offset != 0: - self._offset -= read_offset - self._fp.seek(-read_offset, os.SEEK_CUR) - num_chunks = math.ceil((read_size + read_offset) / 2048) - if read_offset + read_size > 2048 * num_chunks: - num_chunks += 1 - decrypted = io.BytesIO() - bytes_read = 0 - for chunk in range(num_chunks): - block = super().read(min(size, 2048)) - if not block: - break - bytes_read += len(block) - # Read the remainder of the sector - if len(block) != 2048: - block += self._fp.read(2048 - len(block)) - decrypted.write(self.decrypt_block(block, self.disc_key)) + def decrypt_blocks(self, blocks): + if not self.disc_key: + return blocks + + decrypted = bytearray() + start_pos = (self._fp.tell() // 2048) - (len(blocks) // 2048) - self._offset = orig_offset + read_size - self._fp.seek(orig_pos + read_size) - decrypted.seek(read_offset) - return decrypted.read(read_size) + for i in range(0, len(blocks), 2048): + block = blocks[i:i+2048] + pos = start_pos + (i // 2048) + iv = bytearray(b'\x00' * 16) - def decrypt_block(self, block, disc_key): - if not disc_key: - return block + for j in range(16): + iv[16 - j - 1] = (pos & 0xFF) + pos >>= 8 - pos = (self._fp.tell() // 2048) - 1 - iv = bytearray(b'\x00' * 16) + cipher = AES.new(self.disc_key, AES.MODE_CBC, bytes(iv)) + decrypted.extend(cipher.decrypt(block)) - for j in range(16): - iv[16 - j - 1] = (pos & 0xFF) - pos >>= 8 + return bytes(decrypted) - cipher = AES.new(disc_key, AES.MODE_CBC, bytes(iv)) - return cipher.decrypt(block) class Ps3PathReader(PyCdLibPathReader): def __init__(self, iso, fp, *args, **kwargs): @@ -136,14 +110,16 @@ def get_disc_key(self): else: continue - block_dec = f.decrypt_block(block,key) + f.disc_key = key + block_dec = f.decrypt_blocks(block) if block_dec[:7] in expected_magic: LOGGER.info("Found key: %s", key.hex()) return key # Check if the disc decrypted with a debug key debug_key = '67c0758cf4996fef7e88f90cc6959d66' - block_dec = f.decrypt_block(block, bytes.fromhex(debug_key)) + f.disc_key = bytes.fromhex(debug_key) + block_dec = f.decrypt_blocks(block) if block_dec[:7] in expected_magic: LOGGER.info("Found key: %s", debug_key) return bytes.fromhex(debug_key) @@ -158,7 +134,8 @@ def get_disc_key(self): c = db.cursor() keys = c.execute('SELECT * FROM keys WHERE size = ?', [str(self.fp.length())]).fetchall() for key in keys: - block_dec = f.decrypt_block(block, key[-1]) + f.disc_key = key[-1] + block_dec = f.decrypt_blocks(block) if block_dec[:7] in expected_magic: LOGGER.info("Found key: %s", key[-1].hex()) return key[-1] diff --git a/utils/archives.py b/utils/archives.py index 4e834fe..63ba0b3 100644 --- a/utils/archives.py +++ b/utils/archives.py @@ -196,6 +196,7 @@ def iter(self, skip_entries=False): return entry = None + last_entry = next(reversed(self.entries), None) for entry in self.reader: if isinstance(entry, (rarfile.RarInfo, zipfile.ZipInfo)): file_path = entry.filename @@ -215,7 +216,11 @@ def iter(self, skip_entries=False): if isinstance(file_path, bytes): file_path = file_path.decode(errors='replace') - entry_fp = OffsetFile(self.uncompressed, self.uncompressed.tell(), self.uncompressed.tell() + file_size, file_path) + if last_entry: + entry_fp = OffsetFile(self.uncompressed, self.entries[last_entry].end_pos, + self.entries[last_entry].end_pos + file_size, file_path) + else: + entry_fp = OffsetFile(self.uncompressed, 0, file_size, file_path) entry_wrapper = ArchiveEntryWrapper(self, entry, entry_fp, self.reader, pbar=self.counter) self.entries[file_path] = entry_wrapper self.counter.update(incr=0, file_name=format_bar_desc(entry_wrapper.file_name, 30)) @@ -227,6 +232,7 @@ def iter(self, skip_entries=False): raise self.entries[file_path] = CompletedEntryWrapper(entry_wrapper) del entry_wrapper + last_entry = file_path self.ctx.__exit__(None, None, None) self.ctx = None entry = None diff --git a/utils/files.py b/utils/files.py index 34a35ef..1d7098c 100644 --- a/utils/files.py +++ b/utils/files.py @@ -502,57 +502,53 @@ def __init__(self, mmap, offset, end_pos, file_name=None): self.end_pos = end_pos self.mmap = mmap self.name = file_name or self.mmap.name + self._length = self.end_pos - self.offset # Pre-calculate length def seek(self, pos, whence=os.SEEK_SET): if whence == os.SEEK_CUR: - pos = self.pos - self.offset + pos + pos += self.pos - self.offset elif whence == os.SEEK_END: - pos = len(self) + pos - if pos >= self.length(): - pos = self.length() - return super().seek(pos+self.offset, os.SEEK_SET) + pos += self._length + pos = max(0, min(pos, self._length)) + return super().seek(pos + self.offset, os.SEEK_SET) def tell(self): return super().tell() - self.offset def read(self, n=None): - if n: - ret = self[self.tell():self.tell()+n] - else: - ret = self[self.tell():self.end_pos-self.offset] - + current_pos = self.tell() + if n is None: + n = self._length - current_pos + if current_pos == self._length: + return b"" + n = min(n, self._length - current_pos) + ret = self.mmap[current_pos + self.offset:current_pos + self.offset + n] self.pos += len(ret) return ret def __getitem__(self, item): if isinstance(item, slice): - read_pos = item.start - read_len = item.stop - item.start + start = item.start or 0 + stop = item.stop or self._length + start = max(0, min(start, self._length)) + stop = max(0, min(stop, self._length)) + return self.mmap[start + self.offset:stop + self.offset] else: - read_pos = item - read_len = 1 - self.seek(read_pos) - file_pos = super().tell() - if file_pos + read_len > self.end_pos: - read_len = self.end_pos - file_pos - if self.tell() == self.length(): - return b'' - return self.mmap[file_pos:file_pos+read_len] + if 0 <= item < self._length: + return self.mmap[item + self.offset] + raise IndexError("OffsetFile index out of range") def write(self, data): - try: - self.mmap.seek(self.pos) - ret = self.mmap.write(data) - self.pos += len(data) - return ret - except ValueError: - raise + write_len = min(len(data), self._length - (self.pos - self.offset)) + if write_len > 0: + self.mmap[self.tell() + self.offset:self.tell() + self.offset + write_len] = data[:write_len] + self.pos += write_len + return write_len def length(self): - return self.end_pos - self.offset + return self._length - def __len__(self): - return self.length() + __len__ = length def close(self): pass diff --git a/utils/in_memory_rollover_temp_file.py b/utils/in_memory_rollover_temp_file.py index 0179cc6..139c793 100644 --- a/utils/in_memory_rollover_temp_file.py +++ b/utils/in_memory_rollover_temp_file.py @@ -67,6 +67,17 @@ def __getitem__(self, item): self.seek(read_pos) return self._get_data(read_len) + def __setitem__(self, item, value): + if isinstance(item, slice): + write_pos = item.start + else: + write_pos = item + + original_pos = self.tell() + self.seek(write_pos) + self.write(value) + self.seek(original_pos) + def close(self): try: if self.mmap: diff --git a/common/pycdlib.py b/utils/pycdlib/__init__.py similarity index 99% rename from common/pycdlib.py rename to utils/pycdlib/__init__.py index 8c68aa7..7e841a9 100644 --- a/common/pycdlib.py +++ b/utils/pycdlib/__init__.py @@ -18,6 +18,7 @@ _logger = logging.getLogger(__name__) + class PyCdlib(_PyCdlib): is_hs = False @@ -350,6 +351,7 @@ def patched_dr_data_length(*args, **kwargs): def _get_iso_size(self): return float("inf") + # Volume descriptor parsers with HS filesystem support class PrimaryOrSupplementaryVD(_PrimaryOrSupplementaryVD): FMT_HS = '= self._length: + return b'' + if size is None: + size = self._length - offset + + read_size = min(size, self._length - offset) + result = bytearray(read_size) + bytes_read = 0 + + while bytes_read < read_size: + if self._buffer_offset >= self._buffer_filled: + self._fill_buffer(read_size - bytes_read) + if self._buffer_filled == 0: + break + + chunk_size = min(self._buffer_filled - self._buffer_offset, read_size - bytes_read) + result[bytes_read:bytes_read + chunk_size] = self._buffer[ + self._buffer_offset:self._buffer_offset + chunk_size] + + self._buffer_offset += chunk_size + bytes_read += chunk_size + return bytes(result) + + def _fill_buffer(self, desired_size): + # Align to block boundary + block_offset = self._offset % self.logical_block_size + aligned_offset = self._offset - block_offset + + if aligned_offset != self._offset: + super().seek(aligned_offset) + + # Calculate how many full blocks to read + blocks_to_read = min( + math.ceil((desired_size + block_offset) / self.logical_block_size), + self.buffer_size // self.logical_block_size + ) + + data = super().read(blocks_to_read * self.logical_block_size) + if not data: + self._buffer_filled = 0 + return + + if len(data) < self.logical_block_size * blocks_to_read: + read_extra = (self.logical_block_size * blocks_to_read) - len(data) + data += self._fp.read(read_extra) + self._offset += read_extra + + # Decrypt all blocks at once + decrypted_data = self.decrypt_blocks(data) + + self._buffer[:len(decrypted_data)] = decrypted_data + self._buffer_filled = len(decrypted_data) + self._buffer_offset = block_offset + + def seek(self, offset, whence=0): + new_offset = super().seek(offset, whence) + self._buffer_offset = 0 + self._buffer_size = 0 + self._buffer_filled = 0 + return new_offset + + def tell(self): + return super().tell() - self._buffer_filled + self._buffer_offset + + def decrypt_blocks(self, blocks): + raise NotImplementedError + + def decrypt_block(self, block): + raise NotImplementedError From 24e16d369b079d8d35a3fc7f354052bf2109700e Mon Sep 17 00:00:00 2001 From: Sazpaimon Date: Thu, 26 Dec 2024 23:37:53 -0500 Subject: [PATCH 2/8] PSX: Detect system type for NPDRM packages --- common/processor.py | 9 +++++- post_psx/npdrm/path_reader.py | 54 +++++++++++++++++++++++--------- post_psx/npdrm/pkg.py | 59 ++++++++++++++++++++++------------- post_psx/types.py | 34 +++++++++++++++++++- ps3/processor.py | 55 ++++++++++++++++++++++++-------- ps3/self_parser.py | 37 +++++----------------- 6 files changed, 168 insertions(+), 80 deletions(-) diff --git a/common/processor.py b/common/processor.py index ef8e0d8..0164480 100644 --- a/common/processor.py +++ b/common/processor.py @@ -10,12 +10,14 @@ from common.iso_path_reader.methods.compressed import CompressedPathReader from common.iso_path_reader.methods.pathlab import PathlabPathReader from common.iso_path_reader.methods.pycdlib import PyCdLibPathReader +from post_psx.npdrm.path_reader import NPDRMPathReader from utils.common import format_bar_desc from utils.hash_progress_wrapper import HashProgressWrapper from xbox.path_reader import XboxPathReader, XboxStfsPathReader LOGGER = logging.getLogger(__name__) + class BaseIsoProcessor: globally_ignored_paths = [ re.compile(".*\.nfo$", re.IGNORECASE), @@ -41,6 +43,11 @@ def get_system_type(iso_path_reader): if isinstance(iso_path_reader, XboxStfsPathReader): return "xbla" + if isinstance(iso_path_reader, NPDRMPathReader): + system_type = iso_path_reader.system_type + if system_type: + return system_type + if not isinstance(iso_path_reader, CompressedPathReader): fp = iso_path_reader.fp fp.seek(0) @@ -149,7 +156,6 @@ def get_system_type(iso_path_reader): if f.read(2) == b"MZ": return "pc" - def __init__(self, iso_path_reader, filename, system_type, progress_manager): self.iso_path_reader = iso_path_reader self.filename = filename @@ -313,6 +319,7 @@ def get_pvd_info(self): def close(self): self.iso_path_reader.close() + class GenericIsoProcessor(BaseIsoProcessor): def hash_exe(self): return {} diff --git a/post_psx/npdrm/path_reader.py b/post_psx/npdrm/path_reader.py index a33fd28..62bd01b 100644 --- a/post_psx/npdrm/path_reader.py +++ b/post_psx/npdrm/path_reader.py @@ -3,17 +3,14 @@ import os import pathlib -from Crypto.Cipher import AES -from Crypto.Util import Counter - from common.iso_path_reader.methods.base import IsoPathReader from common.iso_path_reader.methods.chunked_hash_trait import ChunkedHashTrait +from post_psx.npdrm.pkg import Pkg from post_psx.utils.edat import EdatFile from utils import pycdlib from utils.files import OffsetFile from utils.pycdlib.decrypted_file_io import DecryptedFileIO - LOGGER = logging.getLogger(__name__) @@ -37,7 +34,7 @@ def decrypt_blocks(self, blocks): class NPDRMPathReader(ChunkedHashTrait, IsoPathReader): volume_type = "npdrm" - def __init__(self, iso, fp, parent_container): + def __init__(self, iso: Pkg, fp, parent_container): super().__init__(iso, fp, parent_container) pkg_header = self.iso.pkg_header self.fp = OffsetFile(self.fp, pkg_header.data_offset, pkg_header.data_offset + pkg_header.data_size) @@ -79,20 +76,28 @@ def test_key(self, key, file): return f.decrypt_block(0, key) != -1 def get_root_dir(self): - return None + return next(self.iso.entries()).directory def iso_iterator(self, base_dir, recursive=False, include_dirs=False): - # always recursive - for file in self.iso.entries(): - if self.is_directory(file) and not include_dirs: + for entry in base_dir.entries: + if self.is_directory(entry) and not include_dirs: continue - yield file + yield entry + + if not recursive: + return + for dir in base_dir.directories.values(): + yield from self.iso_iterator(dir, recursive) + def get_file(self, path): - try: - return next(file for file in self.iso.entries() if file.name_decoded == path) - except StopIteration: - raise FileNotFoundError + normalized_path = path.strip('/') + + for file in self.iso_iterator(self.get_root_dir(), recursive=True): + if file.name_decoded == normalized_path: + return file + + raise FileNotFoundError() def get_file_date(self, file): return None @@ -143,3 +148,24 @@ def get_pvd(self): def get_pvd_info(self): return {} + + @property + def system_type(self): + if self.iso.pkg_header.pkg_platform == self.iso.PKG_PLATFORM_TYPE_PS3: + return "ps3" + + title_id = self.iso.pkg_header.title_id + psp_title_ids = [b"UL", b"UC", b"G", b"H", b"W", b"Y"] + ps3_title_ids = [b"BL", b"BC"] + if title_id[0:2] in ps3_title_ids: + return "ps3" + elif any(title_id.startswith(tid) for tid in psp_title_ids): + return "psp" + elif title_id[0:3] == b"PCS": + return "vita" + + if self.iso.pkg_ext_header: + if self.iso.pkg_ext_header.pkg_key_id == 0x1: + return "psp" + elif self.iso.pkg_ext_header.pkg_key_id == 0xC0000002: + return "vita" diff --git a/post_psx/npdrm/pkg.py b/post_psx/npdrm/pkg.py index 55ea358..80253ae 100644 --- a/post_psx/npdrm/pkg.py +++ b/post_psx/npdrm/pkg.py @@ -6,7 +6,7 @@ from Crypto.Util import Counter, strxor from Crypto.Util.number import long_to_bytes -from post_psx.types import PKGMetaData, PKGHeader, PKGExtHeader, PKGEntry +from post_psx.types import PKGMetaData, PKGHeader, PKGExtHeader, PKGEntry, PKGDirectory from utils.files import get_file_size from xor_cipher import cyclic_xor @@ -111,6 +111,7 @@ def parse_header(self): LOGGER.error("PKG type not supported") return False + self.pkg_ext_header = None if self.pkg_header.pkg_platform == self.PKG_PLATFORM_TYPE_PSP_PSVITA: # Parse extended header for PSP/Vita packages self.fp.seek(PKGHeader.struct.size) @@ -218,27 +219,41 @@ def init_cipher(self, block_num, key): self.aes_context = DebugCTR(self.pkg_header.digest, block_num) def entries(self): - if len(self._entries) == self.pkg_header.file_count: - yield from iter(self._entries.values()) - - for block in range(0, self.pkg_header.file_count * 2, 2): - self.init_cipher(block, self.pkg_key) - self.fp.seek(self.pkg_header.data_offset + (16 * block)) - enc = self.fp.read(32) - entry_data = self.decrypt(enc) - entry = PKGEntry.unpack(entry_data) - - name_block_start = entry.name_offset // 16 - self.fp.seek(self.pkg_header.data_offset + (16 * name_block_start)) - if entry.type & self.PKG_FILE_ENTRY_PSP: - entry.key = self.PSP_AES_KEY - else: - entry.key = self.PS3_AES_KEY - self.init_cipher(name_block_start, entry.key) - name = self.decrypt(self.fp.read(entry.name_size)) - entry.name_decoded = name.rstrip(b"\x00").decode("utf-8", errors="replace") - self._entries[entry.name_decoded] = entry - yield entry + if not self._entries: + self._root = PKGDirectory(None) + + for block in range(0, self.pkg_header.file_count * 2, 2): + self.init_cipher(block, self.pkg_key) + self.fp.seek(self.pkg_header.data_offset + (16 * block)) + enc = self.fp.read(32) + entry_data = self.decrypt(enc) + entry = PKGEntry.unpack(entry_data) + + name_block_start = entry.name_offset // 16 + self.fp.seek(self.pkg_header.data_offset + (16 * name_block_start)) + if entry.type & self.PKG_FILE_ENTRY_PSP: + entry.key = self.PSP_AES_KEY + else: + entry.key = self.PS3_AES_KEY + self.init_cipher(name_block_start, entry.key) + name = self.decrypt(self.fp.read(entry.name_size)) + entry.name_decoded = name.rstrip(b"\x00").decode("utf-8", errors="replace") + + path_parts = entry.name_decoded.split('/') + current_dir = self._root + for part in path_parts[:-1]: + if not hasattr(current_dir, 'directories'): + current_dir.directories = {} + if part not in current_dir.directories: + current_dir.directories[part] = PKGDirectory(entry) + current_dir = current_dir.directories[part] + + if not hasattr(current_dir, 'entries'): + current_dir.entries = [] + current_dir.entries.append(entry) + entry.directory = current_dir + + yield from self._root.entries def decrypt(self, blocks): return self.aes_context.encrypt(blocks) diff --git a/post_psx/types.py b/post_psx/types.py index a576e31..b4c10a4 100644 --- a/post_psx/types.py +++ b/post_psx/types.py @@ -1,9 +1,30 @@ +import dataclasses import struct from dataclasses import dataclass, field from typing import List -from ps3.self_parser import Struct + +class StructMeta(type): + def __init__(cls, name, bases, d): + if dataclasses.is_dataclass(d): + raise ValueError("Class {} is not a dataclass".format(name)) + if 'struct' not in d: + raise ValueError("Class {} doesn't define struct".format(name)) + type.__init__(cls, name, bases, d) + + +class Struct: + __metaclass__ = StructMeta + struct = None + size = 0 + + def pack(self): + return self.struct.pack(*dataclasses.astuple(self)) + + @classmethod + def unpack(cls, data): + return cls(*cls.struct.unpack(data)) @dataclass @@ -49,6 +70,16 @@ def check_magic(self): return self.magic == b"\x7Fext" +class PKGDirectory: + def __init__(self, entry): + self.entry = entry + self.entries = [] + self.directories = {} + + def entries(self): + yield from self.entries + + @dataclass class PKGEntry(Struct): name_offset: int # File name offset @@ -59,6 +90,7 @@ class PKGEntry(Struct): pad: int # Padding (zeros) name_decoded: str = "" key: bytes = b"" + directory: PKGDirectory = None struct = struct.Struct(">IIQQII") diff --git a/ps3/processor.py b/ps3/processor.py index 548c85b..7846aa0 100644 --- a/ps3/processor.py +++ b/ps3/processor.py @@ -4,6 +4,7 @@ import xxhash +from post_psx.npdrm.path_reader import NPDRMPathReader from post_psx.processor import PostPsxIsoProcessor from ps3.self_parser import SELFDecrypter @@ -12,19 +13,37 @@ class Ps3IsoProcessor(PostPsxIsoProcessor): update_folder = re.compile(".*PS3_UPDATE$", re.IGNORECASE) - sfo_path = "/PS3_GAME/PARAM.SFO" + + def __init__(self, iso_path_reader, filename, system_type, progress_manager): + super().__init__(iso_path_reader, filename, system_type, progress_manager) + self.base_dir = "" + for file in self.iso_path_reader.iso_iterator(self.iso_path_reader.get_root_dir(), include_dirs=True): + if self.iso_path_reader.is_directory(file): + if self.iso_path_reader.get_file_path(file).strip("/") == "PS3_GAME": + self.base_dir = "/PS3_GAME" + break + + @property + def sfo_path(self): + return f"{self.base_dir}/PARAM.SFO" @property def ignored_paths(self): paths = self.exe_patterns + [self.update_folder] - paths.append(re.compile("(?!^\/PS3_GAME\/USRDIR\/)", re.IGNORECASE)) + paths.append(re.compile(f"(?!^{self.base_dir}/?USRDIR/)", re.IGNORECASE)) return paths def get_disc_type(self): + if isinstance(self.iso_path_reader, NPDRMPathReader): + return {"disc_type": "hdd"} return {"disc_type": "bd-r"} def get_exe_filename(self): - return "/PS3_GAME/USRDIR/EBOOT.BIN" + try: + self.iso_path_reader.get_file(f"{self.base_dir}/USRDIR/EBOOT.BIN") + return f"{self.base_dir}/USRDIR/EBOOT.BIN" + except FileNotFoundError: + return None def get_file_hashes(self, hash_type=xxhash.xxh64): file_hashes, alt_file_hashes, incomplete_files = super().get_file_hashes(hash_type) @@ -59,13 +78,23 @@ def _parse_exe(self, filename): return result def get_extra_fields(self): - params = self.parse_param_sfo() - fields = { - "sfo_category": params.get("CATEGORY"), - "sfo_disc_id": params.get("TITLE_ID"), - "sfo_disc_version": params.get("DISC_VERSION"), - "sfo_parental_level": params.get("PARENTAL_LEVEL"), - "sfo_psp_system_version": params.get("PS3_SYSTEM_VER"), - "sfo_title": params.get("TITLE"), - } - return {**fields, **self._parse_exe(self.get_exe_filename())} + fields = {} + try: + params = self.parse_param_sfo() + fields = { + "sfo_category": params.get("CATEGORY"), + "sfo_disc_id": params.get("TITLE_ID"), + "sfo_disc_version": params.get("DISC_VERSION"), + "sfo_parental_level": params.get("PARENTAL_LEVEL"), + "sfo_psp_system_version": params.get("PS3_SYSTEM_VER"), + "sfo_title": params.get("TITLE"), + } + except FileNotFoundError: + LOGGER.warning("No param.sfo found.") + + try: + if self.get_exe_filename(): + return {**fields, **self._parse_exe(self.get_exe_filename())} + except FileNotFoundError: + return fields + return fields diff --git a/ps3/self_parser.py b/ps3/self_parser.py index 78fcec2..4b57245 100644 --- a/ps3/self_parser.py +++ b/ps3/self_parser.py @@ -1,33 +1,12 @@ -import dataclasses import io import struct import zlib -from dataclasses import dataclass +from dataclasses import dataclass, field from Crypto.Cipher import AES from Crypto.Util import Counter - -class StructMeta(type): - def __init__(cls, name, bases, d): - if dataclasses.is_dataclass(d): - raise ValueError("Class {} is not a dataclass".format(name)) - if 'struct' not in d: - raise ValueError("Class {} doesn't define struct".format(name)) - type.__init__(cls, name, bases, d) - - -class Struct: - __metaclass__ = StructMeta - struct = None - size = 0 - - def pack(self): - return self.struct.pack(*dataclasses.astuple(self)) - - @classmethod - def unpack(cls, data): - return cls(*cls.struct.unpack(data)) +from post_psx.types import Struct @dataclass @@ -126,18 +105,18 @@ class SegmentCertHeader(Struct): @dataclass class Attributes(Struct): - key: bytearray = bytearray(0x10) - iv: bytearray = bytearray(0x10) + key: bytearray = field(default_factory=lambda: bytearray(0x10)) + iv: bytearray = field(default_factory=lambda: bytearray(0x10)) struct = struct.Struct(">16s16s") @dataclass class EncryptionRootHeader(Struct): - key: bytearray = bytearray(0x10) - key_pad: bytearray = bytearray(0x10) - iv: bytearray = bytearray(0x10) - iv_pad: bytearray = bytearray(0x10) + key: bytearray = field(default_factory=lambda: bytearray(0x10)) + key_pad: bytearray = field(default_factory=lambda: bytearray(0x10)) + iv: bytearray = field(default_factory=lambda: bytearray(0x10)) + iv_pad: bytearray = field(default_factory=lambda: bytearray(0x10)) struct = struct.Struct(">16s16s16s16s") From dd083b368a861299f27a476dac72be003cc2bf39 Mon Sep 17 00:00:00 2001 From: Sazpaimon Date: Sun, 23 Mar 2025 21:21:44 -0400 Subject: [PATCH 3/8] PSX: Decrypt eboot files in NPDRM packages --- post_psx/npdrm/path_reader.py | 36 ++++-- post_psx/utils/edat.py | 7 +- post_psx/utils/lz.py | 236 ++++++++++++++++++++++++++++++++++ ps3/processor.py | 7 +- ps3/self_parser.py | 200 ++++++++++++++++++++++++++-- 5 files changed, 456 insertions(+), 30 deletions(-) create mode 100644 post_psx/utils/lz.py diff --git a/post_psx/npdrm/path_reader.py b/post_psx/npdrm/path_reader.py index 62bd01b..835d89f 100644 --- a/post_psx/npdrm/path_reader.py +++ b/post_psx/npdrm/path_reader.py @@ -7,6 +7,7 @@ from common.iso_path_reader.methods.chunked_hash_trait import ChunkedHashTrait from post_psx.npdrm.pkg import Pkg from post_psx.utils.edat import EdatFile +from ps3.self_parser import SELFDecrypter from utils import pycdlib from utils.files import OffsetFile from utils.pycdlib.decrypted_file_io import DecryptedFileIO @@ -41,11 +42,6 @@ def __init__(self, iso: Pkg, fp, parent_container): self.edat_key = self.get_edat_key() def get_edat_key(self): - edat_file = next((file for file in self.iso_iterator(self.get_root_dir(), recursive=True) if - self.get_file_path(file).lower().endswith(".edat")), None) - if not edat_file: - return - edat_key = None file_dir = pathlib.Path(self.fp.name).parent try: @@ -59,13 +55,29 @@ def get_edat_key(self): except UnicodeDecodeError: pass - if edat_key: - if self.test_key(edat_key, edat_file): - return edat_key - else: - test_edat_key = bytes.fromhex("0" * 32) - if self.test_key(test_edat_key, edat_file): - return test_edat_key + edat_file = next((file for file in self.iso_iterator(self.get_root_dir(), recursive=True) if + self.get_file_path(file).lower().endswith(".edat")), None) + if edat_file: + if edat_key: + if self.test_key(edat_key, edat_file): + return edat_key + else: + test_edat_key = bytes.fromhex("0" * 32) + if self.test_key(test_edat_key, edat_file): + return test_edat_key + elif edat_key: + eboot_file = next((file for file in self.iso_iterator(self.get_root_dir(), recursive=True) if + self.get_file_path(file).lower().endswith("eboot.bin")), None) + if eboot_file: + inode = pycdlib.inode.Inode() + inode.parse(eboot_file.file_offset // 16, eboot_file.file_size, self.fp, 16) + with DecryptedFileReader(inode, 16, self.iso, eboot_file) as f: + decryptor = SELFDecrypter(f, edat_key) + decryptor.load_headers() + if decryptor.get_npd_header(): + decryptor.load_metadata() + if decryptor.data_keys: + return edat_key def test_key(self, key, file): file_path = self.get_file_path(file) diff --git a/post_psx/utils/edat.py b/post_psx/utils/edat.py index 5ea2318..88728f0 100644 --- a/post_psx/utils/edat.py +++ b/post_psx/utils/edat.py @@ -5,6 +5,7 @@ from Crypto.Hash import CMAC, SHA1, HMAC from post_psx.types import NPDHeader, EDATHeader +from post_psx.utils import lz class EdatFile(io.RawIOBase): @@ -316,12 +317,10 @@ def decrypt_block(self, block_num, decrypt_key): return -1 # Handle decompression if needed - size_left = self.edat_header.file_size if should_decompress: - res = decompress(dec_data, self.edat_header.block_size) - size_left -= res + res = lz.LZDecompressor(dec_data.getvalue()).decompress(self.edat_header.block_size) - if size_left == 0 and res < 0: + if not res: return -1 return res diff --git a/post_psx/utils/lz.py b/post_psx/utils/lz.py new file mode 100644 index 0000000..ca4a187 --- /dev/null +++ b/post_psx/utils/lz.py @@ -0,0 +1,236 @@ +from dataclasses import dataclass +from typing import Optional, Tuple + + +@dataclass +class LZDecompressor: + in_bytes: bytes + + def __init__(self, in_bytes: bytes): + self.in_bytes = in_bytes + self.src = 0 + self.range = 0xFFFFFFFF + + # Construct the 32-bit "code" from bytes 1..4 + self.code = int.from_bytes(in_bytes[1:5], byteorder='big') + + # Initialize temporary buffer with 0x80 bytes + self.tmp = bytearray([0x80] * 0xCA8 + [0] * (3272 - 0xCA8)) + + def decode_range(self) -> None: + """Updates range and code values based on input bytes.""" + if not (self.range >> 24): + self.range = (self.range << 8) & 0xFFFFFFFF + self.code = ((self.code << 8) & 0xFFFFFFFF) + self.in_bytes[self.src + 5] + self.src += 1 + + def decode_bit(self, acc: Optional[list], tmp_index: int) -> int: + """ + Decodes a single bit from the compressed data. + + Args: + acc: Optional accumulator list containing a single value + tmp_index: Index into temporary buffer + + Returns: + Decoded bit (0 or 1) + """ + self.decode_range() + + c = self.tmp[tmp_index] + self.tmp[tmp_index] = (c - (c >> 3)) & 0xFF + + if acc is not None: + acc[0] <<= 1 + + val = (self.range >> 8) * c + + if self.code < val: + self.range = val + self.tmp[tmp_index] = (self.tmp[tmp_index] + 31) & 0xFF + if acc is not None: + acc[0] += 1 + return 1 + else: + self.code -= val + self.range -= val + return 0 + + def decode_number(self, base_offset: int, index_val: int) -> Tuple[int, int]: + """ + Decodes a number from the compressed data. + + Args: + base_offset: Offset into temporary buffer + index_val: Index value from caller + + Returns: + Tuple of (decoded number, bit flag) + """ + acc = [1] + + # Handle higher index values + if index_val >= 3: + self.decode_bit(acc, base_offset + 0x18) + if index_val >= 4: + self.decode_bit(acc, base_offset + 0x18) + if index_val >= 5: + self.decode_range() + for _ in range(index_val - 4): + acc[0] <<= 1 + self.range >>= 1 + if self.code < self.range: + acc[0] += 1 + else: + self.code -= self.range + + bit_flag = self.decode_bit(acc, base_offset) + + # Handle lower index values + if index_val >= 1: + self.decode_bit(acc, base_offset + 0x8) + if index_val >= 2: + self.decode_bit(acc, base_offset + 0x10) + + return acc[0], bit_flag + + def decode_word(self, base_offset: int, index_val: int) -> Tuple[int, int]: + """ + Decodes a word from the compressed data. + + Args: + base_offset: Offset into temporary buffer + index_val: Index value divided by 8 + + Returns: + Tuple of (decoded word, bit flag) + """ + index_val //= 8 + acc = [1] + + if index_val >= 3: + self.decode_bit(acc, base_offset + 4) + if index_val >= 4: + self.decode_bit(acc, base_offset + 4) + if index_val >= 5: + self.decode_range() + for _ in range(index_val - 4): + acc[0] <<= 1 + self.range >>= 1 + if self.code < self.range: + acc[0] += 1 + else: + self.code -= self.range + + bit_flag = self.decode_bit(acc, base_offset) + + if index_val >= 1: + self.decode_bit(acc, base_offset + 1) + if index_val >= 2: + self.decode_bit(acc, base_offset + 2) + + return acc[0], bit_flag + + def decompress(self, out_size: int) -> Optional[bytes]: + """ + Decompresses the input data. + + Args: + out_size: Expected size of decompressed output + + Returns: + Decompressed data as bytes, or None if decompression fails + """ + out = bytearray(out_size) + pos = 0 + head = self.in_bytes[0] + prev = 0 + offset = 0 + + # Handle uncompressed data + if head > 0x80: + length = self.code + if length <= out_size: + out[:length] = self.in_bytes[5:5 + length] + return bytes(out) + return None + + while pos < out_size: + sect1_index = offset + 0xB68 + + if self.decode_bit(None, sect1_index) == 0: + # Handle raw character + if offset > 0: + offset -= 1 + if pos >= out_size: + return bytes(out[:pos]) + + sect = (((((pos & 7) << 8) + prev) >> head) & 7) * 0xFF - 1 + acc = [1] + + while not (acc[0] >> 8): + self.decode_bit(acc, sect + acc[0]) + + out[pos] = acc[0] & 0xFF + pos += 1 + + else: + # Handle compressed stream + index_val = -1 + sect1 = offset + 0xB68 + + while True: + sect1 += 8 + bit = self.decode_bit(None, sect1) + index_val += bit + if not bit or index_val >= 6: + break + + b_size = 0x160 + tmp_sect2 = index_val + 0x7F1 + + if index_val >= 0 or bit: + sect = (index_val << 5) | ((((pos << index_val) & 3) << 3) | (offset & 7)) + tmp_sect1 = 0xBA8 + sect + data_length, _ = self.decode_number(tmp_sect1, index_val) + + if data_length == 0xFF: + return bytes(out[:pos]) + else: + data_length = 1 + + if data_length <= 2: + tmp_sect2 += 0xF8 + b_size = 0x40 + + shift_val = [1] + + while True: + diff = (shift_val[0] << 4) - b_size + bit = self.decode_bit(shift_val, tmp_sect2 + (shift_val[0] << 3)) + if diff >= 0: + break + + if diff > 0 or bit: + if not bit: + diff -= 8 + tmp_sect3 = 0x928 + diff + data_offset, _ = self.decode_word(tmp_sect3, diff) + else: + data_offset = 1 + + buf_start = pos - data_offset + buf_end = pos + data_length + 1 + + if buf_start < 0 or buf_end > out_size: + return None + + offset = (((buf_end + 1) & 1) + 6) + + for i in range(data_length + 1): + out[pos] = out[buf_start + i] + pos += 1 + + prev = out[pos - 1] + + return bytes(out) diff --git a/ps3/processor.py b/ps3/processor.py index 7846aa0..fab9dfa 100644 --- a/ps3/processor.py +++ b/ps3/processor.py @@ -22,6 +22,9 @@ def __init__(self, iso_path_reader, filename, system_type, progress_manager): if self.iso_path_reader.get_file_path(file).strip("/") == "PS3_GAME": self.base_dir = "/PS3_GAME" break + self.eboot_key = None + if iso_path_reader.volume_type == 'npdrm': + self.eboot_key = iso_path_reader.edat_key @property def sfo_path(self): @@ -53,7 +56,7 @@ def get_file_hashes(self, hash_type=xxhash.xxh64): if any(regex.match(file_path) for regex in self.exe_patterns): with self.iso_path_reader.open_file(file) as f: - decryptor = SELFDecrypter(f) + decryptor = SELFDecrypter(f, self.eboot_key) decryptor.load_headers() decryptor.load_metadata() elf = decryptor.get_decrypted_elf() @@ -63,7 +66,7 @@ def get_file_hashes(self, hash_type=xxhash.xxh64): def _parse_exe(self, filename): result = {} with self.iso_path_reader.open_file(self.iso_path_reader.get_file(filename)) as f: - decryptor = SELFDecrypter(f) + decryptor = SELFDecrypter(f, self.eboot_key) decryptor.load_headers() result["exe_signing_type"] = "debug" if decryptor.sce_hdr.attribute & 0x8000 == 0x8000 else "retail" result["exe_num_symbols"] = 0 diff --git a/ps3/self_parser.py b/ps3/self_parser.py index 4b57245..733e26a 100644 --- a/ps3/self_parser.py +++ b/ps3/self_parser.py @@ -2,11 +2,12 @@ import struct import zlib from dataclasses import dataclass, field +from typing import Union, List, Optional from Crypto.Cipher import AES from Crypto.Util import Counter -from post_psx.types import Struct +from post_psx.types import Struct, NPDHeader @dataclass @@ -247,6 +248,56 @@ class ElfSectionHeader32(Struct): struct = struct.Struct(">IIIIIIIIII") +@dataclass +class PS3PlaintextCapabilityHeader(Struct): + ctrl_flag1: int = 0 + unknown1: int = 0 + unknown2: int = 0 + unknown3: int = 0 + unknown4: int = 0 + unknown5: int = 0 + unknown6: int = 0 + unknown7: int = 0 + struct = struct.Struct(">8I") # 8 uint32 + + +@dataclass +class PS3ElfDigestHeader40(Struct): + constant: bytes = b'\x00' * 0x14 + elf_digest: bytes = b'\x00' * 0x14 + required_system_version: int = 0 + struct = struct.Struct(">20s20sQ") # 20 bytes + 20 bytes + uint64 + + +@dataclass +class PS3ElfDigestHeader30(Struct): + constant_or_elf_digest: bytes = b'\x00' * 0x14 + padding: bytes = b'\x00' * 0xC + struct = struct.Struct(">20s12s") # 20 bytes + 12 bytes padding + + +@dataclass +class SupplementalHeader(Struct): + type: int = 0 # uint32 + size: int = 0 # uint32 + next: int = 0 # uint64 + struct = struct.Struct(">IIQ") + + def __post_init__(self): + # Based on type, create the appropriate subheader + self.subheader: Optional[Struct] = None + self.subheader_cls = None + if self.type == 1: + self.subheader_cls = PS3PlaintextCapabilityHeader + elif self.type == 2: + if self.size == 0x40: + self.subheader_cls = PS3ElfDigestHeader40 + elif self.size == 0x30: + self.subheader_cls = PS3ElfDigestHeader30 + elif self.type == 3: + self.subheader_cls = NPDHeader + + class SELFDecrypter: apploader_keys = { 0x0000: [ @@ -362,19 +413,91 @@ class SELFDecrypter: "ACB9945914EBB7B9A31ECE320AE09F2D", ], } + npdrm_keys = { + 0x0001: [ + "F9EDD0301F770FABBA8863D9897F0FEA6551B09431F61312654E28F43533EA6B", + "A551CCB4A42C37A734A2B4F9657D5540", + ], + 0x0002: [ + "8E737230C80E66AD0162EDDD32F1F774EE5E4E187449F19079437A508FCF9C86", + "7AAECC60AD12AED90C348D8C11D2BED5", + ], + 0x0003: [ + "1B715B0C3E8DC4C1A5772EBA9C5D34F7CCFE5B82025D453F3167566497239664", + "E31E206FBB8AEA27FAB0D9A2FFB6B62F", + ], + 0x0004: [ + "BB4DBF66B744A33934172D9F8379A7A5EA74CB0F559BB95D0E7AECE91702B706", + "ADF7B207A15AC601110E61DDFC210AF6", + ], + 0x0006: [ + "8B4C52849765D2B5FA3D5628AFB17644D52B9FFEE235B4C0DB72A62867EAA020", + "05719DF1B1D0306C03910ADDCE4AF887", + ], + 0x0007: [ + "3946DFAA141718C7BE339A0D6C26301C76B568AEBC5CD52652F2E2E0297437C3", + "E4897BE553AE025CDCBF2B15D1C9234E", + ], + 0x0009: [ + "0786F4B0CA5937F515BDCE188F569B2EF3109A4DA0780A7AA07BD89C3350810A", + "04AD3C2F122A3B35E804850CAD142C6D", + ], + 0x000A: [ + "03C21AD78FBB6A3D425E9AAB1298F9FD70E29FD4E6E3A3C151205DA50C413DE4", + "0A99D4D4F8301A88052D714AD2FB565E", + ], + 0x000C: [ + "357EBBEA265FAEC271182D571C6CD2F62CFA04D325588F213DB6B2E0ED166D92", + "D26E6DD2B74CD78E866E742E5571B84F", + ], + 0x000D: [ + "337A51416105B56E40D7CAF1B954CDAF4E7645F28379904F35F27E81CA7B6957", + "8405C88E042280DBD794EC7E22B74002", + ], + 0x000F: [ + "135C098CBE6A3E037EBE9F2BB9B30218DDE8D68217346F9AD33203352FBB3291", + "4070C898C2EAAD1634A288AA547A35A8", + ], + 0x0010: [ + "4B3CD10F6A6AA7D99F9B3A660C35ADE08EF01C2C336B9E46D1BB5678B4261A61", + "C0F2AB86E6E0457552DB50D7219371C5", + ], + 0x0013: [ + "265C93CF48562EC5D18773BEB7689B8AD10C5EB6D21421455DEBC4FB128CBF46", + "8DEA5FF959682A9B98B688CEA1EF4A1D", + ], + 0x0016: [ + "7910340483E419E55F0D33E4EA5410EEEC3AF47814667ECA2AA9D75602B14D4B", + "4AD981431B98DFD39B6388EDAD742A8E", + ], + 0x0019: [ + "FBDA75963FE690CFF35B7AA7B408CF631744EDEF5F7931A04D58FD6A921FFDB3", + "F72C1D80FFDA2E3BF085F4133E6D2805", + ], + 0x001C: [ + "8103EA9DB790578219C4CEDF0592B43064A7D98B601B6C7BC45108C4047AA80F", + "246F4B8328BE6A2D394EDE20479247C5", + ], + } + NP_KLIC_KEY = bytearray([0xF2, 0xFB, 0xCA, 0x7A, 0x75, 0xB0, 0x4E, 0xDC, 0x13, 0x90, 0x63, 0x8C, 0xCD, 0xFD, 0xD1, 0xEE]) + NP_KLIC_FREE = bytearray([0x72, 0xF9, 0x90, 0x78, 0x8F, 0x9C, 0xFF, 0x74, 0x57, 0x25, 0xF0, 0x8E, 0x4C, 0x12, 0x83, 0x87]) - def __init__(self, fp): + def __init__(self, fp, eboot_key=None): self.fp = fp - self.sce_hdr = None - self.elf_hdr = None - self.encryption_root_header = None - self.cert_header = None - self.segment_cert_header = [] + self.sce_hdr: SceHeader + self.elf_hdr: Union[ElfHeader, ElfHeader32] + self.encryption_root_header: EncryptionRootHeader + self.cert_header: CertificationHeader + self.segment_cert_header: List[SegmentCertHeader] = [] self.data_keys = None - self.self_header = None + self.self_header: SelfHeader self.key_v = None - self.segment_headers = [] - self.segment_ext_table = [] + self.segment_headers: List[Union[ElfSectionHeader, ElfSectionHeader32]] = [] + self.segment_ext_table: List[SegmentExtendedHeader] = [] + self.program_identification_hdr: ProgramIdentificationHeader + self.program_headers: List[Union[ProgramSegmentHeader, ProgramSegmentHeader32]] = [] + self.supplemental_headers: List[SupplementalHeader] = [] + self.eboot_key = eboot_key def load_headers(self): # Read SCE header. @@ -389,6 +512,10 @@ def load_headers(self): # Read SELF header. self.self_header = SelfHeader.unpack(self.fp.read(SelfHeader.struct.size)) + # Read APP INFO + self.fp.seek(self.self_header.program_identification_hdr_offset) + self.program_identification_hdr = ProgramIdentificationHeader.unpack(self.fp.read(ProgramIdentificationHeader.struct.size)) + # Determine if this is a 32 or 64 bit ELF self.fp.seek(self.self_header.elf_hdr_offset) is_elf32 = self.fp.read(8)[4] == 1 @@ -422,6 +549,15 @@ def load_headers(self): if self.scev_info.present: self.scev_version = SCEVersionBody.unpack(self.fp.read(SCEVersionBody.struct.size)) + # Read control info. + self.fp.seek(self.self_header.supplemental_hdr_offset) + i = 0 + while i < self.self_header.supplemental_hdr_size: + supp_hdr = SupplementalHeader.unpack(self.fp.read(SupplementalHeader.struct.size)) + supp_hdr.subheader = supp_hdr.subheader_cls.unpack(self.fp.read(supp_hdr.subheader_cls.struct.size)) + self.supplemental_headers.append(supp_hdr) + i += SupplementalHeader.struct.size + supp_hdr.subheader_cls.struct.size + # Read ELF section headers. self.segment_headers = [] if self.elf_hdr.e_shoff == 0 and self.elf_hdr.e_shnum: @@ -496,12 +632,22 @@ def load_metadata(self): # Find the right keyset from the key vault. try: - metadata_key = bytes.fromhex(self.apploader_keys[self.sce_hdr.attribute][0]) - metadata_iv = bytes.fromhex(self.apploader_keys[self.sce_hdr.attribute][1]) + if self.program_identification_hdr.program_type == 4: + metadata_key = bytes.fromhex(self.apploader_keys[self.sce_hdr.attribute][0]) + metadata_iv = bytes.fromhex(self.apploader_keys[self.sce_hdr.attribute][1]) + elif self.program_identification_hdr.program_type == 8: + metadata_key = bytes.fromhex(self.npdrm_keys[self.sce_hdr.attribute][0]) + metadata_iv = bytes.fromhex(self.npdrm_keys[self.sce_hdr.attribute][1]) except KeyError: print("Could not find decryption key") return + if npd := self.get_npd_header(): + metadata_info = self.decrypt_npdrm(npd, metadata_info) + if not metadata_info: + print("Failed to decrypt SCE metadata info!") + return + # Decrypt the metadata info. aes = AES.new(metadata_key, AES.MODE_CBC, metadata_iv) metadata_info = aes.decrypt(metadata_info) @@ -547,6 +693,36 @@ def decrypt_data(self, segment_cert_header): aes = AES.new(data_key, AES.MODE_CTR, counter=Counter.new(128, initial_value=int.from_bytes(data_iv, "big"))) return aes.encrypt(buf) + def get_npd_header(self) -> Optional[NPDHeader]: + # Iterate through supplemental headers + for info in self.supplemental_headers: + if info.type == 3: # Type 3 indicates NPDRM control info + return info.subheader + + return None + + def decrypt_npdrm(self, npd, metadata) : + if npd.license in [1, 2]: + if not self.eboot_key: + return False + npdrm_key = self.eboot_key + elif npd.license == 3: # Free license + npdrm_key = self.NP_KLIC_FREE + else: + print("Invalid NPDRM license type!") + return metadata + + # Decrypt our key with NP_KLIC_KEY + cipher = AES.new(self.NP_KLIC_KEY, AES.MODE_ECB) + npdrm_key = cipher.decrypt(bytes(npdrm_key)) + + # IV is empty (all zeros) + npdrm_iv = bytes(16) + + # Use our final key to decrypt the NPDRM layer + cipher = AES.new(npdrm_key, AES.MODE_CBC, npdrm_iv) + return cipher.decrypt(metadata) + def get_decrypted_elf(self): # Allocate a buffer to store decrypted data. elf = io.BytesIO() From 1b77aba9ff0881d56754abc04243e047f37b7657 Mon Sep 17 00:00:00 2001 From: Sazpaimon Date: Wed, 26 Mar 2025 00:56:52 -0400 Subject: [PATCH 4/8] PSX: Support more ways of getting EDAT keys from NPDRM packages 1. Add more common keys 2. Try keys from the database 3. Bruteforce the key from EBOOT.BIN if we can decrypt it. --- build_ps3_key_db.py | 100 +++++- common/factory.py | 4 +- common/iso_path_reader/methods/compressed.py | 2 + post_psx/npdrm/path_reader.py | 343 +++++++++++++++++-- post_psx/path_reader.py | 12 + post_psx/utils/edat.py | 14 +- ps2exe.py | 3 +- ps3/path_reader.py | 12 +- ps3/processor.py | 15 +- ps3/self_parser.py | 73 ++-- 10 files changed, 492 insertions(+), 86 deletions(-) create mode 100644 post_psx/path_reader.py diff --git a/build_ps3_key_db.py b/build_ps3_key_db.py index b52c6eb..ff3c6b2 100644 --- a/build_ps3_key_db.py +++ b/build_ps3_key_db.py @@ -1,4 +1,5 @@ import argparse +import csv import pathlib import sqlite3 import xml.etree.ElementTree as ET @@ -19,24 +20,58 @@ required=True, default=argparse.SUPPRESS) + parser.add_argument('-l', + '--dev-klics', + help="dev_klics.txt file", + type=str, + required=False, + default=argparse.SUPPRESS) + + parser.add_argument('-r', + '--rap-dir', + help="Directory containing rap files", + type=str, + required=False, + default=argparse.SUPPRESS) + parser.add_argument('-o', '--output', help="Output directory", type=str, - default=str(((pathlib.Path(__file__).parent / 'ps3').absolute()))) + default=str(((pathlib.Path(__file__).parent / 'post_psx').absolute()))) + + parser.add_argument('-t', + '--tsv', + help="PS3/PSP_GAMES.TSV file path", + nargs='+', + action='append', + type=str) + + parser.add_argument('-a', + '--append', + help="Append to db instead of overwriting it", + action='store_true', + default=False) args = parser.parse_args() key_dir = pathlib.Path(args.keyfolder) db_file = pathlib.Path(args.output) / 'keys.db' - db_file.unlink(missing_ok=True) + if not args.append: + db_file.unlink(missing_ok=True) db = sqlite3.connect(db_file) c = db.cursor() try: c.execute("BEGIN") c.execute(""" - CREATE TABLE keys (name TEXT, size TEXT, crc32 TEXT, md5 TEXT, sha1 TEXT, key BLOB); + CREATE TABLE IF NOT EXISTS keys (name TEXT, size TEXT, crc32 TEXT, md5 TEXT, sha1 TEXT, key BLOB, UNIQUE (size, md5, sha1, key)); + """) + c.execute(""" + CREATE TABLE IF NOT EXISTS dev_klics (key BLOB, title_id TEXT, name TEXT, UNIQUE(key, title_id)); + """) + c.execute(""" + CREATE TABLE IF NOT EXISTS raps (key BLOB, title_id TEXT, UNIQUE(key)); """) key_list = [] @@ -63,7 +98,64 @@ key) ) c.executemany(""" - INSERT INTO keys VALUES (?, ?, ?, ?, ?, ?)""", key_list) + INSERT OR IGNORE INTO keys VALUES (?, ?, ?, ?, ?, ?)""", key_list) + + + if args.dev_klics: + klics = [] + dev_klics = pathlib.Path(args.dev_klics) + if dev_klics.exists(): + with dev_klics.open("r") as f: + for line in f: + if line.startswith("-"): + continue + entry = line.strip().split(' ', maxsplit=2) + if len(entry) != 3 or len(entry[0]) != 32 or len(entry[1]) != 36: + continue + + klics.append(( + bytes.fromhex(entry[0]), + entry[1], + entry[2] + )) + if klics: + c.executemany(""" + INSERT OR IGNORE INTO dev_klics VALUES (?, ?, ?)""", klics) + + raps = set() + if args.rap_dir: + rap_dir = pathlib.Path(args.rap_dir) + if rap_dir.exists(): + for rap_file in rap_dir.glob("*.rap"): + with rap_file.open("rb") as f: + rap_data = f.read() + if len(rap_data) == 256: + rap_data = rap_data[:16] + title_id = rap_file.stem + raps.add(( + rap_data, + title_id + )) + if raps: + c.executemany(""" + INSERT OR IGNORE INTO raps VALUES (?, ?)""", raps) + + if args.tsv: + for tsv in args.tsv: + tsv_path = pathlib.Path(tsv[0]) + if tsv_path.exists(): + with tsv_path.open("r", encoding="utf-8") as infile: + reader = csv.DictReader(infile, delimiter="\t") + for row in reader: + if row["RAP"] and row["Content ID"] and len(row["RAP"]) == 32 and len(row["Content ID"]) == 36: + raps.add(( + bytes.fromhex(row["RAP"]), + row["Content ID"] + )) + if raps: + c.executemany(""" + INSERT OR IGNORE INTO raps VALUES (?, ?)""", list(raps)) + c.execute("COMMIT") except: c.execute("ROLLBACK") diff --git a/common/factory.py b/common/factory.py index 4758329..6a15553 100644 --- a/common/factory.py +++ b/common/factory.py @@ -199,7 +199,9 @@ def get_iso_path_readers(fp, file_name, parent_container, pbar): reader = Pkg(wrapper) reader.parse_header() reader.parse_metadata() - path_readers.append(NPDRMPathReader(reader, wrapper, parent_container)) + path_reader = NPDRMPathReader(reader, wrapper, parent_container) + path_reader.edat_key, path_reader.self_key = path_reader.get_edat_key(pbar) + path_readers.append(path_reader) except Exception as e: exceptions[NPDRMPathReader.volume_type] = e diff --git a/common/iso_path_reader/methods/compressed.py b/common/iso_path_reader/methods/compressed.py index 75712e5..5d02ff5 100644 --- a/common/iso_path_reader/methods/compressed.py +++ b/common/iso_path_reader/methods/compressed.py @@ -48,6 +48,8 @@ def get_file_date(self, file): return None def get_file_path(self, file): + if file == self.get_root_dir(): + return "" if self.is_directory(file): return file.path.rstrip("/").rstrip("\\") return file.path diff --git a/post_psx/npdrm/path_reader.py b/post_psx/npdrm/path_reader.py index 835d89f..079a049 100644 --- a/post_psx/npdrm/path_reader.py +++ b/post_psx/npdrm/path_reader.py @@ -1,11 +1,14 @@ import io import logging +import multiprocessing import os import pathlib +import sqlite3 from common.iso_path_reader.methods.base import IsoPathReader from common.iso_path_reader.methods.chunked_hash_trait import ChunkedHashTrait from post_psx.npdrm.pkg import Pkg +from post_psx.path_reader import PostPsxPathReader from post_psx.utils.edat import EdatFile from ps3.self_parser import SELFDecrypter from utils import pycdlib @@ -32,53 +35,327 @@ def decrypt_blocks(self, blocks): return self.pkg.decrypt(blocks) -class NPDRMPathReader(ChunkedHashTrait, IsoPathReader): +class NPDRMPathReader(ChunkedHashTrait, PostPsxPathReader, IsoPathReader): volume_type = "npdrm" def __init__(self, iso: Pkg, fp, parent_container): super().__init__(iso, fp, parent_container) pkg_header = self.iso.pkg_header self.fp = OffsetFile(self.fp, pkg_header.data_offset, pkg_header.data_offset + pkg_header.data_size) - self.edat_key = self.get_edat_key() + self._decryption_status = {"edat": 1, "eboot": 1} + self.edat_key = None + self.self_key = None + + @property + def decryption_status(self): + if self._decryption_status["edat"] == 1 and self._decryption_status["eboot"] == 1: + return "decrypted" + elif self._decryption_status["edat"] == 0 and self._decryption_status["eboot"] == 1: + return "eboot only" + else: + return "encrypted" + + def get_edat_key(self, pbar_manager): + # Find relevant files + files = self._find_key_files() + edat_file, eboot_file, self_file = files['edat'], files['eboot'], files['self'] + + if not edat_file and not eboot_file and not self_file: + return None, None + + keys_to_try = self._get_potential_keys() + + # Try to decrypt EDAT file directly with keys + if edat_file: + for edat_key in keys_to_try: + if self.test_key(edat_key, edat_file): + LOGGER.info("Found EDAT/SELF key: %s", edat_key.hex()) + return edat_key, edat_key + + # Try to extract key from EBOOT or SELF + if eboot_file or self_file: + # Process EBOOT file + if eboot_file: + eboot_decryptor = self._create_decryptor(eboot_file) + if eboot_decryptor is None: + return None, None + + # Try to decrypt with potential keys + self_key = self._try_decrypt_with_keys(eboot_decryptor, keys_to_try, edat_file) + if self_key: + LOGGER.info("Found SELF key: %s", self_key.hex()) + return None, self_key + + if not eboot_decryptor.data_keys: + self._decryption_status["eboot"] = 0 + return None, None + + # Process SELF file if needed + if self_file: + self_decryptor = self._create_decryptor(self_file) + if self_decryptor is None: + return None, None + + # Try to decrypt with potential keys + self_key = self._try_decrypt_with_keys(self_decryptor, keys_to_try, edat_file) + if self_key: + LOGGER.info("Found SELF key: %s", self_key.hex()) + return None, self_key + + # If we have a decrypted EBOOT and an encrypted file (EDAT or SELF), try brute force search + if eboot_file and (edat_file or self_file): + encrypted_file = self._prepare_search_target(edat_file, self_file) + if encrypted_file: + eboot_elf = eboot_decryptor.get_decrypted_elf() + eboot_elf.seek(0, 2) + total_candidates = eboot_elf.tell() - 15 + + # Determine number of processes + num_processes = multiprocessing.cpu_count() + + # Create enlighten manager and start the search + result = self.process_chunks( + pbar_manager, + eboot_elf, + encrypted_file, + total_candidates, + num_processes + ) + + if result: + if eboot_decryptor.self_key and eboot_decryptor.self_key != result: + LOGGER.info("Found EDAT key: %s", result.hex()) + LOGGER.info("Found SELF key: %s", eboot_decryptor.self_key.hex()) + return result, eboot_decryptor.self_key + else: + LOGGER.info("Found EDAT/SELF key: %s", result.hex()) + return result, result + + return None, None + + def _get_potential_keys(self): + """Get a list of potential keys to try.""" + keys_to_try = [bytes.fromhex("F69F9C3711ADA2EC0AB663B3FE7002B0")] + file_dir = pathlib.Path( + self.parent_container.get_file_path( + self.parent_container.get_file(str(pathlib.Path(self.fp.name).parent)) + )) - def get_edat_key(self): - edat_key = None - file_dir = pathlib.Path(self.fp.name).parent try: title_id = self.iso.pkg_header.title_id.decode("utf-8") - # Check for a rap file in the dir + # Check for a RAP file in the directory + rap_path = str((file_dir / title_id).with_suffix(".rap")) try: - with self.parent_container.open_file(self.parent_container.get_file(str((file_dir / title_id).with_suffix(".rap")))) as f: - edat_key = EdatFile.rap_to_rif(f.read()) + with self.parent_container.open_file(self.parent_container.get_file(rap_path)) as f: + keys_to_try.append(EdatFile.rap_to_rif(f.read())) except FileNotFoundError: pass - except UnicodeDecodeError: + except (UnicodeDecodeError, AttributeError): pass - edat_file = next((file for file in self.iso_iterator(self.get_root_dir(), recursive=True) if - self.get_file_path(file).lower().endswith(".edat")), None) - if edat_file: - if edat_key: - if self.test_key(edat_key, edat_file): - return edat_key + # Check the key db if it exists + if self.db: + c = self.db.cursor() + # Try to find a rap file with a title + rap = c.execute('SELECT * FROM raps WHERE title_id = ?', [title_id]).fetchone() + if rap and len(rap[0]) == 16: + keys_to_try.append(EdatFile.rap_to_rif(rap[0])) + + # Try to find a match. If no matches, just load the entire klics list + keys = c.execute('SELECT * FROM dev_klics WHERE title_id = ?', [title_id]).fetchall() + if keys: + for key in keys: + keys_to_try.append(key[0]) else: - test_edat_key = bytes.fromhex("0" * 32) - if self.test_key(test_edat_key, edat_file): - return test_edat_key - elif edat_key: - eboot_file = next((file for file in self.iso_iterator(self.get_root_dir(), recursive=True) if - self.get_file_path(file).lower().endswith("eboot.bin")), None) - if eboot_file: + keys = c.execute('SELECT * FROM dev_klics').fetchall() + for key in keys: + keys_to_try.append(key[0]) + + # Add standard keys + keys_to_try.append(bytes(EdatFile.NP_KLIC_FREE)) + keys_to_try.append(bytes(bytearray(16))) + + return set(keys_to_try) + + def _find_key_files(self): + """Find and return the EDAT, EBOOT, and SELF files.""" + root_dir = self.get_root_dir() + + edat_file = next((file for file in self.iso_iterator(root_dir, recursive=True) if + self.get_file_path(file).lower().endswith(".edat") and self.get_file_size(file) > 0), None) + eboot_file = next((file for file in self.iso_iterator(root_dir, recursive=True) if + self.get_file_path(file).lower().endswith("eboot.bin")), None) + self_file = next((file for file in self.iso_iterator(root_dir, recursive=True) if + self.get_file_path(file).lower().endswith(".sprx") or + self.get_file_path(file).lower().endswith(".self")), None) + + return { + 'edat': edat_file, + 'eboot': eboot_file, + 'self': self_file + } + + def _create_decryptor(self, file): + """Create and initialize a SELFDecrypter for the given file.""" + try: + inode = pycdlib.inode.Inode() + inode.parse(file.file_offset // 16, file.file_size, self.fp, 16) + + with DecryptedFileReader(inode, 16, self.iso, file) as f: + decryptor = SELFDecrypter(f) + if not decryptor.load_headers() or decryptor.sce_hdr.attribute & 0x8000 == 0x8000: + return None + return decryptor + except Exception as e: + LOGGER.error(f"Failed to create decryptor for {self.get_file_path(file)}: {str(e)}") + return None + + def _try_decrypt_with_keys(self, decryptor, keys_to_try, edat_file=None): + npd = decryptor.get_npd_header() + + # Not an npdrm self, ignore + if npd is None: + if not decryptor.load_metadata(): + return None + return None + + # Determine if we need a key + need_klic = True + if npd.license == 3: + need_klic = not decryptor.load_metadata() + + # Try each key + if need_klic: + for edat_key in keys_to_try: + decryptor.self_key = edat_key + if decryptor.load_metadata(): + if not edat_file or self.test_key(edat_key, edat_file): + return decryptor.self_key + + return None + + def _prepare_search_target(self, edat_file, self_file): + """Prepare the target file for key search.""" + if edat_file: + try: + inode = pycdlib.inode.Inode() + inode.parse(edat_file.file_offset // 16, edat_file.file_size, self.fp, 16) + with DecryptedFileReader(inode, 16, self.iso, edat_file) as f: + edat_data = io.BytesIO(f.read()) + return EdatFile(edat_data, os.path.basename(self.get_file_path(edat_file)), None) + except Exception as e: + LOGGER.error(f"Failed to prepare EDAT file for search: {str(e)}") + + if self_file: + try: inode = pycdlib.inode.Inode() - inode.parse(eboot_file.file_offset // 16, eboot_file.file_size, self.fp, 16) - with DecryptedFileReader(inode, 16, self.iso, eboot_file) as f: - decryptor = SELFDecrypter(f, edat_key) - decryptor.load_headers() - if decryptor.get_npd_header(): - decryptor.load_metadata() - if decryptor.data_keys: + inode.parse(self_file.file_offset // 16, self_file.file_size, self.fp, 16) + with DecryptedFileReader(inode, 16, self.iso, self_file) as f: + self_data = io.BytesIO(f.read()) + decrypter = SELFDecrypter(self_data, os.path.basename(self.get_file_path(self_file))) + decrypter.load_headers() + return decrypter + except Exception as e: + LOGGER.error(f"Failed to prepare SELF file for search: {str(e)}") + + return None + + def process_chunks(self, manager, elf, encrypted_file, total_candidates, num_processes): + # Create main progress bar + bar_format = ('{desc}{desc_pad}{percentage:3.0f}%|{bar}| ' + 'Processed: {count}/{total} ' + '[{elapsed}<{eta}, {rate:.2f}{unit_pad}{unit}/s]') + + pb_main = manager.counter( + total=total_candidates, + desc='Searching for key:', + unit='keys', + bar_format=bar_format, + leave=False, + ) + + # Shared counter for progress tracking + m = multiprocessing.Manager() + progress = m.Value('i', 0) + progress_lock = m.Lock() + + def create_chunks(total, num_processes): + chunk_size = max(1000, total // (num_processes * 10)) + for i in range(0, total, chunk_size): + yield (i, min(chunk_size, total - i)) + + chunks = [(pos, size, elf, encrypted_file, total_candidates, progress, progress_lock) + for pos, size in create_chunks(total_candidates, num_processes)] + + found_key = None + last_progress = 0 + with multiprocessing.Pool(processes=num_processes) as pool: + try: + results = pool.imap_unordered(self._test_chunk_wrapper, chunks) + + while True: + try: + # Check for results without blocking + result = results.next(timeout=0.1) + if result: + found_key = result + pool.terminate() + break + except StopIteration: + break + except multiprocessing.TimeoutError: + pass + + # Update progress bar + current_progress = progress.value + if current_progress > last_progress: + pb_main.update(current_progress - last_progress) + last_progress = current_progress + + except Exception as e: + LOGGER.error(f"Error in key search: {str(e)}") + finally: + pb_main.close(clear=True) + + return found_key + + @staticmethod + def _test_chunk_wrapper(args): + start_pos, chunk_size, elf, encrypted_file, total_candidates, progress, progress_lock = args + try: + elf.seek(start_pos) + local_progress = 0 + + for i in range(chunk_size): + current_pos = start_pos + i + if current_pos >= total_candidates: + break + + edat_key = elf.read(16) + elf.seek(current_pos + 1) + + local_progress += 1 + if local_progress % 100 == 0: # Batch progress updates to reduce lock contention + with progress_lock: + progress.value += 100 + + try: + if isinstance(encrypted_file, EdatFile) and encrypted_file.decrypt_block(0, edat_key) != -1: + return edat_key + elif isinstance(encrypted_file, SELFDecrypter): + encrypted_file.self_key = edat_key + if encrypted_file.load_metadata(): return edat_key + except Exception as e: + LOGGER.debug(f"Failed to test key at position {current_pos}: {str(e)}") + continue + + except Exception as e: + LOGGER.error(f"Error processing chunk starting at {start_pos}: {str(e)}") + + return None + def test_key(self, key, file): file_path = self.get_file_path(file) inode = pycdlib.inode.Inode() @@ -101,11 +378,10 @@ def iso_iterator(self, base_dir, recursive=False, include_dirs=False): for dir in base_dir.directories.values(): yield from self.iso_iterator(dir, recursive) - def get_file(self, path): - normalized_path = path.strip('/') + normalized_path = pathlib.Path(path).as_posix().strip('/') - for file in self.iso_iterator(self.get_root_dir(), recursive=True): + for file in self.iso_iterator(self.get_root_dir(), recursive=True, include_dirs=True): if file.name_decoded == normalized_path: return file @@ -125,12 +401,13 @@ def open_file(self, file): if file_path.lower().endswith("edat"): f.__enter__() edat_file = EdatFile(f, os.path.basename(file_path), self.edat_key) - if not edat_file.validate_npd_hashes(os.path.basename(file_path), edat_file.hash_key): + if not edat_file.validate_npd_hashes(os.path.basename(file_path)): LOGGER.warning("Could not validate header hashes for %s", file_path) if edat_file.edat_header.file_size == 0: f.__exit__() f = io.BytesIO(b"") elif not self.edat_key: + self._decryption_status["edat"] = 0 LOGGER.warning("Could not locate decryption key for %s. File will not be decrypted", file_path) else: f = edat_file diff --git a/post_psx/path_reader.py b/post_psx/path_reader.py new file mode 100644 index 0000000..e71d53b --- /dev/null +++ b/post_psx/path_reader.py @@ -0,0 +1,12 @@ +import pathlib +import sqlite3 + + +class PostPsxPathReader: + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.db = None + db_file = pathlib.Path(__file__).parent / "keys.db" + if not db_file.exists(): + return + self.db = sqlite3.connect(db_file) diff --git a/post_psx/utils/edat.py b/post_psx/utils/edat.py index 88728f0..a7cf051 100644 --- a/post_psx/utils/edat.py +++ b/post_psx/utils/edat.py @@ -39,6 +39,8 @@ class EdatFile(io.RawIOBase): 0x96, 0x9B, 0xEC, 0x68, 0xAA, 0x0B, 0xC0, 0x81]) EDAT_HASH_1 = bytearray([0x3D, 0x92, 0x69, 0x9B, 0x70, 0x5B, 0x07, 0x38, 0x54, 0xD8, 0xFC, 0xC6, 0xC7, 0x67, 0x27, 0x47]) + NP_KLIC_FREE = bytearray([0x72, 0xF9, 0x90, 0x78, 0x8F, 0x9C, 0xFF, 0x74, 0x57, 0x25, 0xF0, 0x8E, 0x4C, 0x12, 0x83, 0x87]) + EDAT_IV = bytearray(0x10) def __init__(self, fp, file_name, edat_key): @@ -90,11 +92,11 @@ def rap_to_rif(cls, rap): return bytes(key) - def validate_npd_hashes(self, file_name, dec_key): + def validate_npd_hashes(self, file_name): if self.edat_header.flags & self.EDAT_DEBUG_DATA_FLAG: return True - if not self.validate_dev_klic(dec_key): + if not self.validate_dev_klic(self.edat_key): return False # Build the title buffer (content_id + file_name). @@ -158,6 +160,9 @@ def validate_dev_klic(self, dec_key): if (self.npd_header.license & 0x3) != 0x3: return True + if not dec_key: + return False + dev = bytearray(0x60) # Build the dev buffer (first 0x60 bytes of NPD header in big-endian) @@ -171,11 +176,8 @@ def validate_dev_klic(self, dec_key): dev[0x8:0xC] = license dev[0xC:0x10] = app_type - # Check for an empty dev_hash (can't validate if devklic is NULL) - klic = dec_key.to_bytes(length=16, byteorder="little") - # Generate klicensee xor key - key = bytes(a ^ b for a, b in zip(klic, self.NP_OMAC_KEY_2)) + key = bytes(a ^ b for a, b in zip(dec_key, self.NP_OMAC_KEY_2)) # Hash with generated key and compare with dev_hash. key_bytes = key diff --git a/ps2exe.py b/ps2exe.py index 870c8c2..2687fe5 100755 --- a/ps2exe.py +++ b/ps2exe.py @@ -223,7 +223,8 @@ def cleanup_bars(): "new_alt_all_files_hash", "volume_type", "parent_volume_type", - "incomplete_files" + "incomplete_files", + "decryption_status", ) if __name__ == '__main__': diff --git a/ps3/path_reader.py b/ps3/path_reader.py index aa987b9..3c3b445 100644 --- a/ps3/path_reader.py +++ b/ps3/path_reader.py @@ -2,12 +2,12 @@ import functools import logging import pathlib -import sqlite3 import struct from Crypto.Cipher import AES from common.iso_path_reader.methods.pycdlib import PyCdLibPathReader +from post_psx.path_reader import PostPsxPathReader from utils.pycdlib.decrypted_file_io import DecryptedFileIO LOGGER = logging.getLogger(__name__) @@ -40,7 +40,7 @@ def decrypt_blocks(self, blocks): return bytes(decrypted) -class Ps3PathReader(PyCdLibPathReader): +class Ps3PathReader(PostPsxPathReader, PyCdLibPathReader): def __init__(self, iso, fp, *args, **kwargs): super().__init__(iso, fp, *args, **kwargs) self.fp.seek(0) @@ -48,6 +48,7 @@ def __init__(self, iso, fp, *args, **kwargs): self.regions = self.get_regions() self.disc_key = None self.disc_key = self.get_disc_key() + self.decryption_status = "decrypted" def get_regions(self): Region = collections.namedtuple("Region", "start end encrypted") @@ -125,13 +126,11 @@ def get_disc_key(self): return bytes.fromhex(debug_key) # Check the key db if it exists - db_file = (pathlib.Path(__file__).parent) / "keys.db" - if not db_file.exists(): + if not self.db: LOGGER.warning("Could not find key db. Disc will not be decrypted!") return - db = sqlite3.connect(db_file) - c = db.cursor() + c = self.db.cursor() keys = c.execute('SELECT * FROM keys WHERE size = ?', [str(self.fp.length())]).fetchall() for key in keys: f.disc_key = key[-1] @@ -140,6 +139,7 @@ def get_disc_key(self): LOGGER.info("Found key: %s", key[-1].hex()) return key[-1] + self.decryption_status = "encrypted" LOGGER.warning("Could not find key for disc. Disc will not be decrypted!") return diff --git a/ps3/processor.py b/ps3/processor.py index fab9dfa..4fb1543 100644 --- a/ps3/processor.py +++ b/ps3/processor.py @@ -22,9 +22,10 @@ def __init__(self, iso_path_reader, filename, system_type, progress_manager): if self.iso_path_reader.get_file_path(file).strip("/") == "PS3_GAME": self.base_dir = "/PS3_GAME" break - self.eboot_key = None + self.self_key = None if iso_path_reader.volume_type == 'npdrm': - self.eboot_key = iso_path_reader.edat_key + self.self_key = iso_path_reader.self_key + self.decryption_status = "decrypted" @property def sfo_path(self): @@ -56,9 +57,11 @@ def get_file_hashes(self, hash_type=xxhash.xxh64): if any(regex.match(file_path) for regex in self.exe_patterns): with self.iso_path_reader.open_file(file) as f: - decryptor = SELFDecrypter(f, self.eboot_key) + decryptor = SELFDecrypter(f, self.self_key) decryptor.load_headers() - decryptor.load_metadata() + if not decryptor.load_metadata(): + LOGGER.warning("Could not decrypt executable %s", file_path) + self.decryption_status = "encrypted" elf = decryptor.get_decrypted_elf() alt_file_hashes[file_path] = hash_type(elf.read()).digest() return file_hashes, alt_file_hashes, incomplete_files @@ -66,7 +69,7 @@ def get_file_hashes(self, hash_type=xxhash.xxh64): def _parse_exe(self, filename): result = {} with self.iso_path_reader.open_file(self.iso_path_reader.get_file(filename)) as f: - decryptor = SELFDecrypter(f, self.eboot_key) + decryptor = SELFDecrypter(f, self.self_key) decryptor.load_headers() result["exe_signing_type"] = "debug" if decryptor.sce_hdr.attribute & 0x8000 == 0x8000 else "retail" result["exe_num_symbols"] = 0 @@ -95,6 +98,8 @@ def get_extra_fields(self): except FileNotFoundError: LOGGER.warning("No param.sfo found.") + fields["decryption_status"] = getattr(self.iso_path_reader, "decryption_status", self.decryption_status) + try: if self.get_exe_filename(): return {**fields, **self._parse_exe(self.get_exe_filename())} diff --git a/ps3/self_parser.py b/ps3/self_parser.py index 733e26a..9965fd8 100644 --- a/ps3/self_parser.py +++ b/ps3/self_parser.py @@ -482,7 +482,7 @@ class SELFDecrypter: NP_KLIC_KEY = bytearray([0xF2, 0xFB, 0xCA, 0x7A, 0x75, 0xB0, 0x4E, 0xDC, 0x13, 0x90, 0x63, 0x8C, 0xCD, 0xFD, 0xD1, 0xEE]) NP_KLIC_FREE = bytearray([0x72, 0xF9, 0x90, 0x78, 0x8F, 0x9C, 0xFF, 0x74, 0x57, 0x25, 0xF0, 0x8E, 0x4C, 0x12, 0x83, 0x87]) - def __init__(self, fp, eboot_key=None): + def __init__(self, fp, self_key=None): self.fp = fp self.sce_hdr: SceHeader self.elf_hdr: Union[ElfHeader, ElfHeader32] @@ -497,7 +497,7 @@ def __init__(self, fp, eboot_key=None): self.program_identification_hdr: ProgramIdentificationHeader self.program_headers: List[Union[ProgramSegmentHeader, ProgramSegmentHeader32]] = [] self.supplemental_headers: List[SupplementalHeader] = [] - self.eboot_key = eboot_key + self.self_key = self_key def load_headers(self): # Read SCE header. @@ -617,7 +617,7 @@ def load_metadata(self): iv_idx=0xFFFFFFFF, comp_algorithm=1, )) - return + return True meta_headers_and_section_size = self.sce_hdr.file_offset - ( SceHeader.struct.size + self.sce_hdr.ext_header_size + EncryptionRootHeader.struct.size) @@ -639,27 +639,31 @@ def load_metadata(self): metadata_key = bytes.fromhex(self.npdrm_keys[self.sce_hdr.attribute][0]) metadata_iv = bytes.fromhex(self.npdrm_keys[self.sce_hdr.attribute][1]) except KeyError: - print("Could not find decryption key") - return + return False if npd := self.get_npd_header(): metadata_info = self.decrypt_npdrm(npd, metadata_info) if not metadata_info: - print("Failed to decrypt SCE metadata info!") - return + return False + else: + metadata_info = [metadata_info] - # Decrypt the metadata info. - aes = AES.new(metadata_key, AES.MODE_CBC, metadata_iv) - metadata_info = aes.decrypt(metadata_info) + for metadata_info_candidate in metadata_info: + # Decrypt the metadata info. + aes = AES.new(metadata_key, AES.MODE_CBC, metadata_iv) + metadata_info = aes.decrypt(metadata_info_candidate) - # Load the metadata info. - self.encryption_root_header = EncryptionRootHeader.unpack(metadata_info) + # Load the metadata info. + self.encryption_root_header = EncryptionRootHeader.unpack(metadata_info) - # If the padding is not NULL for the key or iv fields, the metadata info - # is not properly decrypted. - if self.encryption_root_header.key_pad[0] != 0x00 or self.encryption_root_header.iv_pad[0] != 0x00: - print("Failed to decrypt SCE metadata info!") - return + # Verify the metadata info. + # If the padding is not NULL for the key or iv fields, the metadata info + # is not properly decrypted. + if self.encryption_root_header.key_pad == b'\x00' * 16 and self.encryption_root_header.iv_pad == b'\x00' * 16: + break + + if self.encryption_root_header.key_pad != b'\x00' * 16 or self.encryption_root_header.iv_pad != b'\x00' * 16: + return False # Perform AES-CTR encryption on the metadata headers. aes = AES.new(self.encryption_root_header.key, AES.MODE_CTR, @@ -678,6 +682,8 @@ def load_metadata(self): data_keys_length = self.cert_header.attr_entry_num * 0x10 self.data_keys = bytearray(metadata_headers.read(data_keys_length)) + return True + def decrypt_data(self, segment_cert_header): # Get the key and iv from the previously stored key buffer. data_key_offset = segment_cert_header.key_idx * 0x10 @@ -703,25 +709,32 @@ def get_npd_header(self) -> Optional[NPDHeader]: def decrypt_npdrm(self, npd, metadata) : if npd.license in [1, 2]: - if not self.eboot_key: + if not self.self_key: return False - npdrm_key = self.eboot_key - elif npd.license == 3: # Free license - npdrm_key = self.NP_KLIC_FREE + npdrm_key = [self.self_key] + elif npd.license == 3: + npdrm_key = [] + if self.self_key: + npdrm_key.append(self.self_key) + npdrm_key.append(self.NP_KLIC_FREE) else: print("Invalid NPDRM license type!") - return metadata + return [metadata] + + decrypted_metadata_candidates = [] - # Decrypt our key with NP_KLIC_KEY - cipher = AES.new(self.NP_KLIC_KEY, AES.MODE_ECB) - npdrm_key = cipher.decrypt(bytes(npdrm_key)) + for key in npdrm_key: + # Decrypt our key with NP_KLIC_KEY + cipher = AES.new(self.NP_KLIC_KEY, AES.MODE_ECB) + k = cipher.decrypt(bytes(key)) - # IV is empty (all zeros) - npdrm_iv = bytes(16) + # IV is empty (all zeros) + npdrm_iv = bytes(16) - # Use our final key to decrypt the NPDRM layer - cipher = AES.new(npdrm_key, AES.MODE_CBC, npdrm_iv) - return cipher.decrypt(metadata) + # Use our final key to decrypt the NPDRM layer + cipher = AES.new(k, AES.MODE_CBC, npdrm_iv) + decrypted_metadata_candidates.append(cipher.decrypt(metadata)) + return decrypted_metadata_candidates def get_decrypted_elf(self): # Allocate a buffer to store decrypted data. From f1ee6040a9f6111267bf6394f4778e8edabb6b54 Mon Sep 17 00:00:00 2001 From: Sazpaimon Date: Wed, 26 Mar 2025 15:54:57 -0400 Subject: [PATCH 5/8] PSX: Improve lz performance using Numba --- Pipfile | 7 +- post_psx/utils/edat.py | 2 +- post_psx/utils/lz.py | 417 ++++++++++++++++++++--------------------- 3 files changed, 207 insertions(+), 219 deletions(-) diff --git a/Pipfile b/Pipfile index 37cc796..091c2a5 100755 --- a/Pipfile +++ b/Pipfile @@ -9,7 +9,7 @@ verify_ssl = true pycdlib = "*" pathlab = "*" xxhash = "*" -numpy = "*" +numpy = "1.24" libarchive-c = "*" pefile = "*" pycryptodome = "*" @@ -20,9 +20,10 @@ machfs = "*" rarfile = ">=4.2" psutil = "*" stream-inflate = "*" -inflate64 = "*" +inflate64 = "1.0.0" bitarray = "*" -xor-cipher = "*" +xor-cipher = "5.0.0" +numba = "0.58.1" [requires] python_version = "3.8" diff --git a/post_psx/utils/edat.py b/post_psx/utils/edat.py index a7cf051..4435a2a 100644 --- a/post_psx/utils/edat.py +++ b/post_psx/utils/edat.py @@ -320,7 +320,7 @@ def decrypt_block(self, block_num, decrypt_key): # Handle decompression if needed if should_decompress: - res = lz.LZDecompressor(dec_data.getvalue()).decompress(self.edat_header.block_size) + res = lz.decompress_bytes(dec_data.getvalue(), self.edat_header.block_size) if not res: return -1 diff --git a/post_psx/utils/lz.py b/post_psx/utils/lz.py index ca4a187..13fac29 100644 --- a/post_psx/utils/lz.py +++ b/post_psx/utils/lz.py @@ -1,236 +1,223 @@ -from dataclasses import dataclass +import numba +import numpy as np from typing import Optional, Tuple -@dataclass -class LZDecompressor: - in_bytes: bytes +@numba.jit(nopython=True) +def decode_range(in_bytes: np.ndarray, src: np.int32, range_val: np.uint32, code: np.uint32) -> Tuple[ + np.int32, np.uint32, np.uint32]: + """Updates range and code values based on input bytes.""" + if not (range_val >> 24): + range_val = (range_val << 8) & 0xFFFFFFFF + code = ((code << 8) & 0xFFFFFFFF) + in_bytes[src + 5] + src += 1 + return src, range_val, code - def __init__(self, in_bytes: bytes): - self.in_bytes = in_bytes - self.src = 0 - self.range = 0xFFFFFFFF - # Construct the 32-bit "code" from bytes 1..4 - self.code = int.from_bytes(in_bytes[1:5], byteorder='big') +@numba.jit(nopython=True) +def decode_bit(in_bytes: np.ndarray, tmp: np.ndarray, src: np.int32, range_val: np.uint32, + code: np.uint32, tmp_index: np.int32, acc: Optional[np.ndarray] = None) -> Tuple[ + np.int32, np.uint32, np.uint32, np.int32]: + """Decodes a single bit from the compressed data.""" + src, range_val, code = decode_range(in_bytes, src, range_val, code) - # Initialize temporary buffer with 0x80 bytes - self.tmp = bytearray([0x80] * 0xCA8 + [0] * (3272 - 0xCA8)) + c = np.uint32(tmp[tmp_index]) + tmp[tmp_index] = np.uint8((c - (c >> 3)) & 0xFF) - def decode_range(self) -> None: - """Updates range and code values based on input bytes.""" - if not (self.range >> 24): - self.range = (self.range << 8) & 0xFFFFFFFF - self.code = ((self.code << 8) & 0xFFFFFFFF) + self.in_bytes[self.src + 5] - self.src += 1 + if acc is not None: + acc[0] = np.int32(acc[0] << 1) - def decode_bit(self, acc: Optional[list], tmp_index: int) -> int: - """ - Decodes a single bit from the compressed data. - - Args: - acc: Optional accumulator list containing a single value - tmp_index: Index into temporary buffer - - Returns: - Decoded bit (0 or 1) - """ - self.decode_range() - - c = self.tmp[tmp_index] - self.tmp[tmp_index] = (c - (c >> 3)) & 0xFF + val = np.uint32((range_val >> 8) * c) + if code < val: + range_val = val + tmp[tmp_index] = np.uint8((tmp[tmp_index] + 31) & 0xFF) if acc is not None: - acc[0] <<= 1 - - val = (self.range >> 8) * c + acc[0] += 1 + return src, range_val, code, 1 + else: + code -= val + range_val -= val + return src, range_val, code, 0 + + +@numba.jit(nopython=True) +def decode_number(in_bytes: np.ndarray, tmp: np.ndarray, src: np.int32, range_val: np.uint32, + code: np.uint32, base_offset: np.int32, index_val: np.int32) -> Tuple[ + np.int32, np.uint32, np.uint32, np.int32, np.int32]: + """Decodes a number from the compressed data.""" + acc = np.array([1], dtype=np.int32) + + if index_val >= 3: + src, range_val, code, _ = decode_bit(in_bytes, tmp, src, range_val, code, base_offset + 0x18, acc) + if index_val >= 4: + src, range_val, code, _ = decode_bit(in_bytes, tmp, src, range_val, code, base_offset + 0x18, acc) + if index_val >= 5: + src, range_val, code = decode_range(in_bytes, src, range_val, code) + for _ in range(index_val - 4): + acc[0] = np.int32(acc[0] << 1) + range_val >>= 1 + if code < range_val: + acc[0] += 1 + else: + code -= range_val + + src, range_val, code, bit_flag = decode_bit(in_bytes, tmp, src, range_val, code, base_offset, acc) + + if index_val >= 1: + src, range_val, code, _ = decode_bit(in_bytes, tmp, src, range_val, code, base_offset + 0x8, acc) + if index_val >= 2: + src, range_val, code, _ = decode_bit(in_bytes, tmp, src, range_val, code, base_offset + 0x10, acc) + + return src, range_val, code, acc[0], bit_flag + + +@numba.jit(nopython=True) +def decode_word(in_bytes: np.ndarray, tmp: np.ndarray, src: np.int32, range_val: np.uint32, + code: np.uint32, base_offset: np.int32, index_val: np.int32) -> Tuple[ + np.int32, np.uint32, np.uint32, np.int32, np.int32]: + """Decodes a word from the compressed data.""" + index_val = np.int32(index_val // 8) + acc = np.array([1], dtype=np.int32) + + if index_val >= 3: + src, range_val, code, _ = decode_bit(in_bytes, tmp, src, range_val, code, base_offset + 4, acc) + if index_val >= 4: + src, range_val, code, _ = decode_bit(in_bytes, tmp, src, range_val, code, base_offset + 4, acc) + if index_val >= 5: + src, range_val, code = decode_range(in_bytes, src, range_val, code) + for _ in range(index_val - 4): + acc[0] = np.int32(acc[0] << 1) + range_val >>= 1 + if code < range_val: + acc[0] += 1 + else: + code -= range_val + + src, range_val, code, bit_flag = decode_bit(in_bytes, tmp, src, range_val, code, base_offset, acc) + + if index_val >= 1: + src, range_val, code, _ = decode_bit(in_bytes, tmp, src, range_val, code, base_offset + 1, acc) + if index_val >= 2: + src, range_val, code, _ = decode_bit(in_bytes, tmp, src, range_val, code, base_offset + 2, acc) + + return src, range_val, code, acc[0], bit_flag + + +@numba.jit(nopython=True) +def decompress(in_bytes: np.ndarray, out_size: np.int32) -> Optional[np.ndarray]: + """Decompresses the input data.""" + out = np.zeros(out_size, dtype=np.uint8) + src = np.int32(0) + pos = np.int32(0) + head = np.uint8(in_bytes[0]) + prev = np.uint8(0) + offset = np.int32(0) + range_val = np.uint32(0xFFFFFFFF) + + # Construct the 32-bit "code" from bytes 1..4 + code = np.uint32(int(in_bytes[1]) << 24 | int(in_bytes[2]) << 16 | int(in_bytes[3]) << 8 | int(in_bytes[4])) + + # Initialize temporary buffer + tmp = np.zeros(3272, dtype=np.uint8) + tmp[:0xCA8] = np.uint8(0x80) + + # Handle uncompressed data + if head > 0x80: + length = code + if length <= out_size: + out[:length] = in_bytes[5:5 + length] + return out + return None + + while pos < out_size: + sect1_index = offset + 0xB68 + + src, range_val, code, bit = decode_bit(in_bytes, tmp, src, range_val, code, sect1_index) + + if bit == 0: + # Handle raw character + if offset > 0: + offset -= 1 + if pos >= out_size: + return out[:pos] + + sect = np.int32((((((pos & 7) << 8) + prev) >> head) & 7) * 0xFF - 1) + acc = np.array([1], dtype=np.int32) + + while not (acc[0] >> 8): + src, range_val, code, _ = decode_bit(in_bytes, tmp, src, range_val, code, sect + acc[0], acc) + + out[pos] = np.uint8(acc[0] & 0xFF) + pos += 1 - if self.code < val: - self.range = val - self.tmp[tmp_index] = (self.tmp[tmp_index] + 31) & 0xFF - if acc is not None: - acc[0] += 1 - return 1 else: - self.code -= val - self.range -= val - return 0 - - def decode_number(self, base_offset: int, index_val: int) -> Tuple[int, int]: - """ - Decodes a number from the compressed data. - - Args: - base_offset: Offset into temporary buffer - index_val: Index value from caller - - Returns: - Tuple of (decoded number, bit flag) - """ - acc = [1] - - # Handle higher index values - if index_val >= 3: - self.decode_bit(acc, base_offset + 0x18) - if index_val >= 4: - self.decode_bit(acc, base_offset + 0x18) - if index_val >= 5: - self.decode_range() - for _ in range(index_val - 4): - acc[0] <<= 1 - self.range >>= 1 - if self.code < self.range: - acc[0] += 1 - else: - self.code -= self.range - - bit_flag = self.decode_bit(acc, base_offset) - - # Handle lower index values - if index_val >= 1: - self.decode_bit(acc, base_offset + 0x8) - if index_val >= 2: - self.decode_bit(acc, base_offset + 0x10) - - return acc[0], bit_flag - - def decode_word(self, base_offset: int, index_val: int) -> Tuple[int, int]: - """ - Decodes a word from the compressed data. - - Args: - base_offset: Offset into temporary buffer - index_val: Index value divided by 8 - - Returns: - Tuple of (decoded word, bit flag) - """ - index_val //= 8 - acc = [1] - - if index_val >= 3: - self.decode_bit(acc, base_offset + 4) - if index_val >= 4: - self.decode_bit(acc, base_offset + 4) - if index_val >= 5: - self.decode_range() - for _ in range(index_val - 4): - acc[0] <<= 1 - self.range >>= 1 - if self.code < self.range: - acc[0] += 1 - else: - self.code -= self.range - - bit_flag = self.decode_bit(acc, base_offset) - - if index_val >= 1: - self.decode_bit(acc, base_offset + 1) - if index_val >= 2: - self.decode_bit(acc, base_offset + 2) - - return acc[0], bit_flag - - def decompress(self, out_size: int) -> Optional[bytes]: - """ - Decompresses the input data. - - Args: - out_size: Expected size of decompressed output - - Returns: - Decompressed data as bytes, or None if decompression fails - """ - out = bytearray(out_size) - pos = 0 - head = self.in_bytes[0] - prev = 0 - offset = 0 - - # Handle uncompressed data - if head > 0x80: - length = self.code - if length <= out_size: - out[:length] = self.in_bytes[5:5 + length] - return bytes(out) - return None - - while pos < out_size: - sect1_index = offset + 0xB68 - - if self.decode_bit(None, sect1_index) == 0: - # Handle raw character - if offset > 0: - offset -= 1 - if pos >= out_size: - return bytes(out[:pos]) - - sect = (((((pos & 7) << 8) + prev) >> head) & 7) * 0xFF - 1 - acc = [1] - - while not (acc[0] >> 8): - self.decode_bit(acc, sect + acc[0]) - - out[pos] = acc[0] & 0xFF - pos += 1 - + # Handle compressed stream + index_val = np.int32(-1) + sect1 = offset + 0xB68 + + while True: + sect1 += 8 + src, range_val, code, bit = decode_bit(in_bytes, tmp, src, range_val, code, sect1) + index_val += bit + if not bit or index_val >= 6: + break + + b_size = np.int32(0x160) + tmp_sect2 = index_val + 0x7F1 + + if index_val >= 0 or bit: + sect = np.int32((index_val << 5) | ((((pos << index_val) & 3) << 3) | (offset & 7))) + tmp_sect1 = np.int32(0xBA8 + sect) + src, range_val, code, data_length, _ = decode_number(in_bytes, tmp, src, range_val, code, tmp_sect1, + index_val) + + if data_length == 0xFF: + return out[:pos] else: - # Handle compressed stream - index_val = -1 - sect1 = offset + 0xB68 - - while True: - sect1 += 8 - bit = self.decode_bit(None, sect1) - index_val += bit - if not bit or index_val >= 6: - break - - b_size = 0x160 - tmp_sect2 = index_val + 0x7F1 - - if index_val >= 0 or bit: - sect = (index_val << 5) | ((((pos << index_val) & 3) << 3) | (offset & 7)) - tmp_sect1 = 0xBA8 + sect - data_length, _ = self.decode_number(tmp_sect1, index_val) - - if data_length == 0xFF: - return bytes(out[:pos]) - else: - data_length = 1 - - if data_length <= 2: - tmp_sect2 += 0xF8 - b_size = 0x40 - - shift_val = [1] + data_length = np.int32(1) + + if data_length <= 2: + tmp_sect2 += 0xF8 + b_size = np.int32(0x40) + + shift_val = np.array([1], dtype=np.int32) + + while True: + diff = np.int32((shift_val[0] << 4) - b_size) + src, range_val, code, bit = decode_bit(in_bytes, tmp, src, range_val, code, + tmp_sect2 + (shift_val[0] << 3), shift_val) + if diff >= 0: + break + + if diff > 0 or bit: + if not bit: + diff -= 8 + tmp_sect3 = np.int32(0x928 + diff) + src, range_val, code, data_offset, _ = decode_word(in_bytes, tmp, src, range_val, code, tmp_sect3, diff) + else: + data_offset = np.int32(1) - while True: - diff = (shift_val[0] << 4) - b_size - bit = self.decode_bit(shift_val, tmp_sect2 + (shift_val[0] << 3)) - if diff >= 0: - break + buf_start = pos - data_offset + buf_end = pos + data_length + 1 - if diff > 0 or bit: - if not bit: - diff -= 8 - tmp_sect3 = 0x928 + diff - data_offset, _ = self.decode_word(tmp_sect3, diff) - else: - data_offset = 1 + if buf_start < 0 or buf_end > out_size: + return None - buf_start = pos - data_offset - buf_end = pos + data_length + 1 + offset = np.int32(((buf_end + 1) & 1) + 6) - if buf_start < 0 or buf_end > out_size: - return None + for i in range(data_length + 1): + out[pos] = out[buf_start + i] + pos += 1 - offset = (((buf_end + 1) & 1) + 6) + prev = out[pos - 1] - for i in range(data_length + 1): - out[pos] = out[buf_start + i] - pos += 1 + return out - prev = out[pos - 1] - return bytes(out) +def decompress_bytes(in_bytes: bytes, out_size: int) -> Optional[bytes]: + """Wrapper function to handle bytes input/output""" + result = decompress(np.frombuffer(in_bytes, dtype=np.uint8), np.int32(out_size)) + if result is not None: + return bytes(result) + return None From 02acf127c3739821434b77f392dc7e1f22d95a45 Mon Sep 17 00:00:00 2001 From: Sazpaimon Date: Wed, 26 Mar 2025 16:05:29 -0400 Subject: [PATCH 6/8] PSX: Store bruteforced klics in the database --- post_psx/npdrm/path_reader.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/post_psx/npdrm/path_reader.py b/post_psx/npdrm/path_reader.py index 079a049..a3bcd91 100644 --- a/post_psx/npdrm/path_reader.py +++ b/post_psx/npdrm/path_reader.py @@ -123,6 +123,12 @@ def get_edat_key(self, pbar_manager): ) if result: + # Store the key in the DB + if self.db: + title_id = self.iso.pkg_header.title_id.decode("utf-8") + c = self.db.cursor() + c.execute('INSERT INTO dev_klics VALUES (?, ?, NULL)', [result, title_id]) + self.db.commit() if eboot_decryptor.self_key and eboot_decryptor.self_key != result: LOGGER.info("Found EDAT key: %s", result.hex()) LOGGER.info("Found SELF key: %s", eboot_decryptor.self_key.hex()) @@ -135,7 +141,7 @@ def get_edat_key(self, pbar_manager): def _get_potential_keys(self): """Get a list of potential keys to try.""" - keys_to_try = [bytes.fromhex("F69F9C3711ADA2EC0AB663B3FE7002B0")] + keys_to_try = [] file_dir = pathlib.Path( self.parent_container.get_file_path( self.parent_container.get_file(str(pathlib.Path(self.fp.name).parent)) From 0bcd2ee8b76722656ca6431a9ae6f47bae84147c Mon Sep 17 00:00:00 2001 From: Sazpaimon Date: Thu, 27 Mar 2025 00:34:18 -0400 Subject: [PATCH 7/8] PSX: Decrypt and process PS2 Classics packages as subcontainers --- post_psx/npdrm/path_reader.py | 19 ++++++++- post_psx/types.py | 8 ++++ post_psx/utils/iso_bin_enc.py | 77 +++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 2 deletions(-) create mode 100644 post_psx/utils/iso_bin_enc.py diff --git a/post_psx/npdrm/path_reader.py b/post_psx/npdrm/path_reader.py index a3bcd91..dbb99b5 100644 --- a/post_psx/npdrm/path_reader.py +++ b/post_psx/npdrm/path_reader.py @@ -10,6 +10,7 @@ from post_psx.npdrm.pkg import Pkg from post_psx.path_reader import PostPsxPathReader from post_psx.utils.edat import EdatFile +from post_psx.utils.iso_bin_enc import IsoBinEncFile from ps3.self_parser import SELFDecrypter from utils import pycdlib from utils.files import OffsetFile @@ -417,6 +418,19 @@ def open_file(self, file): LOGGER.warning("Could not locate decryption key for %s. File will not be decrypted", file_path) else: f = edat_file + elif file_path.lower().endswith("iso.bin.enc"): + f.__enter__() + iso_bin_file = IsoBinEncFile(f, os.path.basename(file_path), self.edat_key) + if iso_bin_file.edat_header.file_size == 0: + f.__exit__() + f = io.BytesIO(b"") + if not self.edat_key: + self._decryption_status["edat"] = 0 + LOGGER.warning("Could not locate decryption key for %s. File will not be decrypted", file_path) + elif not iso_bin_file.test_decrypt(self.edat_key): + LOGGER.warning("Decryption failed for %s, file will not be decrypted", file_path) + else: + f = iso_bin_file f.name = self.get_file_path(file) return f @@ -426,11 +440,12 @@ def get_file_sector(self, file): def get_file_size(self, file): file_path = self.get_file_path(file) - if file_path.lower().endswith("edat"): + if file_path.lower().endswith("edat") or file_path.lower().endswith("iso.bin.enc"): + file_cls = EdatFile if file_path.lower().endswith("edat") else IsoBinEncFile inode = pycdlib.inode.Inode() inode.parse(file.file_offset // 16, file.file_size, self.fp, 16) with DecryptedFileReader(inode, 16, self.iso, file) as f: - edat_size = EdatFile(f, os.path.basename(file_path), self.edat_key).edat_header.file_size + edat_size = file_cls(f, os.path.basename(file_path), self.edat_key).edat_header.file_size if edat_size == 0 or self.edat_key: return edat_size return file.file_size diff --git a/post_psx/types.py b/post_psx/types.py index b4c10a4..7b132e7 100644 --- a/post_psx/types.py +++ b/post_psx/types.py @@ -234,3 +234,11 @@ class EDATHeader(Struct): file_size: int struct = struct.Struct(">iiQ") + + +@dataclass +class IsoBinEncMeta(Struct): + sha1: bytes = field(default_factory=lambda: b'\x00' * 20) + block_id: int = 0 + + struct = struct.Struct(">20sI8x") diff --git a/post_psx/utils/iso_bin_enc.py b/post_psx/utils/iso_bin_enc.py new file mode 100644 index 0000000..96b8600 --- /dev/null +++ b/post_psx/utils/iso_bin_enc.py @@ -0,0 +1,77 @@ +from functools import lru_cache + +from Crypto.Cipher import AES +from Crypto.Hash import SHA1 + +from post_psx.types import IsoBinEncMeta +from post_psx.utils.edat import EdatFile + + +class IsoBinEncFile(EdatFile): + ps2_key_cex_data = bytes([0x10, 0x17, 0x82, 0x34, 0x63, 0xF4, 0x68, 0xC1, + 0xAA, 0x41, 0xD7, 0x00, 0xB1, 0x40, 0xF2, 0x57]) + ps2_key_cex_meta = bytes([0x38, 0x9D, 0xCB, 0xA5, 0x20, 0x3C, 0x81, 0x59, + 0xEC, 0xF9, 0x4C, 0x93, 0x93, 0x16, 0x4C, 0xC9]) + + def __init__(self, fp, file_name, edat_key): + super().__init__(fp, file_name, edat_key) + self.data_blocks_per_section = 512 + self.data_key = self.meta_key = None + if edat_key: + cipher = AES.new(self.ps2_key_cex_data, AES.MODE_CBC, bytearray(16)) + self.data_key = cipher.encrypt(edat_key) + cipher = AES.new(self.ps2_key_cex_meta, AES.MODE_CBC, bytearray(16)) + self.meta_key = cipher.encrypt(edat_key) + + def test_decrypt(self, decrypt_key): + return self.decrypt_block(0, decrypt_key) is not None + + def decrypt_block(self, block_num, decrypt_key): + if not self.data_key: + cipher = AES.new(self.ps2_key_cex_data, AES.MODE_CBC, bytearray(16)) + self.data_key = cipher.encrypt(decrypt_key) + if not self.meta_key: + cipher = AES.new(self.ps2_key_cex_meta, AES.MODE_CBC, bytearray(16)) + self.meta_key = cipher.encrypt(decrypt_key) + + data_position, metadata_position, data_block_section_pos = self._calculate_positions(block_num) + + metadata_decrypted = self.get_metadata_block(metadata_position, self.meta_key) + metadata = IsoBinEncMeta.unpack(metadata_decrypted[0x20*data_block_section_pos:0x20*data_block_section_pos+0x20]) + + if metadata.block_id != block_num: + return None + + self.fp.seek(data_position) + block_enc = self.fp.read(self.edat_header.block_size) + + sha1 = SHA1.new(block_enc) + if metadata.sha1 != sha1.digest(): + return None + + cipher = AES.new(self.data_key, AES.MODE_CBC, bytearray(16)) + decrypted = cipher.decrypt(block_enc) + + return decrypted + + def _calculate_positions(self, block_index): + section = block_index // self.data_blocks_per_section + data_block_section_pos = block_index % self.data_blocks_per_section + + data_position = (self.edat_header.block_size + + (self.edat_header.block_size + # Initial metadata block + (section * self.edat_header.block_size) + # Additional metadata blocks + ((section * self.data_blocks_per_section) + # Previous full sections' data blocks + data_block_section_pos) * self.edat_header.block_size)) # Position within current section + + metadata_position = self.edat_header.block_size + section * (self.data_blocks_per_section + 1) * self.edat_header.block_size + + return data_position, metadata_position, data_block_section_pos + + + @lru_cache + def get_metadata_block(self, pos, dec_key): + self.fp.seek(pos) + metadata_encrypted = self.fp.read(self.edat_header.block_size) + cipher = AES.new(dec_key, AES.MODE_CBC, bytearray(16)) + return cipher.decrypt(metadata_encrypted) From 45feda070fdbdb0ffd0eb5f4bc680a37ccec8ec4 Mon Sep 17 00:00:00 2001 From: Sazpaimon Date: Fri, 4 Apr 2025 12:08:53 -0400 Subject: [PATCH 8/8] PSX: Support PSP Minis and PS1 Classics --- post_psx/npdrm/path_reader.py | 99 ++++-- post_psx/processor.py | 4 + post_psx/types.py | 298 +++++++++++++++++- post_psx/utils/base.py | 76 +++++ post_psx/utils/eboot_pbp.py | 320 +++++++++++++++++++ post_psx/utils/iso_bin_enc.py | 4 +- post_psx/utils/kirk/__init__.py | 0 post_psx/utils/kirk/bb.py | 478 ++++++++++++++++++++++++++++ post_psx/utils/kirk/kirk_engine.py | 481 +++++++++++++++++++++++++++++ post_psx/utils/lz.py | 94 ++++-- post_psx/utils/{edat.py => npd.py} | 9 +- post_psx/utils/pgd.py | 70 +++++ post_psx/utils/pspedat.py | 30 ++ psp/processor.py | 40 ++- utils/pycdlib/decrypted_file_io.py | 5 +- 15 files changed, 1950 insertions(+), 58 deletions(-) create mode 100644 post_psx/utils/base.py create mode 100644 post_psx/utils/eboot_pbp.py create mode 100644 post_psx/utils/kirk/__init__.py create mode 100644 post_psx/utils/kirk/bb.py create mode 100644 post_psx/utils/kirk/kirk_engine.py rename post_psx/utils/{edat.py => npd.py} (98%) create mode 100644 post_psx/utils/pgd.py create mode 100644 post_psx/utils/pspedat.py diff --git a/post_psx/npdrm/path_reader.py b/post_psx/npdrm/path_reader.py index dbb99b5..76234fe 100644 --- a/post_psx/npdrm/path_reader.py +++ b/post_psx/npdrm/path_reader.py @@ -3,14 +3,17 @@ import multiprocessing import os import pathlib -import sqlite3 +import re +from copy import copy from common.iso_path_reader.methods.base import IsoPathReader from common.iso_path_reader.methods.chunked_hash_trait import ChunkedHashTrait from post_psx.npdrm.pkg import Pkg from post_psx.path_reader import PostPsxPathReader -from post_psx.utils.edat import EdatFile +from post_psx.utils.eboot_pbp import EbootPBPFile +from post_psx.utils.npd import NPDFile from post_psx.utils.iso_bin_enc import IsoBinEncFile +from post_psx.utils.pspedat import PSPEdatFile from ps3.self_parser import SELFDecrypter from utils import pycdlib from utils.files import OffsetFile @@ -154,7 +157,7 @@ def _get_potential_keys(self): rap_path = str((file_dir / title_id).with_suffix(".rap")) try: with self.parent_container.open_file(self.parent_container.get_file(rap_path)) as f: - keys_to_try.append(EdatFile.rap_to_rif(f.read())) + keys_to_try.append(NPDFile.rap_to_rif(f.read())) except FileNotFoundError: pass except (UnicodeDecodeError, AttributeError): @@ -166,7 +169,7 @@ def _get_potential_keys(self): # Try to find a rap file with a title rap = c.execute('SELECT * FROM raps WHERE title_id = ?', [title_id]).fetchone() if rap and len(rap[0]) == 16: - keys_to_try.append(EdatFile.rap_to_rif(rap[0])) + keys_to_try.append(NPDFile.rap_to_rif(rap[0])) # Try to find a match. If no matches, just load the entire klics list keys = c.execute('SELECT * FROM dev_klics WHERE title_id = ?', [title_id]).fetchall() @@ -179,7 +182,7 @@ def _get_potential_keys(self): keys_to_try.append(key[0]) # Add standard keys - keys_to_try.append(bytes(EdatFile.NP_KLIC_FREE)) + keys_to_try.append(bytes(NPDFile.NP_KLIC_FREE)) keys_to_try.append(bytes(bytearray(16))) return set(keys_to_try) @@ -188,8 +191,16 @@ def _find_key_files(self): """Find and return the EDAT, EBOOT, and SELF files.""" root_dir = self.get_root_dir() - edat_file = next((file for file in self.iso_iterator(root_dir, recursive=True) if - self.get_file_path(file).lower().endswith(".edat") and self.get_file_size(file) > 0), None) + edat_file = None + for file in self.iso_iterator(root_dir, recursive=True): + if self.get_file_path(file).lower().endswith(".edat"): + inode = pycdlib.inode.Inode() + inode.parse(file.file_offset // 16, file.file_size, self.fp, 16) + with DecryptedFileReader(inode, 16, self.iso, file) as f: + if f.read(0x4) == b"NPD\x00" and self.get_file_size(file) > 0: + edat_file = file + break + eboot_file = next((file for file in self.iso_iterator(root_dir, recursive=True) if self.get_file_path(file).lower().endswith("eboot.bin")), None) self_file = next((file for file in self.iso_iterator(root_dir, recursive=True) if @@ -249,7 +260,7 @@ def _prepare_search_target(self, edat_file, self_file): inode.parse(edat_file.file_offset // 16, edat_file.file_size, self.fp, 16) with DecryptedFileReader(inode, 16, self.iso, edat_file) as f: edat_data = io.BytesIO(f.read()) - return EdatFile(edat_data, os.path.basename(self.get_file_path(edat_file)), None) + return NPDFile(edat_data, os.path.basename(self.get_file_path(edat_file)), None) except Exception as e: LOGGER.error(f"Failed to prepare EDAT file for search: {str(e)}") @@ -347,7 +358,7 @@ def _test_chunk_wrapper(args): progress.value += 100 try: - if isinstance(encrypted_file, EdatFile) and encrypted_file.decrypt_block(0, edat_key) != -1: + if isinstance(encrypted_file, NPDFile) and encrypted_file.decrypt_block(0, edat_key) != -1: return edat_key elif isinstance(encrypted_file, SELFDecrypter): encrypted_file.self_key = edat_key @@ -368,7 +379,7 @@ def test_key(self, key, file): inode = pycdlib.inode.Inode() inode.parse(file.file_offset // 16, file.file_size, self.fp, 16) with DecryptedFileReader(inode, 16, self.iso, file) as f, \ - EdatFile(f, os.path.basename(file_path), key) as f: + NPDFile(f, os.path.basename(file_path), key) as f: return f.decrypt_block(0, key) != -1 def get_root_dir(self): @@ -378,7 +389,23 @@ def iso_iterator(self, base_dir, recursive=False, include_dirs=False): for entry in base_dir.entries: if self.is_directory(entry) and not include_dirs: continue - yield entry + file_path = self.get_file_path(entry) + # Check if this is a multi-disc file and spawn a file for each disc + if file_path.lower().endswith("eboot.pbp"): + inode = pycdlib.inode.Inode() + inode.parse(entry.file_offset // 16, entry.file_size, self.fp, 16) + with DecryptedFileReader(inode, 16, self.iso, entry) as f: + pbp = EbootPBPFile(f) + if pbp.load_header(): + if pbp.num_discs > 1: + disc_entry = copy(entry) + for disc_num in range(pbp.num_discs): + disc_entry.name_decoded = entry.name_decoded.replace("EBOOT.PBP", f"EBOOT.{disc_num+1}.pbp") + yield disc_entry + else: + yield entry + else: + yield entry if not recursive: return @@ -388,6 +415,9 @@ def iso_iterator(self, base_dir, recursive=False, include_dirs=False): def get_file(self, path): normalized_path = pathlib.Path(path).as_posix().strip('/') + if not normalized_path or normalized_path == ".": + return self.get_root_dir() + for file in self.iso_iterator(self.get_root_dir(), recursive=True, include_dirs=True): if file.name_decoded == normalized_path: return file @@ -407,13 +437,19 @@ def open_file(self, file): file_path = self.get_file_path(file) if file_path.lower().endswith("edat"): f.__enter__() - edat_file = EdatFile(f, os.path.basename(file_path), self.edat_key) - if not edat_file.validate_npd_hashes(os.path.basename(file_path)): - LOGGER.warning("Could not validate header hashes for %s", file_path) - if edat_file.edat_header.file_size == 0: + magic = f.read(0x8) + if magic == b"\x00PSPEDAT": + edat_file = PSPEdatFile(f, os.path.basename(file_path), self.edat_key) + file_size = edat_file.size + else: + edat_file = NPDFile(f, os.path.basename(file_path), self.edat_key) + file_size = edat_file.edat_header.file_size + if not edat_file.validate_npd_hashes(os.path.basename(file_path)): + LOGGER.warning("Could not validate header hashes for %s", file_path) + if file_size == 0: f.__exit__() f = io.BytesIO(b"") - elif not self.edat_key: + elif isinstance(edat_file, NPDFile) and not self.edat_key: self._decryption_status["edat"] = 0 LOGGER.warning("Could not locate decryption key for %s. File will not be decrypted", file_path) else: @@ -431,6 +467,15 @@ def open_file(self, file): LOGGER.warning("Decryption failed for %s, file will not be decrypted", file_path) else: f = iso_bin_file + elif file_path.lower().endswith("eboot.pbp") or re.match(r".*eboot\.[0-9]\.pbp$", file_path.lower()): + eboot_match = re.match(r".*eboot\.([0-9])\.pbp$", file_path.lower()) + disc_num = int(eboot_match.group(1)) if eboot_match else 1 + f.__enter__() + eboot_pbp_file = EbootPBPFile(f, disc_num-1) + if not eboot_pbp_file.load_header(): + LOGGER.warning("Decryption failed for %s, file will not be decrypted", file_path) + else: + f = eboot_pbp_file f.name = self.get_file_path(file) return f @@ -441,13 +486,31 @@ def get_file_sector(self, file): def get_file_size(self, file): file_path = self.get_file_path(file) if file_path.lower().endswith("edat") or file_path.lower().endswith("iso.bin.enc"): - file_cls = EdatFile if file_path.lower().endswith("edat") else IsoBinEncFile inode = pycdlib.inode.Inode() inode.parse(file.file_offset // 16, file.file_size, self.fp, 16) with DecryptedFileReader(inode, 16, self.iso, file) as f: - edat_size = file_cls(f, os.path.basename(file_path), self.edat_key).edat_header.file_size + if file_path.lower().endswith("iso.bin.enc"): + edat_size = IsoBinEncFile(f, os.path.basename(file_path), self.edat_key).edat_header.file_size + else: + magic = f.read(0x8) + if magic == b"\x00PSPEDAT": + return PSPEdatFile(f, os.path.basename(file_path), self.edat_key).size + else: + edat_size = NPDFile(f, os.path.basename(file_path), self.edat_key).edat_header.file_size + if edat_size == 0 or self.edat_key: return edat_size + elif file_path.lower().endswith("eboot.pbp") or re.match(r".*eboot\.[0-9]\.pbp$", file_path.lower()): + eboot_match = re.match(r".*eboot\.([0-9])\.pbp$", file_path.lower()) + disc_num = int(eboot_match.group(1)) if eboot_match else 1 + + inode = pycdlib.inode.Inode() + inode.parse(file.file_offset // 16, file.file_size, self.fp, 16) + with DecryptedFileReader(inode, 16, self.iso, file) as f: + pbp = EbootPBPFile(f, disc_num-1) + if pbp.load_header(): + return pbp.size + return file.file_size def is_directory(self, file): diff --git a/post_psx/processor.py b/post_psx/processor.py index 1400547..4b0af2a 100644 --- a/post_psx/processor.py +++ b/post_psx/processor.py @@ -16,6 +16,10 @@ class PostPsxIsoProcessor(BaseIsoProcessor): ] SFO_HEADER_BYTES = 20 + def __init__(self, iso_path_reader, filename, system_type, progress_manager): + + super().__init__(iso_path_reader, filename, system_type, progress_manager) + @property def ignored_paths(self): return self.exe_patterns + [self.update_folder] diff --git a/post_psx/types.py b/post_psx/types.py index 7b132e7..0a22d85 100644 --- a/post_psx/types.py +++ b/post_psx/types.py @@ -2,7 +2,7 @@ import struct from dataclasses import dataclass, field -from typing import List +from typing import List, Tuple, Optional class StructMeta(type): @@ -208,6 +208,302 @@ class PKGMetaData: self_info: VitaSelfInfo = field(default_factory=VitaSelfInfo) +@dataclass +class PBPHeader(Struct): + magic: bytes + version: int + param_sfo_offset: int + icon0_png_offset: int + icon1_pmf_offset: int + pic0_png_offset: int + pic1_png_offset: int + snd0_at3_offset: int + psp_data_offset: int + psar_offset: int + + struct = struct.Struct("<4sIIIIIIIII") + + def check_magic(self): + return self.magic == b'\x00PBP' + + +@dataclass +class NPUMDImg(Struct): + magic: bytes = 0 + unk1: int = 0 + iso_block_size: int = 0 + content_id: bytes = field(default_factory=lambda: b'\x00' * 36) + common1: bytes = field(default_factory=lambda: b'\x00' * 4) + unk2: int = 0 + lba_start: int = 0 + unk3: int = 0 + lba_end: int = 0 + unk4: int = 0 + np_table_offset: int = 0 + game_id: bytes = field(default_factory=lambda: b'\x00' * 10) + common2: bytes = field(default_factory=lambda: b'\x00' * 38) + header_key: bytes = field(default_factory=lambda: b'\x00' * 16) + + struct = struct.Struct("<8sII36s12x4s4xI8xI4xI4xIII10s38s16s80x") + + def check_magic(self): + return self.magic == b'NPUMDIMG' + + +@dataclass +class PSARBlockInfo(Struct): + values: Tuple[int, int, int, int, int, int, int, int] + + struct = struct.Struct(" 1: + return 3 + return 1 + else: + return 2 + + @property + def open_flag(self): + flag = 2 + if self.encryption_mode == 1: + flag |= 4 + + if self.version > 1: + flag |= 8 + return flag + +@dataclass +class PGDSubHeader(Struct): + encrypted_header: bytes = field(default_factory=lambda: bytes(48)) # 0x30 bytes of encrypted header + file_hash: bytes = field(default_factory=lambda: bytes(16)) # File hash + ioctl_hash: bytes = field(default_factory=lambda: bytes(16)) # Hash generated from sceIoIoctl key + encrypted_ioctl_hash: bytes = field(default_factory=lambda: bytes(16)) # Encrypted hash from sceIoIoctl key + data_hash: bytes = field(default_factory=lambda: bytes(16)) # Data hash + encrypted_data_hash: bytes = field(default_factory=lambda: bytes(16)) # Encrypted data hash + + struct = struct.Struct("<48s16s16s16s16s16s") + + # Decrypted header properties + _decrypted_header: bytes = field(default=None, init=False, repr=False, compare=False) + + def set_decrypted_header(self, decrypted_header: bytes): + """Set the decrypted header data after decryption""" + if len(decrypted_header) != 48: + raise ValueError("Decrypted header must be 48 bytes") + self._decrypted_header = decrypted_header + + @property + def decrypted_header(self) -> bytes: + """Get the decrypted header if available""" + return self._decrypted_header + + @property + def hash_key(self) -> Optional[bytes]: + """First field from decrypted header (expected to be NULL)""" + if self._decrypted_header: + return self._decrypted_header[:0x10] + return None + + @property + def null_field(self) -> Optional[int]: + """First field from decrypted header (expected to be NULL)""" + if self._decrypted_header: + return struct.unpack(" Optional[int]: + """Decrypted data size from decrypted header""" + if self._decrypted_header: + return struct.unpack(" Optional[int]: + """Decrypting chunk size from decrypted header""" + if self._decrypted_header: + return struct.unpack(" Optional[int]: + """Data hash address from decrypted header""" + if self._decrypted_header: + return struct.unpack(" Optional[bytes]: + """Hash key from decrypted header (16 bytes)""" + if self._decrypted_header: + return self._decrypted_header[0x26:0x42] + return None + + @property + def align_size(self): + if self.data_size: + return (self.data_size + 15) & ~15 + return None + + @property + def table_offset(self): + if not self.data_hash_address or not self.align_size: + return None + return self.data_hash_address + self.align_size + + @property + def block_nr(self): + if not self.align_size or not self.chunk_size: + return None + block_nr = (self.align_size + self.chunk_size - 1) & ~(self.chunk_size - 1) + return block_nr // self.chunk_size + + +@dataclass +class PSTitleImgHeader(Struct): + """Header structure for PlayStation title images (PSTITLEIMG)""" + + magic: bytes # Magic identifier "PSTITLEIMG000000" + padding1: bytes = field(default_factory=lambda: bytes(0x1F0)) # 496 bytes padding + discs_start_offsets: List[int] = field(default_factory=lambda: [0] * 25) # Start positions for up to 25 discs + game_id: bytes = field(default_factory=lambda: bytes(16)) # Game identifier (e.g. "_SLES_12345") + padding2: bytes = field(default_factory=lambda: bytes(24)) # 24 bytes padding + unknown1: bytes = field(default_factory=lambda: bytes(128)) # 128 bytes of unknown data + unknown2: bytes = field(default_factory=lambda: bytes(128)) # 128 bytes related to data from 0x390 + unknown3: int = 0 # 4 bytes of unknown data + unknown4: bytes = field(default_factory=lambda: bytes(62)) # 62 bytes related to data from 0x30C + padding3: bytes = field(default_factory=lambda: bytes(2)) # 2 bytes padding + unknown5: int = 0 # 4 bytes of unknown data + padding4: bytes = field(default_factory=lambda: bytes(44)) # 44 bytes padding + + struct = struct.Struct("<16s496s100s16s24s128s128sI62s2sI44s") + + def check_magic(self): + return self.magic == b"PSTITLEIMG000000" + + def __post_init__(self): + """Process disc offsets after initialization""" + if isinstance(self.discs_start_offsets, bytes) and len(self.discs_start_offsets) >= 20: + offset_list = [] + for i in range(0, 20, 4): + value = int.from_bytes(self.discs_start_offsets[i:i + 4], byteorder='little') + if value == 0: + break + offset_list.append(value) + self.discs_start_offsets = offset_list + + +@dataclass +class CueEntry(Struct): + type: int # Track Type = 0x41 for DATA, 0x01 for CDDA, 0xA2 for lead out + number: int # Track Number (0x01 to 0x99) + I0m: int # INDEX 00 MM + I0s: int # INDEX 00 SS + I0f: int # INDEX 00 FF + I1m: int # INDEX 01 MM + I1s: int # INDEX 01 SS + I1f: int # INDEX 01 FF + + struct = struct.Struct("> 4) + (bcd_value & 0xF) + + def get_index0_time(self, gap=0): + """Get INDEX 00 time in decimal MM:SS:FF format with optional gap adjustment""" + mm = self.bcd_to_decimal(self.I0m) + ss = max(0, self.bcd_to_decimal(self.I0s) - gap) # Ensure we don't go negative + ff = self.bcd_to_decimal(self.I0f) + return (mm, ss, ff) + + def get_index1_time(self, gap=0): + """Get INDEX 01 time in decimal MM:SS:FF format with optional gap adjustment""" + mm = self.bcd_to_decimal(self.I1m) + ss = max(0, self.bcd_to_decimal(self.I1s) - gap) # Ensure we don't go negative + ff = self.bcd_to_decimal(self.I1f) + return (mm, ss, ff) + + def get_index0_sectors(self, gap=0): + """Convert INDEX 00 time to total sectors (assuming 75 frames per second)""" + mm, ss, ff = self.get_index0_time(gap) + return (mm * 60 + ss) * 75 + ff + + def get_index1_sectors(self, gap=0): + """Convert INDEX 01 time to total sectors (assuming 75 frames per second)""" + mm, ss, ff = self.get_index1_time(gap) + return (mm * 60 + ss) * 75 + ff + + def get_index0_str(self, gap=0): + """Get INDEX 00 time as a formatted string (MM:SS:FF)""" + mm, ss, ff = self.get_index0_time(gap) + return f"{mm:02d}:{ss:02d}:{ff:02d}" + + def get_index1_str(self, gap=0): + """Get INDEX 01 time as a formatted string (MM:SS:FF)""" + mm, ss, ff = self.get_index1_time(gap) + return f"{mm:02d}:{ss:02d}:{ff:02d}" + + @dataclass class NPDHeader(Struct): magic: bytes diff --git a/post_psx/utils/base.py b/post_psx/utils/base.py new file mode 100644 index 0000000..a704541 --- /dev/null +++ b/post_psx/utils/base.py @@ -0,0 +1,76 @@ +import io + + +class BaseFile(io.RawIOBase): + def __init__(self, fp): + self.fp = fp + self._pos = 0 + + def decrypt_block(self, block_num, decrypt_key): + raise NotImplementedError + + def readinto(self, b): + # Determine how many bytes to read + bytes_to_read = len(b) + if self._pos + bytes_to_read > self._size: + bytes_to_read = self._size - self._pos + + if bytes_to_read <= 0: + return 0 + + # Read the data + data = self.read(bytes_to_read) + + # Copy the data into the provided buffer + b[:len(data)] = data + + return len(data) + + def read(self, size=-1): + if size < 0: + size = self._size - self._pos + + data = bytearray() + bytes_read = 0 + while bytes_read < size and self._pos < self._size: + block_num = self._pos // self.edat_header.block_size + block_offset = self._pos % self.edat_header.block_size + block_data = self.decrypt_block(block_num, self.edat_key) + + if block_data == -1: + break + + remaining_in_block = min(len(block_data) - block_offset, self._size - self._pos) + remaining_to_read = min(size - bytes_read, remaining_in_block) + + chunk = block_data[block_offset:block_offset + remaining_to_read] + data.extend(chunk) + self._pos += len(chunk) + bytes_read += len(chunk) + + if len(chunk) < remaining_to_read: + break + + return bytes(data) + + def seek(self, offset, whence=io.SEEK_SET): + if whence == io.SEEK_SET: + self._pos = offset + elif whence == io.SEEK_CUR: + self._pos += offset + elif whence == io.SEEK_END: + self._pos = self._size + offset + self._pos = max(0, min(self._pos, self._size)) + return self._pos + + def tell(self): + return self._pos + + def readable(self): + return True + + def writable(self): + return False + + def seekable(self): + return True diff --git a/post_psx/utils/eboot_pbp.py b/post_psx/utils/eboot_pbp.py new file mode 100644 index 0000000..ed801a0 --- /dev/null +++ b/post_psx/utils/eboot_pbp.py @@ -0,0 +1,320 @@ +import io +import struct + +import numpy as np +from Crypto.Cipher import AES +from Crypto.Hash import CMAC + +from post_psx.types import PBPHeader, NPUMDImg, PSARBlockInfo, PS1TableEntry, \ + PSTitleImgHeader, CueEntry +from post_psx.utils.base import BaseFile +from post_psx.utils.lz import decompress_bytes +from post_psx.utils.pgd import PGDFile +from utils.in_memory_rollover_temp_file import InMemoryRolloverTempFile + + +class EbootPBPFile(PGDFile, BaseFile): + ISO_SECTOR_SIZE = 2048 + + # Kirk7 keys + kirk7_key38 = bytes( + [0x12, 0x46, 0x8d, 0x7e, 0x1c, 0x42, 0x20, 0x9b, 0xba, 0x54, 0x26, 0x83, 0x5e, 0xb0, 0x33, 0x03]) + kirk7_key39 = bytes( + [0xc4, 0x3b, 0xb6, 0xd6, 0x53, 0xee, 0x67, 0x49, 0x3e, 0xa9, 0x5f, 0xbc, 0x0c, 0xed, 0x6f, 0x8a]) + kirk7_key63 = bytes( + [0x9c, 0x9b, 0x13, 0x72, 0xf8, 0xc6, 0x40, 0xcf, 0x1c, 0x62, 0xf5, 0xd5, 0x92, 0xdd, 0xb5, 0x82]) + + # PSP AM hash keys + amctl_hashkey_3 = bytes( + [0xe3, 0x50, 0xed, 0x1d, 0x91, 0x0a, 0x1f, 0xd0, 0x29, 0xbb, 0x1c, 0x3e, 0xf3, 0x40, 0x77, 0xfb]) + amctl_hashkey_4 = bytes( + [0x13, 0x5f, 0xa4, 0x7c, 0xab, 0x39, 0x5b, 0xa4, 0x76, 0xb8, 0xcc, 0xa9, 0x8f, 0x3a, 0x04, 0x45]) + amctl_hashkey_5 = bytes( + [0x67, 0x8d, 0x7f, 0xa3, 0x2a, 0x9c, 0xa0, 0xd1, 0x50, 0x8a, 0xd8, 0x38, 0x5e, 0x4b, 0x01, 0x7e]) + + + def __init__(self, fp, disc_num=0): + super().__init__(fp) + self.size = None + self.psp_iv = None + self.psp_key = None + self.pbp_header: PBPHeader + self.iso_table_offset: int + self.total_blocks: int + self.npudimg: NPUMDImg + self.iso_disc_map: PSTitleImgHeader + self.decrypted_buf: InMemoryRolloverTempFile + self.disc_offsets = [] + self.disc_tables = [] + self.disc_sizes = [] + self.blocks_read = 0 + self.bytes_written = 0 + self.type = None + self.iso_block_size = 0 + self.iso_base_offset = 0 + self.num_discs = 1 + self.disc_num = disc_num + + def load_header(self): + self.fp.seek(0) + self.pbp_header = PBPHeader.unpack(self.fp.read(PBPHeader.struct.size)) + if not self.pbp_header.check_magic(): + return False + + self.fp.seek(self.pbp_header.psar_offset) + psar_magic = self.fp.read(8) + if psar_magic == b"NPUMDIMG": + self.type = 2 + self.fp.seek(self.pbp_header.psar_offset) + npumdimg_raw = bytearray(self.fp.read(NPUMDImg.struct.size)) + + self.npudimg = NPUMDImg.unpack(npumdimg_raw) + if not self.npudimg.check_magic(): + return False + + if self.npudimg.iso_block_size > 16: + return False + + self.iso_block_size = self.npudimg.iso_block_size * 2048 + + # Calculate CMAC and initialize PSP decryption + cmac = CMAC.new(self.kirk7_key38, ciphermod=AES) + cmac.update(npumdimg_raw[:0xc0]) + mac = cmac.digest() + self.psp_key, self.psp_iv = self.init_psp_decrypt(1, mac, npumdimg_raw, 0xc0, 0xa0) + + # Decrypt additional PSAR header info + dec_header = self.aes128_psp_decrypt(self.psp_key, self.psp_iv, 0, npumdimg_raw[0x40:0xa0], 0x60) + npumdimg_raw[0x40:0xa0] = dec_header + + npudimg = NPUMDImg.unpack(npumdimg_raw) + + # Extract ISO information + iso_total = npudimg.lba_end - npudimg.lba_start + self.total_blocks = (iso_total + npudimg.iso_block_size - 1) // npudimg.iso_block_size + self.size = self._size = (npudimg.lba_end + 1) * 2048 + self.iso_table_offset = npudimg.np_table_offset + + # Cache the iso table + self.fp.seek(self.pbp_header.psar_offset + self.iso_table_offset) + self.iso_table = io.BytesIO(self.fp.read(self.total_blocks * 32)) + + elif psar_magic in [b"PSISOIMG", b"PSTITLEI"]: + self.type = 1 + self.iso_block_size = 0x9300 + self.iso_base_offset = 0x100000 + + if psar_magic == b"PSISOIMG": + self.num_discs = 1 + self.disc_offsets = [0] + else: + if not (iso_disc_map := self.decrypt_pgd(self.pbp_header.psar_offset + 0x200)): + return False + self.fp.seek(self.pbp_header.psar_offset) + self.iso_disc_map = PSTitleImgHeader.unpack(self.fp.read(0x200) + iso_disc_map) + self.num_discs = len(self.iso_disc_map.discs_start_offsets) + self.disc_offsets = self.iso_disc_map.discs_start_offsets + + self.iso_table_offset = 0x3C00 # Relative to disc offset + for disc_num, offset in enumerate(self.disc_offsets): + pgd_offset = 0x400 + offset + if not (decrypted_table := self.decrypt_pgd(self.pbp_header.psar_offset + pgd_offset)): + return False + decrypted_table = io.BytesIO(decrypted_table) + num_sectors = self.data_track_sectors(decrypted_table) + if num_sectors: + self.disc_sizes.append(num_sectors * 2352) + else: + decrypted_table.seek(self.iso_table_offset) + self.disc_sizes.append(0) + entry = PS1TableEntry.unpack(decrypted_table.read(0x20)) + while entry.block_size: + if entry.block_marker == 0: + break + self.disc_sizes[disc_num] += 0x9300 + entry = PS1TableEntry.unpack(decrypted_table.read(0x20)) + + self.disc_tables.append(decrypted_table) + self._size = self.size = self.disc_sizes[self.disc_num] + else: + self._size = self.size = self.pbp_header.size + return True + + self.decrypted_buf = InMemoryRolloverTempFile(self.size) + return True + + @staticmethod + def extract_frames_from_cue(iso_table, cue_offset, gap): + iso_table.seek(cue_offset) + cue_entry = CueEntry.unpack(iso_table.read(CueEntry.struct.size)) + + if cue_entry.is_valid_track: + return cue_entry.get_index1_sectors(gap) + + return -1 + + def get_track_size_from_cue(self, iso_table, cue_offset): + cur_track_offset = self.extract_frames_from_cue(iso_table, cue_offset, 2) + if cur_track_offset < 0: + return -1 + + next_track_offset = self.extract_frames_from_cue(iso_table, cue_offset + CueEntry.struct.size, 2) + if next_track_offset < 0: + # get disc size to calculate last track, no gap after last track + next_track_offset = self.extract_frames_from_cue(iso_table, 0x414, 0) + if next_track_offset < 0: + return -1 + + return next_track_offset - cur_track_offset + + def data_track_sectors(self, iso_table): + cue_offset = 0x41E # track 01 offset + track_size = self.get_track_size_from_cue(iso_table, cue_offset) - 2 * 75 # subtract 2 seconds + if track_size < 0: + return 0 + + return track_size + + @staticmethod + def aes128_psp_decrypt(key: bytes, iv: bytes, index: int, buffer: bytearray, size: int): + # Calculate the number of 16-byte blocks needed + num_blocks = (size + 15) // 16 + + # Precompute all counter values + start_counter = index + 1 + end_counter = index + num_blocks + counter_vals = np.arange(start_counter, end_counter + 1, dtype=' 0: + prev_initial = np.zeros(16, dtype=np.uint8) + prev_initial[:12] = iv_first_12 + prev_initial[12:] = np.frombuffer(struct.pack(" 1: + keystream[1:] = counter_blocks[:-1] ^ decrypted_blocks[1:] + + # Flatten keystream and XOR with the buffer + buffer_np = np.frombuffer(buffer, dtype=np.uint8, count=size).copy() + keystream_flat = keystream.ravel()[:size] + buffer_np ^= keystream_flat + + return buffer_np.tobytes() + + def init_psp_decrypt(self, eboot: int, mac: bytes, header: bytes, offset1: int, offset2: int): + """Initialize PSP decryption keys""" + tmp = bytearray(16) + + # Set up key using kirk7_key63 + key_cipher = AES.new(self.kirk7_key63, AES.MODE_ECB) + tmp_data = key_cipher.decrypt(header[offset1:offset1 + 16]) + tmp[:] = tmp_data + + # Process with kirk7_key38 + aes_cipher = AES.new(self.kirk7_key38, AES.MODE_ECB) + tmp_data = aes_cipher.decrypt(bytes(tmp)) + tmp[:] = tmp_data + + # Create IV + iv = bytearray(16) + for i in range(16): + iv[i] = mac[i] ^ tmp[i] ^ header[offset2 + i] ^ self.amctl_hashkey_3[i] ^ self.amctl_hashkey_5[i] + + # Process with kirk7_key39 + aes_cipher = AES.new(self.kirk7_key39, AES.MODE_ECB) + tmp_data = aes_cipher.decrypt(bytes(iv)) + iv[:] = tmp_data + + # Final XOR with amctl_hashkey_4 + for i in range(16): + iv[i] ^= self.amctl_hashkey_4[i] + + return self.kirk7_key63, bytes(iv) + + def decrypt_block(self, block_num, decrypt_key): + # Read and decrypt table entry + (table_info, abs_offset) = self.read_table_entry(block_num) + + self.fp.seek(abs_offset) + data = bytearray(self.fp.read(table_info.block_size)) + + # Additional decryption if needed + if table_info.block_flags is not None and table_info.block_flags & 4 == 0: + try: + data = self.aes128_psp_decrypt(self.psp_key, self.psp_iv, table_info.block_offset // 16, data, table_info.block_size) + except: + raise + + # Process the data block + is_compressed = table_info.block_size != self.iso_block_size + + if is_compressed: + # Decompress LZRC data + out_data = decompress_bytes(bytes(data), self.iso_block_size, self.type) + if out_data is None: + raise RuntimeError("ERROR: LZRC decompression failed!") + else: + out_data = bytes(data) + + return out_data + + def read(self, size=-1): + if size < 0: + size = self.size - self._pos + + read_pos = self._pos + + if self._pos >= self.size: + return b"" + + if self._pos + size > self.bytes_written: + self.decrypted_buf.seek(self.bytes_written) + bytes_to_read = (self._pos + size) - self.bytes_written + + bytes_read = 0 + while bytes_read < bytes_to_read and self.bytes_written < self.size: + block_data = self.decrypt_block(self.blocks_read, None) + + if block_data == -1: + break + + self.decrypted_buf.write(block_data) + self.bytes_written += len(block_data) + bytes_read += len(block_data) + self.blocks_read += 1 + + data = self.decrypted_buf[read_pos:min(self.size, read_pos + size)] + self._pos = read_pos + len(data) + return data + + def read_table_entry(self, block_num): + if self.type == 1: + table_offset = self.iso_table_offset + block_num * 0x20 + iso_table = self.disc_tables[self.disc_num] + + iso_table.seek(table_offset) + table = PS1TableEntry.unpack(iso_table.read(0x20)) + return table, self.pbp_header.psar_offset + table.block_offset + self.iso_base_offset + self.disc_offsets[self.disc_num] + else: + table_offset = 32 * block_num + + self.iso_table.seek(table_offset) + table = PSARBlockInfo(struct.unpack(" Optional[bytearray]: + """Set up and execute a KIRK operation on the buffer.""" + header = [ + # Set CBC mode + cbc, + # Set unknown parameters to 0 + 0, + 0, + # Set the key seed + seed, + # Set the data size + size, + ] + + # Pack the header back to buf + buf[:20] = struct.pack("<5I", *header) + + try: + return self.kirk.sceUtilsBufferCopyWithRange(size, buf, len(buf), kirk_code)[20:] + except KirkError as e: + # Handle KIRK errors - in this case we'll just print a warning + print(f"KIRK error during scramble_bb: {e}") + + def update(self, data: bytes) -> 'BBMac': + """Update the MAC with data""" + if self.pad_size > 0x10 or len(data) < 0: + raise BBMacError("Invalid key or data length") + + if (self.pad_size + len(data)) <= 0x10: + # The key hasn't been set yet + # Extract the hash from the data and set it as the key + self.pad[self.pad_size:self.pad_size + len(data)] = data[:len(data)] + self.pad_size += len(data) + else: + # Calculate the seed + seed = 0x3A if self.mode == 0x2 else 0x38 + + # Setup the buffer + scramble_buf = bytearray(0x800 + 0x14) + + # Copy the previous pad key to the buffer + scramble_buf[0x14:0x14 + self.pad_size] = self.pad[:self.pad_size] + + # Calculate new key length + k_len = (self.pad_size + len(data)) & 0x0F + k_len = 0x10 if k_len == 0 else k_len + + # Calculate new data length + n_len = self.pad_size + self.pad_size = k_len + + # Copy data's footer to make a new key + remaining = len(data) - k_len + self.pad[:k_len] = data[remaining:remaining + k_len] + + # Process the encryption in 0x800 blocks + block_size = 0x800 + pos = 0 + + while pos < remaining: + current_block_size = min(block_size, remaining - pos) + + scramble_buf[0x14:0x14 + current_block_size] = data[pos:pos + current_block_size] + + # XOR with key and encrypt + scramble_buf[0x14:0x24] = xor_cipher.cyclic_xor(bytes(scramble_buf[0x14:0x24]), bytes(self.key)) + + result = self._scramble_bb(scramble_buf, current_block_size, seed, 0x4, KIRK.PSP_KIRK_CMD_ENCRYPT) + self.key[:] = result[-16:] + + # Move to next block + pos += current_block_size + + return self + + def digest(self, external_key: bytes = None) -> bytes: + """Finalize and return the MAC digest""" + if self.pad_size > 0x10: + raise BBMacError("Invalid key length") + + # Calculate the seed + seed = 0x3A if self.mode == 0x2 else 0x38 + + # Set up the buffer + scramble_buf = bytearray(0x800 + 0x14) + + # Set up necessary buffers + key_buf = bytearray(0x10) + result_buf = bytearray(0x10) + + # Encrypt the buffer with KIRK CMD 4 + key_buf = self._scramble_bb(scramble_buf, 0x10, seed, 0x4, KIRK.PSP_KIRK_CMD_ENCRYPT) + + # Apply custom padding management to the stored key + b = 0x87 if (key_buf[0] & 0x80) else 0 + + # Shift bytes left by 1 bit + for i in range(15): + key_buf[i] = ((key_buf[i] << 1) | (key_buf[i + 1] >> 7)) & 0xFF + key_buf[15] = ((key_buf[15] << 1) ^ b) & 0xFF + + if self.pad_size < 0x10: + # Do another round of shifting + b = 0x87 if (key_buf[0] & 0x80) else 0 + for i in range(15): + key_buf[i] = ((key_buf[i] << 1) | (key_buf[i + 1] >> 7)) & 0xFF + key_buf[15] = ((key_buf[15] << 1) ^ b) & 0xFF + + # Add padding + self.pad[self.pad_size] = 0x80 + if (self.pad_size + 1) < 0x10: + self.pad[self.pad_size + 1:0x10] = bytes(0x10 - self.pad_size - 1) + + # XOR previous pad key with new one + self.pad = scramble_buf[0x14:0x24] = xor_cipher.cyclic_xor(bytes(self.pad), bytes(key_buf)) + + # Save the previous result key + result_buf[:] = self.key[:] + + # XOR the decrypted key with the result key + scramble_buf[0x14:0x24] = xor_cipher.cyclic_xor(bytes(scramble_buf[0x14:0x24]), bytes(result_buf)) + + # Encrypt the key with KIRK CMD 4 + result_buf = self._scramble_bb(scramble_buf, 0x10, seed, 0x4, KIRK.PSP_KIRK_CMD_ENCRYPT) + + # XOR with amHashKey3 + result_buf[:0x10] = xor_cipher.cyclic_xor(bytes(result_buf[:0x10]), bytes(KeyVault.amHashKey3)) + + # If mode is 2, encrypt again with KIRK CMD 5 and then KIRK CMD 4 + if self.mode == 0x2: + # Copy the result buffer into the data buffer + scramble_buf[0x14:0x24] = result_buf[:] + + # Encrypt with KIRK CMD 5 (seed is always 0x100) + result_buf = self._scramble_bb(scramble_buf, 0x10, 0x100, 0x4, KIRK.PSP_KIRK_CMD_ENCRYPT_FUSE) + scramble_buf[0x14:0x24] = result_buf[:] + + # Encrypt again with KIRK CMD 4 + result_buf = self._scramble_bb(scramble_buf, 0x10, seed, 0x4, KIRK.PSP_KIRK_CMD_ENCRYPT) + + # XOR with the supplied key and encrypt with KIRK CMD 4 + if external_key is not None: + # XOR result buffer with user key + result_buf[:0x10] = xor_cipher.cyclic_xor(bytes(result_buf[:0x10]), bytes(external_key)) + + # Copy the result buffer into the data buffer + scramble_buf[0x14:0x24] = result_buf[:] + + # Encrypt with KIRK CMD 4 + result_buf = self._scramble_bb(scramble_buf, 0x10, seed, 0x4, KIRK.PSP_KIRK_CMD_ENCRYPT) + + self.mode = 0 + self.pad_size = 0 + self.pad = bytearray(16) + self.key = bytearray(16) + + return bytes(result_buf) + + def verify(self, mac_value: bytes, external_key: bytes = None) -> bool: + """Verify a MAC value against the current state""" + # Save the current mode since digest() will reset it + saved_mode = self.mode + + # Generate our MAC + our_mac = self.digest(external_key) + + # If mode is 3, decrypt the mac_value first + if (saved_mode & 0x3) == 0x3: + mac_to_verify = self.decrypt_mac_key(mac_value, 0x63) + else: + mac_to_verify = mac_value + + # Compare the MACs + return our_mac == mac_to_verify + + def decrypt_mac_key(self, key: bytes, seed: int) -> bytes: + """Decrypt a BBMac key""" + scramble_buf = bytearray(0x10 + 0x14) + scramble_buf[0x14:0x24] = key[:0x10] + dec_key = self._scramble_bb(scramble_buf, 0x10, seed, 0x5, KIRK.PSP_KIRK_CMD_DECRYPT) + + return bytes(dec_key) if dec_key else bytes(16) + + def extract_key(self, mac_value: bytes) -> bytes: + """Extract a key from a BBMac and its context""" + # Save the mode as it will be reset by digest() + saved_mode = self.mode + + # Generate MAC key + mac_key = self.digest() + + # Get decrypted MAC value + if (saved_mode & 0x3) == 0x3: + dec_key = self.decrypt_mac_key(mac_value, 0x63) + else: + dec_key = mac_value[:0x10] + + # Decrypt the key with the mode seed + seed = 0x3A if saved_mode == 0x2 else 0x38 + final_key = self.decrypt_mac_key(dec_key, seed) + + # XOR to get the final key + key = xor_cipher.cyclic_xor(bytes(mac_key), bytes(final_key)) + + return bytes(key) + + +class BBCipher: + """ + BBCipher - A PyCryptodome style interface for PSP BBCipher algorithm + + Usage: + # For encryption: + cipher = BBCipher.new(mode=BBCipher.MODE_ENCRYPT, key=key) + ciphertext = cipher.update(plaintext) + + # For decryption: + cipher = BBCipher.new(mode=BBCipher.MODE_DECRYPT, key=key, seed=seed) + plaintext = cipher.update(ciphertext) + """ + + # Constants for gen_mode + MODE_ENCRYPT = 0x1 + MODE_DECRYPT = 0x2 + + @classmethod + def new(cls, enc_mode: int, gen_mode: int, key: bytes = None, seed: int = 0): + """Create a new BBCipher object""" + return cls(enc_mode, gen_mode, key, seed) + + def __init__(self, enc_mode: int, gen_mode: int, key: bytes = None, seed: int = 0): + self.enc_mode = enc_mode + self.gen_mode = gen_mode + self.seed = seed + self.key = key if key is not None else bytes(16) + self.buf = bytearray(16) + self.current_seed = 0 + self.kirk = KirkEngine() + + # Initialize the cipher + self._init_cipher() + + def _init_cipher(self): + """Initialize the cipher context""" + data = bytearray(16) + + # Key generator mode 0x1 (encryption): use an encrypted pseudo random number + if self.gen_mode == self.MODE_ENCRYPT: + self.current_seed = 0x1 + + # Generate SHA-1 to act as seed for encryption + try: + rseed = self.kirk.kirk_CMD14(0x14) + except KirkError: + # Fallback to os.urandom if KIRK fails + rseed = os.urandom(0x14) + + # Prepare header with seed + header = bytearray(0x24) + header[:0x14] = rseed + header[0x14:0x24] = rseed[:0x10] + header[0x20:0x24] = bytes(4) # Zero out last 4 bytes + + if self.enc_mode == 0x2: + # Encryption mode 0x2: XOR with AMCTRL keys, encrypt with KIRK CMD5 and XOR with the given key + for i in range(0x10): + header[0x14 + i] ^= KeyVault.amHashKey4[i] + header = self._scramble_bb(header, 0x10, self.seed, 0x4, KIRK.PSP_KIRK_CMD_ENCRYPT) + self.buf[:0x10] = header + + # If the key is not null, XOR the hash with it + if self.key and any(k != 0 for k in self.key): + for i in range(0x10): + self.buf[i] ^= self.key[i] + else: + # Encryption mode 0x1: XOR with AMCTRL keys, encrypt with KIRK CMD4 and XOR with the given key + for i in range(0x10): + header[0x14 + i] ^= KeyVault.amHashKey4[i] + + header = self._scramble_bb(header, 0x10, self.seed, 0x5, KIRK.PSP_KIRK_CMD_DECRYPT) + + self.buf[:0x10] = header + + # If the key is not null, XOR the hash with it + if self.key and any(k != 0 for k in self.key): + for i in range(0x10): + self.buf[i] ^= self.key[i] + + # Key generator mode 0x2 (decryption): directly XOR the data with the given key + elif self.gen_mode == self.MODE_DECRYPT: + self.current_seed = self.seed + # The data hash (first 16-bytes) will be set during the first update + + return data + + def update(self, data: bytes) -> bytes: + """Update cipher with data""" + if not data: + return b'' + + if len(data) % 16 != 0: + raise BBCipherError("Data length must be a multiple of 16") + + # Make a copy of the data that we can modify + data_buffer = bytearray(data) + + # For decryption mode, grab the data hash from the first block + if self.gen_mode == self.MODE_DECRYPT and self.current_seed == self.seed: + self.buf[:0x10] = data[:0x10] + # If the key is not null, XOR the hash with it + if self.key and any(k != 0 for k in self.key): + for i in range(0x10): + self.buf[i] ^= self.key[i] + + # Process the data in 0x800 blocks for efficiency + result = bytearray() + for offset in range(0, len(data), 0x800): + # Get the length to process (0x800 or remaining) + process_len = min(0x800, len(data) - offset) + if process_len < 0x10: # Skip if less than a block + continue + + # Process this chunk + processed = self._cipher_member( + data_buffer, + offset, + process_len + ) + result.extend(processed) + + return bytes(result) + + def _cipher_member(self, data: bytearray, data_offset: int, length: int) -> bytearray: + """Cipher operation for BBCipher context""" + data_buf = bytearray(length + 0x14) + key_buf1 = bytearray(0x10) + key_buf2 = bytearray(0x10) + hash_buf = bytearray(0x10) + + # Copy the hash stored by init + data_buf[0x14:0x24] = self.buf + + if self.enc_mode == 0x2: + # Decryption mode 0x02: XOR the hash with AMCTRL keys and decrypt with KIRK CMD8 + for i in range(0x10): + data_buf[0x14 + i] ^= KeyVault.amHashKey5[i] + + data_buf = self._scramble_bb(data_buf, 0x10, 0x100, 0x5, KIRK.PSP_KIRK_CMD_DECRYPT_FUSE) + else: + # Decryption mode 0x01: XOR the hash with AMCTRL keys and decrypt with KIRK CMD7 + for i in range(0x10): + data_buf[0x14 + i] ^= KeyVault.amHashKey5[i] + + result = self._scramble_bb(data_buf, 0x10, 0x39, 0x5, KIRK.PSP_KIRK_CMD_DECRYPT) + + for i in range(0x10): + data_buf[i] = result[i] ^ KeyVault.amHashKey4[i] + + # Store the calculated key + key_buf2[:] = data_buf[:0x10] + + # Apply extra padding if seed is not 1 + if self.current_seed > 0x1: + key_buf1[:0xC] = key_buf2[:0xC] + struct.pack_into(" Optional[bytearray]: + """Set up and execute a KIRK operation on the buffer.""" + header = [ + # Set CBC mode + cbc, + # Set unknown parameters to 0 + 0, + 0, + # Set the key seed + seed, + # Set the data size + size, + ] + + # Pack the header back to buf + buf[:20] = struct.pack("<5I", *header) + + try: + return self.kirk.sceUtilsBufferCopyWithRange(size, buf, len(buf), kirk_code)[20:] + except KirkError as e: + # Handle KIRK errors - in this case we'll just print a warning + print(f"KIRK error during scramble_bb: {e}") + + def finalize(self) -> None: + """Clean up cipher resources""" + self.enc_mode = 0 + self.current_seed = 0 + self.buf = bytearray(16) diff --git a/post_psx/utils/kirk/kirk_engine.py b/post_psx/utils/kirk/kirk_engine.py new file mode 100644 index 0000000..6bc0272 --- /dev/null +++ b/post_psx/utils/kirk/kirk_engine.py @@ -0,0 +1,481 @@ +import struct + +from Crypto.Cipher import AES +from Crypto.Hash import SHA1 +from Crypto.Util.Padding import pad + + +# KIRK constants +class KIRK: + PSP_KIRK_CMD_ENCRYPT = 4 + PSP_KIRK_CMD_ENCRYPT_FUSE = 5 + PSP_KIRK_CMD_DECRYPT = 7 + PSP_KIRK_CMD_DECRYPT_FUSE = 8 + PSP_KIRK_CMD_PRNG = 14 + + +class KirkError(Exception): + """Base exception for KIRK errors""" + pass + + +class KirkNotEnabledError(KirkError): + """KIRK is not enabled""" + pass + + +class KirkInvalidModeError(KirkError): + """Invalid mode specified for KIRK operation""" + pass + + +class KirkInvalidSeedError(KirkError): + """Invalid seed specified for KIRK operation""" + pass + + +class KirkDataSizeZeroError(KirkError): + """Data size is zero""" + pass + + +class KirkNotInitializedError(KirkError): + """KIRK not initialized""" + pass + + +class KirkEngine: + # PSP KIRK commands + PSP_KIRK_CMD_ENCRYPT_PRIVATE = 0 + PSP_KIRK_CMD_DECRYPT_PRIVATE = 1 + PSP_KIRK_CMD_ENCRYPT_SIGN = 2 + PSP_KIRK_CMD_DECRYPT_SIGN = 3 + PSP_KIRK_CMD_ENCRYPT = 4 # IV_0 + PSP_KIRK_CMD_ENCRYPT_FUSE = 5 + PSP_KIRK_CMD_ENCRYPT_USER = 6 + PSP_KIRK_CMD_DECRYPT = 7 # IV_0 + PSP_KIRK_CMD_DECRYPT_FUSE = 8 + PSP_KIRK_CMD_DECRYPT_USER = 9 + PSP_KIRK_CMD_PRIV_SIG_CHECK = 10 + PSP_KIRK_CMD_SHA1_HASH = 11 + PSP_KIRK_CMD_ECDSA_GEN_KEYS = 12 + PSP_KIRK_CMD_ECDSA_MULTIPLY_POINT = 13 + PSP_KIRK_CMD_PRNG = 14 + PSP_KIRK_CMD_INIT = 15 + PSP_KIRK_CMD_ECDSA_SIGN = 16 + PSP_KIRK_CMD_ECDSA_VERIFY = 17 + + # Kirk modes + KIRK_MODE_CMD1 = 1 + KIRK_MODE_CMD2 = 2 + KIRK_MODE_CMD3 = 3 + KIRK_MODE_ENCRYPT_CBC = 4 + KIRK_MODE_DECRYPT_CBC = 5 + + KEYVAULT = [ + bytearray([0x2C, 0x92, 0xE5, 0x90, 0x2B, 0x86, 0xC1, 0x06, 0xB7, 0x2E, 0xEA, 0x6C, 0xD4, 0xEC, 0x72, 0x48]), + bytearray([0x05, 0x8D, 0xC8, 0x0B, 0x33, 0xA5, 0xBF, 0x9D, 0x56, 0x98, 0xFA, 0xE0, 0xD3, 0x71, 0x5E, 0x1F]), + bytearray([0xB8, 0x13, 0xC3, 0x5E, 0xC6, 0x44, 0x41, 0xE3, 0xDC, 0x3C, 0x16, 0xF5, 0xB4, 0x5E, 0x64, 0x84]), + bytearray([0x98, 0x02, 0xC4, 0xE6, 0xEC, 0x9E, 0x9E, 0x2F, 0xFC, 0x63, 0x4C, 0xE4, 0x2F, 0xBB, 0x46, 0x68]), + bytearray([0x99, 0x24, 0x4C, 0xD2, 0x58, 0xF5, 0x1B, 0xCB, 0xB0, 0x61, 0x9C, 0xA7, 0x38, 0x30, 0x07, 0x5F]), + bytearray([0x02, 0x25, 0xD7, 0xBA, 0x63, 0xEC, 0xB9, 0x4A, 0x9D, 0x23, 0x76, 0x01, 0xB3, 0xF6, 0xAC, 0x17]), + bytearray([0x60, 0x99, 0xF2, 0x81, 0x70, 0x56, 0x0E, 0x5F, 0x74, 0x7C, 0xB5, 0x20, 0xC0, 0xCD, 0xC2, 0x3C]), + bytearray([0x76, 0x36, 0x8B, 0x43, 0x8F, 0x77, 0xD8, 0x7E, 0xFE, 0x5F, 0xB6, 0x11, 0x59, 0x39, 0x88, 0x5C]), + bytearray([0x14, 0xA1, 0x15, 0xEB, 0x43, 0x4A, 0x1B, 0xA4, 0x90, 0x5E, 0x03, 0xB6, 0x17, 0xA1, 0x5C, 0x04]), + bytearray([0xE6, 0x58, 0x03, 0xD9, 0xA7, 0x1A, 0xA8, 0x7F, 0x05, 0x9D, 0x22, 0x9D, 0xAF, 0x54, 0x53, 0xD0]), + bytearray([0xBA, 0x34, 0x80, 0xB4, 0x28, 0xA7, 0xCA, 0x5F, 0x21, 0x64, 0x12, 0xF7, 0x0F, 0xBB, 0x73, 0x23]), + bytearray([0x72, 0xAD, 0x35, 0xAC, 0x9A, 0xC3, 0x13, 0x0A, 0x77, 0x8C, 0xB1, 0x9D, 0x88, 0x55, 0x0B, 0x0C]), + bytearray([0x84, 0x85, 0xC8, 0x48, 0x75, 0x08, 0x43, 0xBC, 0x9B, 0x9A, 0xEC, 0xA7, 0x9C, 0x7F, 0x60, 0x18]), + bytearray([0xB5, 0xB1, 0x6E, 0xDE, 0x23, 0xA9, 0x7B, 0x0E, 0xA1, 0x7C, 0xDB, 0xA2, 0xDC, 0xDE, 0xC4, 0x6E]), + bytearray([0xC8, 0x71, 0xFD, 0xB3, 0xBC, 0xC5, 0xD2, 0xF2, 0xE2, 0xD7, 0x72, 0x9D, 0xDF, 0x82, 0x68, 0x82]), + bytearray([0x0A, 0xBB, 0x33, 0x6C, 0x96, 0xD4, 0xCD, 0xD8, 0xCB, 0x5F, 0x4B, 0xE0, 0xBA, 0xDB, 0x9E, 0x03]), + bytearray([0x32, 0x29, 0x5B, 0xD5, 0xEA, 0xF7, 0xA3, 0x42, 0x16, 0xC8, 0x8E, 0x48, 0xFF, 0x50, 0xD3, 0x71]), + bytearray([0x46, 0xF2, 0x5E, 0x8E, 0x4D, 0x2A, 0xA5, 0x40, 0x73, 0x0B, 0xC4, 0x6E, 0x47, 0xEE, 0x6F, 0x0A]), + bytearray([0x5D, 0xC7, 0x11, 0x39, 0xD0, 0x19, 0x38, 0xBC, 0x02, 0x7F, 0xDD, 0xDC, 0xB0, 0x83, 0x7D, 0x9D]), + bytearray([0x51, 0xDD, 0x65, 0xF0, 0x71, 0xA4, 0xE5, 0xEA, 0x6A, 0xAF, 0x12, 0x19, 0x41, 0x29, 0xB8, 0xF4]), + bytearray([0x03, 0x76, 0x3C, 0x68, 0x65, 0xC6, 0x9B, 0x0F, 0xFE, 0x8F, 0xD8, 0xEE, 0xA4, 0x36, 0x16, 0xA0]), + bytearray([0x7D, 0x50, 0xB8, 0x5C, 0xAF, 0x67, 0x69, 0xF0, 0xE5, 0x4A, 0xA8, 0x09, 0x8B, 0x0E, 0xBE, 0x1C]), + bytearray([0x72, 0x68, 0x4B, 0x32, 0xAC, 0x3B, 0x33, 0x2F, 0x2A, 0x7A, 0xFC, 0x9E, 0x14, 0xD5, 0x6F, 0x6B]), + bytearray([0x20, 0x1D, 0x31, 0x96, 0x4A, 0xD9, 0x9F, 0xBF, 0x32, 0xD5, 0xD6, 0x1C, 0x49, 0x1B, 0xD9, 0xFC]), + bytearray([0xF8, 0xD8, 0x44, 0x63, 0xD6, 0x10, 0xD1, 0x2A, 0x44, 0x8E, 0x96, 0x90, 0xA6, 0xBB, 0x0B, 0xAD]), + bytearray([0x5C, 0xD4, 0x05, 0x7F, 0xA1, 0x30, 0x60, 0x44, 0x0A, 0xD9, 0xB6, 0x74, 0x5F, 0x24, 0x4F, 0x4E]), + bytearray([0xF4, 0x8A, 0xD6, 0x78, 0x59, 0x9C, 0x22, 0xC1, 0xD4, 0x11, 0x93, 0x3D, 0xF8, 0x45, 0xB8, 0x93]), + bytearray([0xCA, 0xE7, 0xD2, 0x87, 0xA2, 0xEC, 0xC1, 0xCD, 0x94, 0x54, 0x2B, 0x5E, 0x1D, 0x94, 0x88, 0xB2]), + bytearray([0xDE, 0x26, 0xD3, 0x7A, 0x39, 0x95, 0x6C, 0x2A, 0xD8, 0xC3, 0xA6, 0xAF, 0x21, 0xEB, 0xB3, 0x01]), + bytearray([0x7C, 0xB6, 0x8B, 0x4D, 0xA3, 0x8D, 0x1D, 0xD9, 0x32, 0x67, 0x9C, 0xA9, 0x9F, 0xFB, 0x28, 0x52]), + bytearray([0xA0, 0xB5, 0x56, 0xB4, 0x69, 0xAB, 0x36, 0x8F, 0x36, 0xDE, 0xC9, 0x09, 0x2E, 0xCB, 0x41, 0xB1]), + bytearray([0x93, 0x9D, 0xE1, 0x9B, 0x72, 0x5F, 0xEE, 0xE2, 0x45, 0x2A, 0xBC, 0x17, 0x06, 0xD1, 0x47, 0x69]), + bytearray([0xA4, 0xA4, 0xE6, 0x21, 0x38, 0x2E, 0xF1, 0xAF, 0x7B, 0x17, 0x7A, 0xE8, 0x42, 0xAD, 0x00, 0x31]), + bytearray([0xC3, 0x7F, 0x13, 0xE8, 0xCF, 0x84, 0xDB, 0x34, 0x74, 0x7B, 0xC3, 0xA0, 0xF1, 0x9D, 0x3A, 0x73]), + bytearray([0x2B, 0xF7, 0x83, 0x8A, 0xD8, 0x98, 0xE9, 0x5F, 0xA5, 0xF9, 0x01, 0xDA, 0x61, 0xFE, 0x35, 0xBB]), + bytearray([0xC7, 0x04, 0x62, 0x1E, 0x71, 0x4A, 0x66, 0xEA, 0x62, 0xE0, 0x4B, 0x20, 0x3D, 0xB8, 0xC2, 0xE5]), + bytearray([0xC9, 0x33, 0x85, 0x9A, 0xAB, 0x00, 0xCD, 0xCE, 0x4D, 0x8B, 0x8E, 0x9F, 0x3D, 0xE6, 0xC0, 0x0F]), + bytearray([0x18, 0x42, 0x56, 0x1F, 0x2B, 0x5F, 0x34, 0xE3, 0x51, 0x3E, 0xB7, 0x89, 0x77, 0x43, 0x1A, 0x65]), + bytearray([0xDC, 0xB0, 0xA0, 0x06, 0x5A, 0x50, 0xA1, 0x4E, 0x59, 0xAC, 0x97, 0x3F, 0x17, 0x58, 0xA3, 0xA3]), + bytearray([0xC4, 0xDB, 0xAE, 0x83, 0xE2, 0x9C, 0xF2, 0x54, 0xA3, 0xDD, 0x37, 0x4E, 0x80, 0x7B, 0xF4, 0x25]), + bytearray([0xBF, 0xAE, 0xEB, 0x49, 0x82, 0x65, 0xC5, 0x7C, 0x64, 0xB8, 0xC1, 0x7E, 0x19, 0x06, 0x44, 0x09]), + bytearray([0x79, 0x7C, 0xEC, 0xC3, 0xB3, 0xEE, 0x0A, 0xC0, 0x3B, 0xD8, 0xE6, 0xC1, 0xE0, 0xA8, 0xB1, 0xA4]), + bytearray([0x75, 0x34, 0xFE, 0x0B, 0xD6, 0xD0, 0xC2, 0x8D, 0x68, 0xD4, 0xE0, 0x2A, 0xE7, 0xD5, 0xD1, 0x55]), + bytearray([0xFA, 0xB3, 0x53, 0x26, 0x97, 0x4F, 0x4E, 0xDF, 0xE4, 0xC3, 0xA8, 0x14, 0xC3, 0x2F, 0x0F, 0x88]), + bytearray([0xEC, 0x97, 0xB3, 0x86, 0xB4, 0x33, 0xC6, 0xBF, 0x4E, 0x53, 0x9D, 0x95, 0xEB, 0xB9, 0x79, 0xE4]), + bytearray([0xB3, 0x20, 0xA2, 0x04, 0xCF, 0x48, 0x06, 0x29, 0xB5, 0xDD, 0x8E, 0xFC, 0x98, 0xD4, 0x17, 0x7B]), + bytearray([0x5D, 0xFC, 0x0D, 0x4F, 0x2C, 0x39, 0xDA, 0x68, 0x4A, 0x33, 0x74, 0xED, 0x49, 0x58, 0xA7, 0x3A]), + bytearray([0xD7, 0x5A, 0x54, 0x22, 0xCE, 0xD9, 0xA3, 0xD6, 0x2B, 0x55, 0x7D, 0x8D, 0xE8, 0xBE, 0xC7, 0xEC]), + bytearray([0x6B, 0x4A, 0xEE, 0x43, 0x45, 0xAE, 0x70, 0x07, 0xCF, 0x8D, 0xCF, 0x4E, 0x4A, 0xE9, 0x3C, 0xFA]), + bytearray([0x2B, 0x52, 0x2F, 0x66, 0x4C, 0x2D, 0x11, 0x4C, 0xFE, 0x61, 0x31, 0x8C, 0x56, 0x78, 0x4E, 0xA6]), + bytearray([0x3A, 0xA3, 0x4E, 0x44, 0xC6, 0x6F, 0xAF, 0x7B, 0xFA, 0xE5, 0x53, 0x27, 0xEF, 0xCF, 0xCC, 0x24]), + bytearray([0x2B, 0x5C, 0x78, 0xBF, 0xC3, 0x8E, 0x49, 0x9D, 0x41, 0xC3, 0x3C, 0x5C, 0x7B, 0x27, 0x96, 0xCE]), + bytearray([0xF3, 0x7E, 0xEA, 0xD2, 0xC0, 0xC8, 0x23, 0x1D, 0xA9, 0x9B, 0xFA, 0x49, 0x5D, 0xB7, 0x08, 0x1B]), + bytearray([0x70, 0x8D, 0x4E, 0x6F, 0xD1, 0xF6, 0x6F, 0x1D, 0x1E, 0x1F, 0xCB, 0x02, 0xF9, 0xB3, 0x99, 0x26]), + bytearray([0x0F, 0x67, 0x16, 0xE1, 0x80, 0x69, 0x9C, 0x51, 0xFC, 0xC7, 0xAD, 0x6E, 0x4F, 0xB8, 0x46, 0xC9]), + bytearray([0x56, 0x0A, 0x49, 0x4A, 0x84, 0x4C, 0x8E, 0xD9, 0x82, 0xEE, 0x0B, 0x6D, 0xC5, 0x7D, 0x20, 0x8D]), + bytearray([0x12, 0x46, 0x8D, 0x7E, 0x1C, 0x42, 0x20, 0x9B, 0xBA, 0x54, 0x26, 0x83, 0x5E, 0xB0, 0x33, 0x03]), + bytearray([0xC4, 0x3B, 0xB6, 0xD6, 0x53, 0xEE, 0x67, 0x49, 0x3E, 0xA9, 0x5F, 0xBC, 0x0C, 0xED, 0x6F, 0x8A]), + bytearray([0x2C, 0xC3, 0xCF, 0x8C, 0x28, 0x78, 0xA5, 0xA6, 0x63, 0xE2, 0xAF, 0x2D, 0x71, 0x5E, 0x86, 0xBA]), + bytearray([0x83, 0x3D, 0xA7, 0x0C, 0xED, 0x6A, 0x20, 0x12, 0xD1, 0x96, 0xE6, 0xFE, 0x5C, 0x4D, 0x37, 0xC5]), + bytearray([0xC7, 0x43, 0xD0, 0x67, 0x42, 0xEE, 0x90, 0xB8, 0xCA, 0x75, 0x50, 0x35, 0x20, 0xAD, 0xBC, 0xCE]), + bytearray([0x8A, 0xE3, 0x66, 0x3F, 0x8D, 0x9E, 0x82, 0xA1, 0xED, 0xE6, 0x8C, 0x9C, 0xE8, 0x25, 0x6D, 0xAA]), + bytearray([0x7F, 0xC9, 0x6F, 0x0B, 0xB1, 0x48, 0x5C, 0xA5, 0x5D, 0xD3, 0x64, 0xB7, 0x7A, 0xF5, 0xE4, 0xEA]), + bytearray([0x91, 0xB7, 0x65, 0x78, 0x8B, 0xCB, 0x8B, 0xD4, 0x02, 0xED, 0x55, 0x3A, 0x66, 0x62, 0xD0, 0xAD]), + bytearray([0x28, 0x24, 0xF9, 0x10, 0x1B, 0x8D, 0x0F, 0x7B, 0x6E, 0xB2, 0x63, 0xB5, 0xB5, 0x5B, 0x2E, 0xBB]), + bytearray([0x30, 0xE2, 0x57, 0x5D, 0xE0, 0xA2, 0x49, 0xCE, 0xE8, 0xCF, 0x2B, 0x5E, 0x4D, 0x9F, 0x52, 0xC7]), + bytearray([0x5E, 0xE5, 0x04, 0x39, 0x62, 0x32, 0x02, 0xFA, 0x85, 0x39, 0x3F, 0x72, 0xBB, 0x77, 0xFD, 0x1A]), + bytearray([0xF8, 0x81, 0x74, 0xB1, 0xBD, 0xE9, 0xBF, 0xDD, 0x45, 0xE2, 0xF5, 0x55, 0x89, 0xCF, 0x46, 0xAB]), + bytearray([0x7D, 0xF4, 0x92, 0x65, 0xE3, 0xFA, 0xD6, 0x78, 0xD6, 0xFE, 0x78, 0xAD, 0xBB, 0x3D, 0xFB, 0x63]), + bytearray([0x74, 0x7F, 0xD6, 0x2D, 0xC7, 0xA1, 0xCA, 0x96, 0xE2, 0x7A, 0xCE, 0xFF, 0xAA, 0x72, 0x3F, 0xF7]), + bytearray([0x1E, 0x58, 0xEB, 0xD0, 0x65, 0xBB, 0xF1, 0x68, 0xC5, 0xBD, 0xF7, 0x46, 0xBA, 0x7B, 0xE1, 0x00]), + bytearray([0x24, 0x34, 0x7D, 0xAF, 0x5E, 0x4B, 0x35, 0x72, 0x7A, 0x52, 0x27, 0x6B, 0xA0, 0x54, 0x74, 0xDB]), + bytearray([0x09, 0xB1, 0xC7, 0x05, 0xC3, 0x5F, 0x53, 0x66, 0x77, 0xC0, 0xEB, 0x36, 0x77, 0xDF, 0x83, 0x07]), + bytearray([0xCC, 0xBE, 0x61, 0x5C, 0x05, 0xA2, 0x00, 0x33, 0x37, 0x8E, 0x59, 0x64, 0xA7, 0xDD, 0x70, 0x3D]), + bytearray([0x0D, 0x47, 0x50, 0xBB, 0xFC, 0xB0, 0x02, 0x81, 0x30, 0xE1, 0x84, 0xDE, 0xA8, 0xD4, 0x84, 0x13]), + bytearray([0x0C, 0xFD, 0x67, 0x9A, 0xF9, 0xB4, 0x72, 0x4F, 0xD7, 0x8D, 0xD6, 0xE9, 0x96, 0x42, 0x28, 0x8B]), + bytearray([0x7A, 0xD3, 0x1A, 0x8B, 0x4B, 0xEF, 0xC2, 0xC2, 0xB3, 0x99, 0x01, 0xA9, 0xFE, 0x76, 0xB9, 0x87]), + bytearray([0xBE, 0x78, 0x78, 0x17, 0xC7, 0xF1, 0x6F, 0x1A, 0xE0, 0xEF, 0x3B, 0xDE, 0x4C, 0xC2, 0xD7, 0x86]), + bytearray([0x7C, 0xD8, 0xB8, 0x91, 0x91, 0x0A, 0x43, 0x14, 0xD0, 0x53, 0x3D, 0xD8, 0x4C, 0x45, 0xBE, 0x16]), + bytearray([0x32, 0x72, 0x2C, 0x88, 0x07, 0xCF, 0x35, 0x7D, 0x4A, 0x2F, 0x51, 0x19, 0x44, 0xAE, 0x68, 0xDA]), + bytearray([0x7E, 0x6B, 0xBF, 0xF6, 0xF6, 0x87, 0xB8, 0x98, 0xEE, 0xB5, 0x1B, 0x32, 0x16, 0xE4, 0x6E, 0x5D]), + bytearray([0x08, 0xEA, 0x5A, 0x83, 0x49, 0xB5, 0x9D, 0xB5, 0x3E, 0x07, 0x79, 0xB1, 0x9A, 0x59, 0xA3, 0x54]), + bytearray([0xF3, 0x12, 0x81, 0xBF, 0xE6, 0x9F, 0x51, 0xD1, 0x64, 0x08, 0x25, 0x21, 0xFF, 0xBB, 0x22, 0x61]), + bytearray([0xAF, 0xFE, 0x8E, 0xB1, 0x3D, 0xD1, 0x7E, 0xD8, 0x0A, 0x61, 0x24, 0x1C, 0x95, 0x92, 0x56, 0xB6]), + bytearray([0x92, 0xCD, 0xB4, 0xC2, 0x5B, 0xF2, 0x35, 0x5A, 0x23, 0x09, 0xE8, 0x19, 0xC9, 0x14, 0x42, 0x35]), + bytearray([0xE1, 0xC6, 0x5B, 0x22, 0x6B, 0xE1, 0xDA, 0x02, 0xBA, 0x18, 0xFA, 0x21, 0x34, 0x9E, 0xF9, 0x6D]), + bytearray([0x14, 0xEC, 0x76, 0xCE, 0x97, 0xF3, 0x8A, 0x0A, 0x34, 0x50, 0x6C, 0x53, 0x9A, 0x5C, 0x9A, 0xB4]), + bytearray([0x1C, 0x9B, 0xC4, 0x90, 0xE3, 0x06, 0x64, 0x81, 0xFA, 0x59, 0xFD, 0xB6, 0x00, 0xBB, 0x28, 0x70]), + bytearray([0x43, 0xA5, 0xCA, 0xCC, 0x0D, 0x6C, 0x2D, 0x3F, 0x2B, 0xD9, 0x89, 0x67, 0x6B, 0x3F, 0x7F, 0x57]), + bytearray([0x00, 0xEF, 0xFD, 0x18, 0x08, 0xA4, 0x05, 0x89, 0x3C, 0x38, 0xFB, 0x25, 0x72, 0x70, 0x61, 0x06]), + bytearray([0xEE, 0xAF, 0x49, 0xE0, 0x09, 0x87, 0x9B, 0xEF, 0xAA, 0xD6, 0x32, 0x6A, 0x32, 0x13, 0xC4, 0x29]), + bytearray([0x8D, 0x26, 0xB9, 0x0F, 0x43, 0x1D, 0xBB, 0x08, 0xDB, 0x1D, 0xDA, 0xC5, 0xB5, 0x2C, 0x92, 0xED]), + bytearray([0x57, 0x7C, 0x30, 0x60, 0xAE, 0x6E, 0xBE, 0xAE, 0x3A, 0xAB, 0x18, 0x19, 0xC5, 0x71, 0x68, 0x0B]), + bytearray([0x11, 0x5A, 0x5D, 0x20, 0xD5, 0x3A, 0x8D, 0xD3, 0x9C, 0xC5, 0xAF, 0x41, 0x0F, 0x0F, 0x18, 0x6F]), + bytearray([0x0D, 0x4D, 0x51, 0xAB, 0x23, 0x79, 0xBF, 0x80, 0x3A, 0xBF, 0xB9, 0x0E, 0x75, 0xFC, 0x14, 0xBF]), + bytearray([0x99, 0x93, 0xDA, 0x3E, 0x7D, 0x2E, 0x5B, 0x15, 0xF2, 0x52, 0xA4, 0xE6, 0x6B, 0xB8, 0x5A, 0x98]), + bytearray([0xF4, 0x28, 0x30, 0xA5, 0xFB, 0x0D, 0x8D, 0x76, 0x0E, 0xA6, 0x71, 0xC2, 0x2B, 0xDE, 0x66, 0x9D]), + bytearray([0xFB, 0x5F, 0xEB, 0x7F, 0xC7, 0xDC, 0xDD, 0x69, 0x37, 0x01, 0x97, 0x9B, 0x29, 0x03, 0x5C, 0x47]), + bytearray([0x02, 0x32, 0x6A, 0xE7, 0xD3, 0x96, 0xCE, 0x7F, 0x1C, 0x41, 0x9D, 0xD6, 0x52, 0x07, 0xED, 0x09]), + bytearray([0x9C, 0x9B, 0x13, 0x72, 0xF8, 0xC6, 0x40, 0xCF, 0x1C, 0x62, 0xF5, 0xD5, 0x92, 0xDD, 0xB5, 0x82]), + bytearray([0x03, 0xB3, 0x02, 0xE8, 0x5F, 0xF3, 0x81, 0xB1, 0x3B, 0x8D, 0xAA, 0x2A, 0x90, 0xFF, 0x5E, 0x61]), + bytearray([0xBC, 0xD7, 0xF9, 0xD3, 0x2F, 0xAC, 0xF8, 0x47, 0xC0, 0xFB, 0x4D, 0x2F, 0x30, 0x9A, 0xBD, 0xA6]), + bytearray([0xF5, 0x55, 0x96, 0xE9, 0x7F, 0xAF, 0x86, 0x7F, 0xAC, 0xB3, 0x3A, 0xE6, 0x9C, 0x8B, 0x6F, 0x93]), + bytearray([0xEE, 0x29, 0x70, 0x93, 0xF9, 0x4E, 0x44, 0x59, 0x44, 0x17, 0x1F, 0x8E, 0x86, 0xE1, 0x70, 0xFC]), + bytearray([0xE4, 0x34, 0x52, 0x0C, 0xF0, 0x88, 0xCF, 0xC8, 0xCD, 0x78, 0x1B, 0x6C, 0xCF, 0x8C, 0x48, 0xC4]), + bytearray([0xC1, 0xBF, 0x66, 0x81, 0x8E, 0xF9, 0x53, 0xF2, 0xE1, 0x26, 0x6B, 0x6F, 0x55, 0x0C, 0xC9, 0xCD]), + bytearray([0x56, 0x0F, 0xFF, 0x8F, 0x3C, 0x96, 0x49, 0x14, 0x45, 0x16, 0xF1, 0xBC, 0xBF, 0xCE, 0xA3, 0x0C]), + bytearray([0x24, 0x08, 0xDC, 0x75, 0x37, 0x60, 0xA2, 0x9F, 0x05, 0x54, 0xB5, 0xF2, 0x43, 0x85, 0x73, 0x99]), + bytearray([0xDD, 0xD5, 0xB5, 0x6A, 0x59, 0xC5, 0x5A, 0xE8, 0x3B, 0x96, 0x67, 0xC7, 0x5C, 0x2A, 0xE2, 0xDC]), + bytearray([0xAA, 0x68, 0x67, 0x72, 0xE0, 0x2D, 0x44, 0xD5, 0xCD, 0xBB, 0x65, 0x04, 0xBC, 0xD5, 0xBF, 0x4E]), + bytearray([0x1F, 0x17, 0xF0, 0x14, 0xE7, 0x77, 0xA2, 0xFE, 0x4B, 0x13, 0x6B, 0x56, 0xCD, 0x7E, 0xF7, 0xE9]), + bytearray([0xC9, 0x35, 0x48, 0xCF, 0x55, 0x8D, 0x75, 0x03, 0x89, 0x6B, 0x2E, 0xEB, 0x61, 0x8C, 0xA9, 0x02]), + bytearray([0xDE, 0x34, 0xC5, 0x41, 0xE7, 0xCA, 0x86, 0xE8, 0xBE, 0xA7, 0xC3, 0x1C, 0xEC, 0xE4, 0x36, 0x0F]), + bytearray([0xDD, 0xE5, 0xFF, 0x55, 0x1B, 0x74, 0xF6, 0xF4, 0xE0, 0x16, 0xD7, 0xAB, 0x22, 0x31, 0x1B, 0x6A]), + bytearray([0xB0, 0xE9, 0x35, 0x21, 0x33, 0x3F, 0xD7, 0xBA, 0xB4, 0x76, 0x2C, 0xCB, 0x4D, 0x80, 0x08, 0xD8]), + bytearray([0x38, 0x14, 0x69, 0xC4, 0xC3, 0xF9, 0x1B, 0x96, 0x33, 0x63, 0x8E, 0x4D, 0x5F, 0x3D, 0xF0, 0x29]), + bytearray([0xFA, 0x48, 0x6A, 0xD9, 0x8E, 0x67, 0x16, 0xEF, 0x6A, 0xB0, 0x87, 0xF5, 0x89, 0x45, 0x7F, 0x2A]), + bytearray([0x32, 0x1A, 0x09, 0x12, 0x50, 0x14, 0x8A, 0x3E, 0x96, 0x3D, 0xEA, 0x02, 0x59, 0x32, 0xE1, 0x8F]), + bytearray([0x4B, 0x00, 0xBE, 0x29, 0xBC, 0xB0, 0x28, 0x64, 0xCE, 0xFD, 0x43, 0xA9, 0x6F, 0xD9, 0x5C, 0xED]), + bytearray([0x57, 0x7D, 0xC4, 0xFF, 0x02, 0x44, 0xE2, 0x80, 0x91, 0xF4, 0xCA, 0x0A, 0x75, 0x69, 0xFD, 0xA8]), + bytearray([0x83, 0x53, 0x36, 0xC6, 0x18, 0x03, 0xE4, 0x3E, 0x4E, 0xB3, 0x0F, 0x6B, 0x6E, 0x79, 0x9B, 0x7A]), + bytearray([0x5C, 0x92, 0x65, 0xFD, 0x7B, 0x59, 0x6A, 0xA3, 0x7A, 0x2F, 0x50, 0x9D, 0x85, 0xE9, 0x27, 0xF8]), + bytearray([0x9A, 0x39, 0xFB, 0x89, 0xDF, 0x55, 0xB2, 0x60, 0x14, 0x24, 0xCE, 0xA6, 0xD9, 0x65, 0x0A, 0x9D]), + bytearray([0x8B, 0x75, 0xBE, 0x91, 0xA8, 0xC7, 0x5A, 0xD2, 0xD7, 0xA5, 0x94, 0xA0, 0x1C, 0xBB, 0x95, 0x91]), + bytearray([0x95, 0xC2, 0x1B, 0x8D, 0x05, 0xAC, 0xF5, 0xEC, 0x5A, 0xEE, 0x77, 0x81, 0x23, 0x95, 0xC4, 0xD7]), + bytearray([0xB9, 0xA4, 0x61, 0x64, 0x36, 0x33, 0xFA, 0x5D, 0x94, 0x88, 0xE2, 0xD3, 0x28, 0x1E, 0x01, 0xA2]), + bytearray([0xB8, 0xB0, 0x84, 0xFB, 0x9F, 0x4C, 0xFA, 0xF7, 0x30, 0xFE, 0x73, 0x25, 0xA2, 0xAB, 0x89, 0x7D]), + bytearray([0x5F, 0x8C, 0x17, 0x9F, 0xC1, 0xB2, 0x1D, 0xF1, 0xF6, 0x36, 0x7A, 0x9C, 0xF7, 0xD3, 0xD4, 0x7C]), + ] + + def __init__(self): + self.is_kirk_initialized = False + self.PRNG_DATA = bytearray(20) # 0x14 bytes + self.g_fuse90 = 0 + self.g_fuse94 = 0 + + # Initialize with a dummy seed + seed = b'KIRKENGINEINITIALIZATION' + self.kirk_init(seed, len(seed)) + + def kirk_init(self, seed: bytes, seed_length: int, fuse_id: int = 0) -> None: + """Initialize the KIRK engine""" + self.is_kirk_initialized = True + + # Initialize PRNG data + if seed_length > 0: + seed_buf = bytearray(seed_length + 4) + struct.pack_into(" bytearray: + """Encrypt with AESCBC128 using keys from table""" + if not self.is_kirk_initialized: + raise KirkNotInitializedError("KIRK not initialized") + + if size is None: + size = len(inbuff) + + # Create a copy of input buffer for the output + outbuff = bytearray(inbuff) + + # Read header + mode = struct.unpack_from(" bytearray: + """Encrypt with AESCBC128 using FUSE ID""" + if not self.is_kirk_initialized: + raise KirkNotInitializedError("KIRK not initialized") + + if size is None: + size = len(inbuff) + + # Create a copy of input buffer for the output + outbuff = bytearray(inbuff) + + # Read header + mode = struct.unpack_from(" bytearray: + """Decrypt with AESCBC128 using keys from table""" + if not self.is_kirk_initialized: + raise KirkNotInitializedError("KIRK not initialized") + + if size is None: + size = len(inbuff) + + # Read header + mode = struct.unpack_from(" bytearray: + """Decrypt with AESCBC128 using FUSE ID""" + if not self.is_kirk_initialized: + raise KirkNotInitializedError("KIRK not initialized") + + if size is None: + size = len(inbuff) + + # Read header + mode = struct.unpack_from(" bytearray: + """Generate SHA1 hash""" + if not self.is_kirk_initialized: + raise KirkNotInitializedError("KIRK not initialized") + + if size is None: + size = len(inbuff) + + if size == 0: + raise KirkDataSizeZeroError("Data size is zero") + + # Read data size from header + data_size = struct.unpack_from(" bytearray: + """Generate pseudo random number""" + if not self.is_kirk_initialized: + raise KirkNotInitializedError("KIRK not initialized") + + if size <= 0: + return bytearray() + + # Create temporary buffer + temp = bytearray(0x104) + temp[0] = 0 + temp[1] = 0 + temp[2] = 1 + temp[3] = 0 + + # Random key data + key = bytes([ + 0xA7, 0x2E, 0x4C, 0xB6, 0xC3, 0x34, 0xDF, 0x85, + 0x70, 0x01, 0x49, 0xFC, 0xC0, 0x87, 0xC4, 0x77 + ]) + + # Get current time for randomness + import time + systime = int(time.time()) + + # Prepare buffer for hashing + temp[4:4 + 20] = self.PRNG_DATA + temp[0x18] = systime & 0xFF + temp[0x19] = (systime >> 8) & 0xFF + temp[0x1A] = (systime >> 16) & 0xFF + temp[0x1B] = (systime >> 24) & 0xFF + temp[0x1C:0x1C + 16] = key + + # Create SHA1 header + struct.pack_into(" 0: + if remaining >= 20: + # Copy full PRNG_DATA block + outbuff[size - remaining:size - remaining + 20] = self.PRNG_DATA + remaining -= 20 + # Update PRNG_DATA for next block if needed + if remaining > 0: + self.PRNG_DATA = self.kirk_CMD11(temp, 0x104) + else: + # Copy partial block + outbuff[size - remaining:] = self.PRNG_DATA[:remaining] + remaining = 0 + + return outbuff + + def sceUtilsBufferCopyWithRange(self, outsize: int, inbuff: bytes, insize: int, cmd: int) -> bytearray: + """Execute a KIRK command""" + if cmd == self.PSP_KIRK_CMD_ENCRYPT: + result = self.kirk_CMD4(inbuff, insize) + elif cmd == self.PSP_KIRK_CMD_ENCRYPT_FUSE: + result = self.kirk_CMD5(inbuff, insize) + elif cmd == self.PSP_KIRK_CMD_DECRYPT: + result = self.kirk_CMD7(inbuff, insize) + elif cmd == self.PSP_KIRK_CMD_DECRYPT_FUSE: + result = self.kirk_CMD8(inbuff, insize) + elif cmd == self.PSP_KIRK_CMD_SHA1_HASH: + result = self.kirk_CMD11(inbuff, insize) + elif cmd == self.PSP_KIRK_CMD_PRNG: + result = self.kirk_CMD14(outsize) + else: + raise KirkError(f"Unimplemented KIRK command: {cmd}") + + # Copy result to output buffer if provided and ensure it doesn't exceed outsize + if result is not None: + length = min(len(result), outsize) + return result[:0x14 + length] + + return result diff --git a/post_psx/utils/lz.py b/post_psx/utils/lz.py index 13fac29..c65102f 100644 --- a/post_psx/utils/lz.py +++ b/post_psx/utils/lz.py @@ -3,7 +3,7 @@ from typing import Optional, Tuple -@numba.jit(nopython=True) +@numba.jit(nopython=True, cache=True) def decode_range(in_bytes: np.ndarray, src: np.int32, range_val: np.uint32, code: np.uint32) -> Tuple[ np.int32, np.uint32, np.uint32]: """Updates range and code values based on input bytes.""" @@ -14,7 +14,7 @@ def decode_range(in_bytes: np.ndarray, src: np.int32, range_val: np.uint32, code return src, range_val, code -@numba.jit(nopython=True) +@numba.jit(nopython=True, cache=True) def decode_bit(in_bytes: np.ndarray, tmp: np.ndarray, src: np.int32, range_val: np.uint32, code: np.uint32, tmp_index: np.int32, acc: Optional[np.ndarray] = None) -> Tuple[ np.int32, np.uint32, np.uint32, np.int32]: @@ -41,7 +41,7 @@ def decode_bit(in_bytes: np.ndarray, tmp: np.ndarray, src: np.int32, range_val: return src, range_val, code, 0 -@numba.jit(nopython=True) +@numba.jit(nopython=True, cache=True) def decode_number(in_bytes: np.ndarray, tmp: np.ndarray, src: np.int32, range_val: np.uint32, code: np.uint32, base_offset: np.int32, index_val: np.int32) -> Tuple[ np.int32, np.uint32, np.uint32, np.int32, np.int32]: @@ -72,18 +72,18 @@ def decode_number(in_bytes: np.ndarray, tmp: np.ndarray, src: np.int32, range_va return src, range_val, code, acc[0], bit_flag -@numba.jit(nopython=True) +@numba.jit(nopython=True, cache=True) def decode_word(in_bytes: np.ndarray, tmp: np.ndarray, src: np.int32, range_val: np.uint32, - code: np.uint32, base_offset: np.int32, index_val: np.int32) -> Tuple[ + code: np.uint32, base_offset: np.int32, index_val: np.int32, version) -> Tuple[ np.int32, np.uint32, np.uint32, np.int32, np.int32]: """Decodes a word from the compressed data.""" index_val = np.int32(index_val // 8) acc = np.array([1], dtype=np.int32) if index_val >= 3: - src, range_val, code, _ = decode_bit(in_bytes, tmp, src, range_val, code, base_offset + 4, acc) + src, range_val, code, _ = decode_bit(in_bytes, tmp, src, range_val, code, base_offset if version == 1 else base_offset + 4, acc) if index_val >= 4: - src, range_val, code, _ = decode_bit(in_bytes, tmp, src, range_val, code, base_offset + 4, acc) + src, range_val, code, _ = decode_bit(in_bytes, tmp, src, range_val, code, base_offset if version == 1 else base_offset + 4, acc) if index_val >= 5: src, range_val, code = decode_range(in_bytes, src, range_val, code) for _ in range(index_val - 4): @@ -94,18 +94,18 @@ def decode_word(in_bytes: np.ndarray, tmp: np.ndarray, src: np.int32, range_val: else: code -= range_val - src, range_val, code, bit_flag = decode_bit(in_bytes, tmp, src, range_val, code, base_offset, acc) + src, range_val, code, bit_flag = decode_bit(in_bytes, tmp, src, range_val, code, base_offset + 3 if version == 1 else base_offset, acc) if index_val >= 1: - src, range_val, code, _ = decode_bit(in_bytes, tmp, src, range_val, code, base_offset + 1, acc) + src, range_val, code, _ = decode_bit(in_bytes, tmp, src, range_val, code, base_offset + 2 if version == 1 else base_offset + 1, acc) if index_val >= 2: - src, range_val, code, _ = decode_bit(in_bytes, tmp, src, range_val, code, base_offset + 2, acc) + src, range_val, code, _ = decode_bit(in_bytes, tmp, src, range_val, code, base_offset + 1 if version == 1 else base_offset + 2, acc) return src, range_val, code, acc[0], bit_flag -@numba.jit(nopython=True) -def decompress(in_bytes: np.ndarray, out_size: np.int32) -> Optional[np.ndarray]: +@numba.jit(nopython=True, cache=True) +def decompress(in_bytes: np.ndarray, out_size: np.int32, version=1) -> Optional[np.ndarray]: """Decompresses the input data.""" out = np.zeros(out_size, dtype=np.uint8) src = np.int32(0) @@ -119,8 +119,12 @@ def decompress(in_bytes: np.ndarray, out_size: np.int32) -> Optional[np.ndarray] code = np.uint32(int(in_bytes[1]) << 24 | int(in_bytes[2]) << 16 | int(in_bytes[3]) << 8 | int(in_bytes[4])) # Initialize temporary buffer - tmp = np.zeros(3272, dtype=np.uint8) - tmp[:0xCA8] = np.uint8(0x80) + if version == 1: + tmp = np.zeros(0xA70, dtype=np.uint8) + tmp[:0xA60] = np.uint8(0x80) + else: + tmp = np.zeros(0xCC8, dtype=np.uint8) + tmp[:0xCA8] = np.uint8(0x80) # Handle uncompressed data if head > 0x80: @@ -131,7 +135,10 @@ def decompress(in_bytes: np.ndarray, out_size: np.int32) -> Optional[np.ndarray] return None while pos < out_size: - sect1_index = offset + 0xB68 + if version == 1: + sect1_index = offset + 0x920 + else: + sect1_index = offset + 0xB68 src, range_val, code, bit = decode_bit(in_bytes, tmp, src, range_val, code, sect1_index) @@ -154,7 +161,10 @@ def decompress(in_bytes: np.ndarray, out_size: np.int32) -> Optional[np.ndarray] else: # Handle compressed stream index_val = np.int32(-1) - sect1 = offset + 0xB68 + if version == 1: + sect1 = offset + 0x920 + else: + sect1 = offset + 0xB68 while True: sect1 += 8 @@ -163,23 +173,35 @@ def decompress(in_bytes: np.ndarray, out_size: np.int32) -> Optional[np.ndarray] if not bit or index_val >= 6: break - b_size = np.int32(0x160) + if version == 1: + b_size = np.int32(0x40) + else: + b_size = np.int32(0x160) tmp_sect2 = index_val + 0x7F1 if index_val >= 0 or bit: sect = np.int32((index_val << 5) | ((((pos << index_val) & 3) << 3) | (offset & 7))) - tmp_sect1 = np.int32(0xBA8 + sect) + if version == 1: + tmp_sect1 = np.int32(0x960 + sect) + else: + tmp_sect1 = np.int32(0xBA8 + sect) src, range_val, code, data_length, _ = decode_number(in_bytes, tmp, src, range_val, code, tmp_sect1, index_val) - if data_length == 0xFF: - return out[:pos] + if version == 2: + if data_length == 0xFF: + return out[:pos] + else: + if data_length != 3 and ((index_val > 0) or (index_val != 0)): + tmp_sect2 += 0x38 + b_size = np.int32(0x80) else: data_length = np.int32(1) - if data_length <= 2: - tmp_sect2 += 0xF8 - b_size = np.int32(0x40) + if version == 2: + if data_length <= 2: + tmp_sect2 += 0xF8 + b_size = np.int32(0x40) shift_val = np.array([1], dtype=np.int32) @@ -193,8 +215,11 @@ def decompress(in_bytes: np.ndarray, out_size: np.int32) -> Optional[np.ndarray] if diff > 0 or bit: if not bit: diff -= 8 - tmp_sect3 = np.int32(0x928 + diff) - src, range_val, code, data_offset, _ = decode_word(in_bytes, tmp, src, range_val, code, tmp_sect3, diff) + if version == 1: + tmp_sect3 = np.int32(0x8A8 + diff) + else: + tmp_sect3 = np.int32(0x928 + diff) + src, range_val, code, data_offset, _ = decode_word(in_bytes, tmp, src, range_val, code, tmp_sect3, diff, version) else: data_offset = np.int32(1) @@ -215,9 +240,24 @@ def decompress(in_bytes: np.ndarray, out_size: np.int32) -> Optional[np.ndarray] return out -def decompress_bytes(in_bytes: bytes, out_size: int) -> Optional[bytes]: +def decompress_bytes(in_bytes: bytes, out_size: int, version=1) -> Optional[bytes]: """Wrapper function to handle bytes input/output""" - result = decompress(np.frombuffer(in_bytes, dtype=np.uint8), np.int32(out_size)) + result = decompress(np.frombuffer(in_bytes, dtype=np.uint8), np.int32(out_size), version) if result is not None: return bytes(result) return None + + +if __name__ == '__main__': + from time import time + b = bytes.fromhex('05ff80010ed6e737043f530bbce7a37214dc388e0caa949346bff87215047e9ce0ec8b6c7deef07a90910eb3c78bd8089d6809e59efe43035b0b7c52e4fefe6626e5cc83fc5516d25e920000c240a1f0') + decompress_bytes(b, 32768, 2) + + start = time() + runs = 0 + while time() - start < 5: + runs += 1 + decompress_bytes(b, 32768, 2) + + elapsed = time() - start + print(f"Processed {int(round(runs / elapsed))} images / sec") diff --git a/post_psx/utils/edat.py b/post_psx/utils/npd.py similarity index 98% rename from post_psx/utils/edat.py rename to post_psx/utils/npd.py index 4435a2a..3cac53e 100644 --- a/post_psx/utils/edat.py +++ b/post_psx/utils/npd.py @@ -6,9 +6,10 @@ from post_psx.types import NPDHeader, EDATHeader from post_psx.utils import lz +from post_psx.utils.base import BaseFile -class EdatFile(io.RawIOBase): +class NPDFile(BaseFile): SDAT_FLAG = 0x01000000 EDAT_COMPRESSED_FLAG = 0x00000001 EDAT_FLAG_0x02 = 0x00000002 @@ -44,7 +45,7 @@ class EdatFile(io.RawIOBase): EDAT_IV = bytearray(0x10) def __init__(self, fp, file_name, edat_key): - self.fp = fp + super().__init__(fp) self.fp.seek(0) self.npd_header = NPDHeader.unpack(fp.read(NPDHeader.struct.size)) self.fp.seek(0x80) @@ -277,6 +278,8 @@ def decrypt_block(self, block_num, decrypt_key): # Prepare decryption buffers self.fp.seek(offset) enc_data = self.fp.read(length) + if len(enc_data) != length: + return -1 dec_data = io.BytesIO() b_key = self.get_block_key(block_num) @@ -320,7 +323,7 @@ def decrypt_block(self, block_num, decrypt_key): # Handle decompression if needed if should_decompress: - res = lz.decompress_bytes(dec_data.getvalue(), self.edat_header.block_size) + res = lz.decompress_bytes(dec_data.getvalue(), self.edat_header.block_size, version=2) if not res: return -1 diff --git a/post_psx/utils/pgd.py b/post_psx/utils/pgd.py new file mode 100644 index 0000000..1021f7c --- /dev/null +++ b/post_psx/utils/pgd.py @@ -0,0 +1,70 @@ +from post_psx.types import PGDHeader, PGDSubHeader +from post_psx.utils.kirk.bb import BBMac, BBCipher + + +class PGDFile: + dnas_key_1 = bytes( + [0xED, 0xE2, 0x5D, 0x2D, 0xBB, 0xF8, 0x12, 0xE5, 0x3C, 0x5C, 0x59, 0x32, 0xFA, 0xE3, 0xE2, 0x43]) + dnas_key_2 = bytes( + [0x27, 0x74, 0xFB, 0xEB, 0xA4, 0xA0, 0x01, 0xD7, 0x02, 0x56, 0x9E, 0x33, 0x8C, 0x19, 0x57, 0x83]) + + # PGD_SIZE = 0xB6600 + fp = None + + def decrypt_pgd(self, pgd_offset): + self.fp.seek(pgd_offset) + pgd_header_raw = bytearray(self.fp.read(PGDHeader.struct.size + PGDSubHeader.struct.size)) + pgd_header = PGDHeader.unpack(pgd_header_raw[0:PGDHeader.struct.size]) + pgd_subheader = PGDSubHeader.unpack(pgd_header_raw[0x30:]) + if not pgd_header.check_magic(): + return False + + # Get the fixed DNAS key + fkey = None + if pgd_header.open_flag & 0x2 == 0x2: + fkey = self.dnas_key_1 + elif pgd_header.open_flag & 0x1 == 0x1: + fkey = self.dnas_key_2 + + if fkey is None: + print(f"PGD: Invalid DNAS flag! {pgd_header.open_flag:08x}") + return False + + # Test MAC hash at 0x80 (DNAS hash) + bbmac = BBMac.new(pgd_header.mac_type, bytearray(16)) + bbmac.update(pgd_header_raw[:0x80]) + result = bbmac.verify(pgd_subheader.encrypted_ioctl_hash, fkey) + + if not result: + print("PGD: Invalid 0x80 MAC hash!") + return False + + # Test MAC hash at 0x70 (key hash) + bbmac = BBMac.new(pgd_header.mac_type, bytearray(16)) + bbmac.update(pgd_header_raw[:0x70]) + + # Generate the key from MAC 0x70 + header_key = bbmac.extract_key(pgd_subheader.ioctl_hash) + + # Now we can decrypt the PGD header using the vkey + cipher = BBCipher.new(pgd_header.cipher_type, BBCipher.MODE_DECRYPT, header_key, 0) + cipher.update(pgd_header.key) + pgd_subheader.set_decrypted_header(cipher.update(pgd_header_raw[0x30:0x60])) + + # Test MAC hash at 0x60 (table hash) + self.fp.seek(pgd_offset + pgd_subheader.table_offset) + table_data = self.fp.read(pgd_subheader.block_nr * 16) + bbmac = BBMac.new(pgd_header.mac_type, bytearray(16)) + bbmac.update(table_data) + result = bbmac.verify(pgd_subheader.file_hash, header_key) + + if not result: + print("PGD: Invalid 0x60 MAC hash!") + return False + + # Finally, decrypt the actual data using the vkey + self.fp.seek(pgd_offset + 0x90) + encrypted_data = self.fp.read(pgd_subheader.align_size) + cipher = BBCipher.new(pgd_header.cipher_type, BBCipher.MODE_DECRYPT, header_key, 0) + cipher.update(pgd_subheader.hash_key) + return cipher.update(encrypted_data) diff --git a/post_psx/utils/pspedat.py b/post_psx/utils/pspedat.py new file mode 100644 index 0000000..489be1f --- /dev/null +++ b/post_psx/utils/pspedat.py @@ -0,0 +1,30 @@ +import io +import struct + +from post_psx.utils.base import BaseFile +from post_psx.utils.pgd import PGDFile + + +class PSPEdatFile(PGDFile, BaseFile): + def __init__(self, fp, *_, **__): + super().__init__(fp) + self.fp.seek(0) + self.decrypted_pgd = None + magic = self.fp.read(0x8) + if magic != b"\x00PSPEDAT": + return + self.fp.seek(0x0C) + pgd_offset, = struct.unpack("