PSX: Add support for PKG files and EDAT files.

Pipfile

@@ -9,7 +9,7 @@ verify_ssl = true
 pycdlib = "*"
 pathlab = "*"
 xxhash = "*"
-numpy = "*"
+numpy = "1.24"
 libarchive-c = "*"
 pefile = "*"
 pycryptodome = "*"
@@ -20,8 +20,10 @@ machfs = "*"
 rarfile = ">=4.2"
 psutil = "*"
 stream-inflate = "*"
-inflate64 = "*"
+inflate64 = "1.0.0"
 bitarray = "*"
+xor-cipher = "5.0.0"
+numba = "0.58.1"
 
 [requires]
 python_version = "3.8"

build_ps3_key_db.py

@@ -1,4 +1,5 @@
 import argparse
+import csv
 import pathlib
 import sqlite3
 import xml.etree.ElementTree as ET
@@ -19,24 +20,58 @@
                         required=True,
                         default=argparse.SUPPRESS)
 
+    parser.add_argument('-l',
+                        '--dev-klics',
+                        help="dev_klics.txt file",
+                        type=str,
+                        required=False,
+                        default=argparse.SUPPRESS)
+
+    parser.add_argument('-r',
+                        '--rap-dir',
+                        help="Directory containing rap files",
+                        type=str,
+                        required=False,
+                        default=argparse.SUPPRESS)
+
     parser.add_argument('-o',
                         '--output',
                         help="Output directory",
                         type=str,
-                        default=str(((pathlib.Path(__file__).parent / 'ps3').absolute())))
+                        default=str(((pathlib.Path(__file__).parent / 'post_psx').absolute())))
+
+    parser.add_argument('-t',
+                        '--tsv',
+                        help="PS3/PSP_GAMES.TSV file path",
+                        nargs='+',
+                        action='append',
+                        type=str)
+
+    parser.add_argument('-a',
+                        '--append',
+                        help="Append to db instead of overwriting it",
+                        action='store_true',
+                        default=False)
 
     args = parser.parse_args()
 
     key_dir = pathlib.Path(args.keyfolder)
 
     db_file = pathlib.Path(args.output) / 'keys.db'
-    db_file.unlink(missing_ok=True)
+    if not args.append:
+        db_file.unlink(missing_ok=True)
     db = sqlite3.connect(db_file)
     c = db.cursor()
     try:
         c.execute("BEGIN")
         c.execute("""
-        CREATE TABLE keys (name TEXT, size TEXT, crc32 TEXT, md5 TEXT, sha1 TEXT, key BLOB);
+        CREATE TABLE IF NOT EXISTS keys (name TEXT, size TEXT, crc32 TEXT, md5 TEXT, sha1 TEXT, key BLOB, UNIQUE (size, md5, sha1, key));
+        """)
+        c.execute("""
+        CREATE TABLE IF NOT EXISTS dev_klics (key BLOB, title_id TEXT, name TEXT, UNIQUE(key, title_id));
+        """)
+        c.execute("""
+        CREATE TABLE IF NOT EXISTS raps (key BLOB, title_id TEXT, UNIQUE(key));
         """)
 
         key_list = []
@@ -63,7 +98,64 @@
                 key)
             )
         c.executemany("""
-            INSERT INTO keys VALUES (?, ?, ?, ?, ?, ?)""", key_list)
+            INSERT OR IGNORE INTO keys VALUES (?, ?, ?, ?, ?, ?)""", key_list)
+
+
+        if args.dev_klics:
+            klics = []
+            dev_klics = pathlib.Path(args.dev_klics)
+            if dev_klics.exists():
+                with dev_klics.open("r") as f:
+                    for line in f:
+                        if line.startswith("-"):
+                            continue
+                        entry = line.strip().split(' ', maxsplit=2)
+                        if len(entry) != 3 or len(entry[0]) != 32 or len(entry[1]) != 36:
+                            continue
+
+                        klics.append((
+                            bytes.fromhex(entry[0]),
+                            entry[1],
+                            entry[2]
+                        ))
+            if klics:
+                c.executemany("""
+                    INSERT OR IGNORE INTO dev_klics VALUES (?, ?, ?)""", klics)
+
+        raps = set()
+        if args.rap_dir:
+            rap_dir = pathlib.Path(args.rap_dir)
+            if rap_dir.exists():
+                for rap_file in rap_dir.glob("*.rap"):
+                    with rap_file.open("rb") as f:
+                        rap_data = f.read()
+                        if len(rap_data) == 256:
+                            rap_data = rap_data[:16]
+                        title_id = rap_file.stem
+                        raps.add((
+                            rap_data,
+                            title_id
+                        ))
+            if raps:
+                c.executemany("""
+                    INSERT OR IGNORE INTO raps VALUES (?, ?)""", raps)
+
+        if args.tsv:
+            for tsv in args.tsv:
+                tsv_path = pathlib.Path(tsv[0])
+                if tsv_path.exists():
+                    with tsv_path.open("r", encoding="utf-8") as infile:
+                        reader = csv.DictReader(infile, delimiter="\t")
+                        for row in reader:
+                            if row["RAP"] and row["Content ID"] and len(row["RAP"]) == 32 and len(row["Content ID"]) == 36:
+                                raps.add((
+                                    bytes.fromhex(row["RAP"]),
+                                    row["Content ID"]
+                                ))
+        if raps:
+            c.executemany("""
+                INSERT OR IGNORE INTO raps VALUES (?, ?)""", list(raps))
+
         c.execute("COMMIT")
     except:
         c.execute("ROLLBACK")

common/factory.py

@@ -5,7 +5,9 @@
 
 import rarfile
 
-from common import pycdlib
+from post_psx.npdrm.path_reader import NPDRMPathReader
+from post_psx.npdrm.pkg import Pkg
+from utils import pycdlib
 from pyisotools.iso import GamecubeISO
 
 from cdi.path_reader import CdiPathReader
@@ -126,7 +128,7 @@ def get_iso_path_readers(fp, file_name, parent_container, pbar):
                         exceptions[CompressedPathReader.volume_type] = e
 
         fp.seek(0)
-        if fp.read(4) == b"LIVE":
+        if fp.read(4) == b"LIVE" and parent_container.get_file_size(parent_container.get_file(fp.name)) >= 0x971A:
             stfs = STFS(filename=None, fd=fp)
             if stfs.content_type == 0x7000:
                 # Not an XBLA archive, actually a GOD game. Process like a concatenated ISO
@@ -191,6 +193,18 @@ def get_iso_path_readers(fp, file_name, parent_container, pbar):
         else:
             wrapper = fp
 
+        wrapper.seek(0)
+        if wrapper.peek(4) == b"\x7FPKG":
+            try:
+                reader = Pkg(wrapper)
+                reader.parse_header()
+                reader.parse_metadata()
+                path_reader = NPDRMPathReader(reader, wrapper, parent_container)
+                path_reader.edat_key, path_reader.self_key = path_reader.get_edat_key(pbar)
+                path_readers.append(path_reader)
+            except Exception as e:
+                exceptions[NPDRMPathReader.volume_type] = e
+
         wrapper.seek(0)
         if wrapper.peek(7) == b"\x01\x5A\x5A\x5A\x5A\x5A\x01" and file_name != "Disc label":
             try:

common/iso_path_reader/methods/compressed.py

@@ -48,6 +48,8 @@ def get_file_date(self, file):
         return None
 
     def get_file_path(self, file):
+        if file == self.get_root_dir():
+            return ""
         if self.is_directory(file):
             return file.path.rstrip("/").rstrip("\\")
         return file.path

common/iso_path_reader/methods/pycdlib.py

@@ -44,9 +44,9 @@ def iso_iterator(self, base_dir, recursive=False, include_dirs=False):
                 continue
 
             if file.is_dir():
+                if include_dirs:
+                    yield file
                 if recursive:
-                    if include_dirs:
-                        yield file
                     yield from self.iso_iterator(file, recursive, include_dirs)
                 continue
 

common/processor.py

@@ -10,12 +10,14 @@
 from common.iso_path_reader.methods.compressed import CompressedPathReader
 from common.iso_path_reader.methods.pathlab import PathlabPathReader
 from common.iso_path_reader.methods.pycdlib import PyCdLibPathReader
+from post_psx.npdrm.path_reader import NPDRMPathReader
 from utils.common import format_bar_desc
 from utils.hash_progress_wrapper import HashProgressWrapper
 from xbox.path_reader import XboxPathReader, XboxStfsPathReader
 
 LOGGER = logging.getLogger(__name__)
 
+
 class BaseIsoProcessor:
     globally_ignored_paths = [
         re.compile(".*\.nfo$", re.IGNORECASE),
@@ -41,6 +43,11 @@ def get_system_type(iso_path_reader):
         if isinstance(iso_path_reader, XboxStfsPathReader):
             return "xbla"
 
+        if isinstance(iso_path_reader, NPDRMPathReader):
+            system_type = iso_path_reader.system_type
+            if system_type:
+                return system_type
+
         if not isinstance(iso_path_reader, CompressedPathReader):
             fp = iso_path_reader.fp
             fp.seek(0)
@@ -149,7 +156,6 @@ def get_system_type(iso_path_reader):
                     if f.read(2) == b"MZ":
                         return "pc"
 
-
     def __init__(self, iso_path_reader, filename, system_type, progress_manager):
         self.iso_path_reader = iso_path_reader
         self.filename = filename
@@ -313,6 +319,7 @@ def get_pvd_info(self):
     def close(self):
         self.iso_path_reader.close()
 
+
 class GenericIsoProcessor(BaseIsoProcessor):
     def hash_exe(self):
         return {}