From 338f2c6ee3266a802b7175a0fc612068a6a516a2 Mon Sep 17 00:00:00 2001 From: Schindler Date: Wed, 18 Nov 2026 20:42:18 -0500 Subject: [PATCH 1/4] Installer: check the picked Dissidia 1 and undub discs are the right game The Dissidia 1 pick only looked for obj/p_fiv100.objx, which Dissidia 012 has too, so a 012 image was taken and the install failed later with a misleading message. Now, as soon as a file is picked (ISO or CSO, also an Android fd:N): - Dissidia 1: PARAM.SFO DISC_ID must be ULUS10437; a 012 image, another D1 release or another game is refused with what it is. - Undub: must be Dissidia 012 and must not have the English voices (Bartz's battle bank compared with the EU / US English one: size + CRC-32). A refused pick shows a WRONG DISC page whose OK goes back to choosing that file. d1_moves.py install checks the DISC_ID too. Tests: synthetic ISO / CSO images, SETUP_TEST_DISCS for real ones (D1 USA, 012 USA/EU, undub checked). Co-Authored-By: Claude Opus 5.5 --- psprecomp/docs/installer.md | 7 + psprecomp/games/dissidia/tools/d1_moves.py | 25 +++ psprecomp/setup/CMakeLists.txt | 2 +- psprecomp/setup/install.cpp | 204 +++++++++++++++++++++ psprecomp/setup/install.h | 10 + psprecomp/setup/setup_main.cpp | 38 +++- psprecomp/setup/tests/test_install.cpp | 131 ++++++++++++- 7 files changed, 410 insertions(+), 7 deletions(-) diff --git a/psprecomp/docs/installer.md b/psprecomp/docs/installer.md index cd618ed..14358e9 100644 --- a/psprecomp/docs/installer.md +++ b/psprecomp/docs/installer.md @@ -71,6 +71,13 @@ of the fork. of what the current stage does in the information panel. The step weights live in `setup/install.cpp`. - **Add to Steam** (desktop only; an Options toggle, and a row on the start page of an installed game): `plan_steam` runs `games/dissidia/tools/steam_shortcut.py` (Linux, Windows; the script also serves the Steam Deck setup) to add a non-Steam shortcut to every Steam account on the machine, pointing at `DissidiaRecomp --launch` (check for an update, then play), with the game's icon and the disc's PIC1.PNG as banner. Steam is closed for a moment (`-shutdown`) and started again, because it rewrites its shortcut list on exit. The step is optional: no Steam installed only adds a note to the last page. - **Change extras** (start page of an installed game): the Extras page again, applied without a rebuild (`plan_extras`): downloads / swaps / removes the HD texture pack, adds the Japanese voices, builds the Android version once. + - **Disc checks at pick time** (`install.cpp` `read_disc_id`, `d1_disc_problem`, `undub_disc_problem`; ISO or CSO, also an Android `fd:N`): + the picked file's `PSP_GAME/PARAM.SFO` says which game it is. The Dissidia 1 pick must be `ULUS10437` (a Dissidia 012 image, another + D1 release or another game is refused with what it is). The undub pick must be Dissidia 012 and must not carry the English voices: + Bartz's battle bank (`voice/en/battle/voice_p_fiv100_00.scd`, package CRC `4300D781`) is 302368 bytes with CRC-32 `6EE2B876` on the + EU and US discs, and differs on an undub. A refused pick shows a WRONG DISC page whose OK goes back to choosing that file. + `tools/d1_moves.py install` refuses a disc whose DISC_ID is not `ULUS10437` too. `tests/test_install.cpp` builds small ISO / CSO + images; `SETUP_TEST_DISCS="||"` adds the real ones. - **Options** (one page in the game's Settings-screen look, without the category sidebar: HD textures on / off, then Texture pack when on, Japanese voices, Android version, Import saves, update check, install folder, Start installing): check for updates; HD textures (on / off) and, when on, Texture pack (LiinkPK's hand-retouched pack or the AI upscaled pack; the help line credits the selected pack); Japanese voices (undub; asks for the undub disc image first); **Bartz: Dissidia 1 moves** (`d1_moves=1`; asks for a Dissidia Final Fantasy (2008) disc image, then `tools/d1_moves.py install --d1 --work ` rebuilds the mod's six files from the shipped patches in `game/d1_moves/` and the player's two discs, checks each against its SHA-256, adds the abilities to main.bin and writes `mods/retail_moves.txt`; also offered by Change extras); Android version; **Import saves** (cycles Off / each PPSSPP or memory-stick SAVEDATA folder found with Dissidia 012 saves, found by `psp_savemgr::detect_sources`; it adds the last install step, `DissidiaRecomp --import-saves FROM --dir WORK`, which copies the diff --git a/psprecomp/games/dissidia/tools/d1_moves.py b/psprecomp/games/dissidia/tools/d1_moves.py index b725f98..951ff37 100644 --- a/psprecomp/games/dissidia/tools/d1_moves.py +++ b/psprecomp/games/dissidia/tools/d1_moves.py @@ -101,6 +101,9 @@ def mark_d1(desc): return desc +D1_DISC_ID = 'ULUS10437' # Dissidia Final Fantasy (2008), USA: the release the shipped patches are made from + + class IsoPackage(pe.Package): """PACKAGE.BIN / PACKAGE_INFO.BIN read in place from an ISO 9660 image.""" @@ -145,6 +148,22 @@ def _find(self, path): rec = found return struct.unpack_from('>:pthread>) add_test(NAME setup_install COMMAND test_setup_install) add_executable(test_payload tests/test_payload.cpp payload.cpp third_party/miniz.c) diff --git a/psprecomp/setup/install.cpp b/psprecomp/setup/install.cpp index 89a6eb1..df47f09 100644 --- a/psprecomp/setup/install.cpp +++ b/psprecomp/setup/install.cpp @@ -9,7 +9,13 @@ #include #ifdef _WIN32 #include +#else +#include #endif +#include +#include +#include +#include "third_party/miniz.h" namespace fs = std::filesystem; @@ -98,6 +104,204 @@ const char* texture_pack_credit(const std::string& pack) { : "Credits: Dissidia 012 HD Project by LiinkPK (github.com/LiinkPK/Dissidia-012-HD-Textures); artwork also by ajb3art."; } +// ---- disc identity: which game a picked disc image is (before anything is installed with it) ---- +namespace { +// A disc image read in place: plain ISO or CSO (CISO v1, raw deflate blocks; the wizard's undub pick accepts both). +struct DiscReader { + int fd = -1; // "fd:N": pread, so the build tools that inherit the descriptor find it where it was + std::ifstream f; + bool cso = false; + uint32_t block = 0, align = 0; + uint64_t total = 0; + std::vector index; + int64_t cached = -1; + std::string cache; + explicit DiscReader(const std::string& path) { + if (path.compare(0, 3, "fd:") == 0) fd = std::atoi(path.c_str() + 3); + else f.open(fs::u8path(path), std::ios::binary); + std::string h; + if (!ok() || !raw(0, 24, &h) || h.compare(0, 4, "CISO") != 0) return; + block = (uint32_t)(uint8_t)h[16] | (uint32_t)(uint8_t)h[17] << 8 | (uint32_t)(uint8_t)h[18] << 16 | (uint32_t)(uint8_t)h[19] << 24; + for (int i = 7; i >= 0; i--) total = total << 8 | (uint8_t)h[8 + i]; + align = (uint8_t)h[21]; + if ((uint8_t)h[20] > 1 || block < 2048 || (block & (block - 1)) || total == 0 || total > (1ull << 40)) { fd = -1; f.close(); return; } + const uint64_t n = (total + block - 1) / block + 1; + std::string ix; + if (!raw(24, (size_t)n * 4, &ix)) { fd = -1; f.close(); return; } + index.resize((size_t)n); + for (size_t i = 0; i < n; i++) + index[i] = (uint32_t)(uint8_t)ix[4 * i] | (uint32_t)(uint8_t)ix[4 * i + 1] << 8 | (uint32_t)(uint8_t)ix[4 * i + 2] << 16 | (uint32_t)(uint8_t)ix[4 * i + 3] << 24; + cso = true; + } + bool ok() const { return fd >= 0 || f.is_open(); } + bool raw(uint64_t off, size_t n, std::string* out) { + out->assign(n, '\0'); + if (n == 0) return true; +#ifndef _WIN32 + if (fd >= 0) { + size_t got = 0; + while (got < n) { + const ssize_t r = ::pread(fd, &(*out)[got], n - got, static_cast(off + got)); + if (r <= 0) return false; + got += static_cast(r); + } + return true; + } +#endif + if (!f.is_open()) return false; + f.clear(); + f.seekg(static_cast(off)); + f.read(&(*out)[0], static_cast(n)); + return static_cast(f.gcount()) == n; + } + bool cso_block(uint64_t i) { + if ((int64_t)i == cached) return true; + if (i + 1 >= index.size()) return false; + const uint64_t a = (uint64_t)(index[i] & 0x7FFFFFFFu) << align, b = (uint64_t)(index[i + 1] & 0x7FFFFFFFu) << align; + if (b < a || b - a > (uint64_t)block * 2 + 64) return false; + std::string src; + if (!raw(a, (size_t)(b - a), &src)) return false; + if (index[i] & 0x80000000u) { src.resize(std::min(src.size(), block)); cache.swap(src); } + else { + cache.assign(block, '\0'); + const size_t got = tinfl_decompress_mem_to_mem(&cache[0], block, src.data(), src.size(), 0); + if (got == TINFL_DECOMPRESS_MEM_TO_MEM_FAILED) return false; + cache.resize(got); + } + cached = (int64_t)i; + return true; + } + bool read(uint64_t off, size_t n, std::string* out) { // `n` bytes of the disc at `off` + if (!cso) return raw(off, n, out); + out->clear(); + if (off + n > total) return false; + while (out->size() < n) { + const uint64_t p = off + out->size(), bi = p / block, in = p % block; + if (!cso_block(bi) || in >= cache.size()) return false; + out->append(cache, (size_t)in, std::min(cache.size() - (size_t)in, n - out->size())); + } + return true; + } +}; +uint32_t le32(const std::string& s, size_t o) { + return o + 4 <= s.size() ? (uint32_t)(uint8_t)s[o] | (uint32_t)(uint8_t)s[o + 1] << 8 | (uint32_t)(uint8_t)s[o + 2] << 16 | (uint32_t)(uint8_t)s[o + 3] << 24 : 0; +} +uint16_t le16(const std::string& s, size_t o) { return o + 2 <= s.size() ? (uint16_t)((uint8_t)s[o] | (uint8_t)s[o + 1] << 8) : 0; } +// an ISO 9660 file or folder by its path ("PSP_GAME/PARAM.SFO"): extent and size +bool iso_find(DiscReader& r, const std::string& path, uint32_t* out_lba, uint32_t* out_size) { + std::string pvd; + if (!r.read(16u * 2048u, 2048, &pvd) || pvd.compare(1, 5, "CD001") != 0) return false; + uint32_t lba = le32(pvd, 156 + 2), size = le32(pvd, 156 + 10); + size_t from = 0; + while (from <= path.size()) { + size_t to = path.find('/', from); + if (to == std::string::npos) to = path.size(); + const std::string want = path.substr(from, to - from); + from = to + 1; + if (size == 0 || size > (1u << 20)) return false; + std::string d; + if (!r.read((uint64_t)lba * 2048u, size, &d)) return false; + bool found = false; + for (size_t p = 0; p < d.size();) { + const uint8_t len = (uint8_t)d[p]; + if (len == 0) { p = (p / 2048 + 1) * 2048; continue; } // records do not cross a sector + if (p + 33 > d.size() || p + len > d.size()) break; + const size_t nl = (uint8_t)d[p + 32]; + std::string nm = d.substr(p + 33, std::min(nl, d.size() - p - 33)); + if (size_t semi = nm.find(';'); semi != std::string::npos) nm.resize(semi); + for (char& ch : nm) ch = (char)std::toupper((unsigned char)ch); + if (nm == want) { lba = le32(d, p + 2); size = le32(d, p + 10); found = true; break; } + p += len; + } + if (!found) return false; + } + *out_lba = lba; *out_size = size; + return true; +} +bool disc_id(DiscReader& r, std::string* id, std::string* title) { + id->clear(); title->clear(); + uint32_t lba = 0, size = 0; + if (!r.ok() || !iso_find(r, "PSP_GAME/PARAM.SFO", &lba, &size) || size < 20 || size > 65536) return false; + std::string s; + if (!r.read((uint64_t)lba * 2048u, size, &s) || s.compare(0, 4, std::string("\0PSF", 4)) != 0) return false; + const uint32_t keys = le32(s, 8), data = le32(s, 12), n = le32(s, 16); + for (uint32_t i = 0; i < n && 20 + 16 * (i + 1) <= s.size(); i++) { + const size_t e = 20 + 16 * i; + const size_t ko = keys + le16(s, e), dofs = data + le32(s, e + 12), dl = le32(s, e + 4); + if (ko >= s.size() || dofs > s.size() || le16(s, e + 2) == 0x404) continue; // 0x404: integers, not needed + const std::string key = s.c_str() + ko; + std::string v = s.substr(dofs, std::min(dl, s.size() - dofs)); + if (size_t z = v.find('\0'); z != std::string::npos) v.resize(z); + if (key == "DISC_ID") *id = v; + else if (key == "TITLE") *title = v; + } + return !id->empty(); +} +std::string lower(std::string s) { for (char& ch : s) ch = (char)std::tolower((unsigned char)ch); return s; } +bool is_012(const std::string& id, const std::string& title) { + return lower(title).find("duodecim") != std::string::npos || id == "ULUS10566" || id == "ULES01505" || id == "ULJM05814" || id == "NPJH50377"; +} +bool is_d1(const std::string& id, const std::string& title) { + return id == "ULUS10437" || (!is_012(id, title) && lower(title).find("dissidia") != std::string::npos); +} +std::string disc_words(const std::string& id, const std::string& title) { return (title.empty() ? std::string("this disc") : title) + " (" + id + ")"; } +} // namespace + +bool read_disc_id(const std::string& path, std::string* id, std::string* title) { + DiscReader r(path); + return disc_id(r, id, title); +} + +std::string d1_disc_problem(const std::string& path) { + std::string id, title; + DiscReader r(path); + if (!disc_id(r, &id, &title)) + return "This file is not a PSP disc image that can be read. Pick an uncompressed .iso of Dissidia Final Fantasy (USA); " + "unpack a .zip / .7z first."; + if (id == "ULUS10437") return ""; + const std::string what = disc_words(id, title); + if (is_012(id, title)) + return "This is Dissidia 012, " + what + ". The moves come from the first game: pick your Dissidia Final Fantasy " + "(2008, USA, ULUS-10437) disc image instead."; + if (is_d1(id, title)) + return "This is another release of the first Dissidia, " + what + ". The moves are made from the US release only: " + "pick Dissidia Final Fantasy (USA, ULUS-10437)."; + return "This is not Dissidia Final Fantasy: the disc is " + what + ". Pick Dissidia Final Fantasy (2008, USA, ULUS-10437)."; +} + +// The English battle voice bank of Bartz (voice/en/battle/voice_p_fiv100_00.scd, package CRC 0x4300D781) as the EU and US +// discs have it: an image whose bank is this one has the English voices, so it is not an undub. +constexpr uint32_t kVoiceProbeCrc = 0x4300D781u, kVoiceProbeEnSize = 302368u, kVoiceProbeEnCrc32 = 0x6EE2B876u; +std::string undub_disc_problem(const std::string& path) { + std::string id, title; + DiscReader r(path); + if (!disc_id(r, &id, &title)) + return "This file is not a PSP disc image that can be read. Pick a .iso or .cso of Dissidia 012 with Japanese voices; " + "unpack a .zip / .7z first."; + const std::string what = disc_words(id, title); + if (!is_012(id, title)) + return (is_d1(id, title) ? "This is the first Dissidia, " : "This is not Dissidia 012: the disc is ") + what + + ". The Japanese voices come from a Dissidia 012 disc image that has them (an undub or the Japanese release)."; + uint32_t il = 0, is = 0, pl = 0, ps = 0; + std::string info; + if (!iso_find(r, "PSP_GAME/USRDIR/DATA/PACKAGE_INFO.BIN", &il, &is) || !iso_find(r, "PSP_GAME/USRDIR/DATA/PACKAGE.BIN", &pl, &ps) || + is < 16 || is > (4u << 20) || !r.read((uint64_t)il * 2048u, is, &info) || info.compare(4, 4, "pack") != 0) + return "The voice files of this disc image cannot be read (" + what + "). Pick another image of the undub or Japanese release."; + const uint32_t count = le32(info, 8); + for (uint32_t i = 0; i < count && 16 + 12 * (i + 1) <= info.size(); i++) { + if (le32(info, 16 + 12 * i) != kVoiceProbeCrc) continue; + const uint64_t off = (uint64_t)pl * 2048u + (uint64_t)le32(info, 16 + 12 * i + 4) * 2048u; + const uint32_t size = le32(info, 16 + 12 * i + 8) & 0x0FFFFFFFu; + std::string bank; + if (size == kVoiceProbeEnSize && r.read(off, size, &bank) && + mz_crc32(MZ_CRC32_INIT, reinterpret_cast(bank.data()), bank.size()) == kVoiceProbeEnCrc32) + return "This disc has the English voices, " + what + ": it is a regular Dissidia 012, like your own. Pick the undub " + "(a patched copy with the Japanese voices) or the Japanese release."; + return ""; + } + return ""; // no English bank to compare with (a release that names its files differently): not refused here +} + bool feature_available(const Layout& l, const std::string& name) { std::ifstream f(join(l.game_dir, "features.ini")); std::string line; diff --git a/psprecomp/setup/install.h b/psprecomp/setup/install.h index bc7a551..c36a638 100644 --- a/psprecomp/setup/install.h +++ b/psprecomp/setup/install.h @@ -57,6 +57,16 @@ const char* texture_pack_credit(const std::string& pack); // does not let it be turned on). bool feature_available(const Layout& l, const std::string& name); +// A PSP disc image's (.iso or .cso) PSP_GAME/PARAM.SFO: DISC_ID (e.g. "ULUS10437") and TITLE. `path` may be "fd:N" (Android picker); +// the descriptor's file offset is not moved. false = not readable as an ISO 9660 PSP disc image. +bool read_disc_id(const std::string& path, std::string* id, std::string* title); +// The Bartz moves need Dissidia Final Fantasy (2008), USA (ULUS-10437). "" when `path` is that disc, else what is wrong +// with it, in words for the player (e.g. a Dissidia 012 image picked by mistake). +std::string d1_disc_problem(const std::string& path); +// The Japanese voices need a Dissidia 012 image whose voices are Japanese (an undub, or the Japanese release; .iso or .cso). +// "" when `path` is one, else what is wrong with it (another game, the first Dissidia, or a regular English 012). +std::string undub_disc_problem(const std::string& path); + std::vector plan_install(const Layout& l, const std::string& iso, const std::string& workdir, const Options& o); // The game build only (rebuild / repair): analysis, recompile, compile (+ the Android build when chosen). // Only the extras of `o` (HD textures unless skip_textures, Japanese voices, saves, the Android build) on an installed game: no rebuild. diff --git a/psprecomp/setup/setup_main.cpp b/psprecomp/setup/setup_main.cpp index 6850671..47f0018 100644 --- a/psprecomp/setup/setup_main.cpp +++ b/psprecomp/setup/setup_main.cpp @@ -23,6 +23,7 @@ #ifdef __ANDROID__ #include +#include #endif #include #include @@ -249,6 +250,8 @@ struct App { int64_t check_start = 0; std::string check_result; std::string msg_title, msg_text; + Screen msg_back = Screen::Welcome; // where the Message page's OK goes (then back to Welcome for the next message) + Pick msg_pick = Pick::Iso; // msg_back PickInfo: the file to choose again // native (system) file dialogs; the built-in browser is the fallback std::unique_ptr file_dlg; @@ -430,7 +433,7 @@ struct App { p.title = "THE UNDUB FILE"; p.tab = "Japanese voices"; p.rows = {row("A second disc image", "The Japanese voices and movies come from a Japanese or undub version of Dissidia 012, as a disc image: a .iso or .cso. It is only read once; your own disc stays the base of the game.", ".iso or .cso"), - row("Valid: the undub / Japanese disc", "A Dissidia 012 image that has the Japanese voices, for example the Japanese release or an undub patch of the game. The same game as your own disc, just with different voice and movie files.", "Japanese / undub"), + row("Valid: the undub / Japanese disc", "A Dissidia 012 image that has the Japanese voices, for example the Japanese release or an undub patch of the game. The same game as your own disc, just with different voice and movie files. A disc with the English voices, or another game, is refused with a message.", "Japanese / undub"), row("Choose the file", how), row("Back")}; } @@ -1037,6 +1040,21 @@ struct App { workdir = path; if (before_pick == Screen::Options) go(Screen::Options, 0); else go(Screen::InstallDir); } else if (pick == Pick::Undub || pick == Pick::D1) { + { // the right game, before anything is installed with it + const std::string why = pick == Pick::D1 ? setup::d1_disc_problem(path) : setup::undub_disc_problem(path); + if (!why.empty()) { +#ifdef __ANDROID__ + if (path.compare(0, 3, "fd:") == 0) ::close(std::atoi(path.c_str() + 3)); // the picker's descriptor of the refused file +#endif + (pick == Pick::D1 ? opt.d1_iso : opt.undub_iso).clear(); + msg_title = "WRONG DISC"; + msg_text = why; + msg_back = Screen::PickInfo; // OK: choose that file again (msg_pick) + msg_pick = pick; + go(Screen::Message); + return; + } + } (pick == Pick::Undub ? opt.undub_iso : opt.d1_iso) = path; go_options(); start_install_or_ask(); @@ -1320,9 +1338,12 @@ struct App { else if (launch_mode) play(); else go(Screen::Welcome); break; - case Screen::Message: - go(Screen::Welcome); + case Screen::Message: { + const Screen back = msg_back; + msg_back = Screen::Welcome; + if (back == Screen::PickInfo) begin_pick(msg_pick, Screen::Options); else go(back); break; + } case Screen::Done: #ifdef __ANDROID__ if (sel == 0) { @@ -1393,9 +1414,12 @@ struct App { quit = true; break; case Screen::Failed: - case Screen::Message: - go(Screen::Welcome); + case Screen::Message: { + const Screen back = msg_back; + msg_back = Screen::Welcome; + if (back == Screen::PickInfo) begin_pick(msg_pick, Screen::Options); else go(back); break; + } case Screen::UpdateAvail: if (launch_mode) play(); else go(Screen::Welcome); break; @@ -1886,6 +1910,10 @@ int main(int argc, char** argv) { app.pick = Pick::Iso; app.go(Screen::PickInfo, 0); shot("2c_pick_info"); app.pick = Pick::Undub; app.go(Screen::PickInfo, 0); shot("2d_pick_info_undub"); app.pick = Pick::D1; app.go(Screen::PickInfo, 0); shot("2e_pick_info_d1"); + app.msg_title = "WRONG DISC"; // a refused pick (setup::d1_disc_problem's words for a Dissidia 012 image) + app.msg_text = "This is Dissidia 012, DISSIDIA duodecim FINAL FANTASY (ULUS10566). The moves come from the first game: pick your " + "Dissidia Final Fantasy (2008, USA, ULUS-10437) disc image instead."; + app.go(Screen::Message, 0); shot("2f_refused_d1"); app.pick = Pick::Iso; app.go(Screen::InstallDir); shot("3_install_dir"); app.go(Screen::Options, 0); shot("4_options"); diff --git a/psprecomp/setup/tests/test_install.cpp b/psprecomp/setup/tests/test_install.cpp index ec7d5c7..8888351 100644 --- a/psprecomp/setup/tests/test_install.cpp +++ b/psprecomp/setup/tests/test_install.cpp @@ -1,17 +1,144 @@ -// Unit tests for the wizard's logic that needs no window: progress parsing, ETA, update lines, plan weights, features. +// Unit tests for the wizard's logic that needs no window: progress parsing, ETA, update lines, plan weights, features, +// the disc checks of the picked Dissidia 1 / undub images (SETUP_TEST_DISCS adds real images). #include #include +#include +#include +#include #include #include #include #include "../install.h" +#include "../third_party/miniz.h" using namespace setup; static int failures = 0; #define CHECK(c) do { if (!(c)) { std::printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #c); failures++; } } while (0) +// ---- tiny PSP disc images for the disc checks (ISO 9660 with PSP_GAME/PARAM.SFO, optionally a package index) ---- +static void put32(std::string& s, size_t o, uint32_t v) { for (int i = 0; i < 4; i++) s[o + i] = (char)(v >> (8 * i)); } +static std::string dirrec(const std::string& name, uint32_t lba, uint32_t size, bool dir) { + std::string r(33 + name.size() + ((33 + name.size()) & 1), '\0'); + r[0] = (char)r.size(); put32(r, 2, lba); put32(r, 10, size); r[25] = dir ? 2 : 0; r[32] = (char)name.size(); + r.replace(33, name.size(), name); + return r; +} +static std::string sfo(const std::string& id, const std::string& title) { + const std::string keys = std::string("DISC_ID\0TITLE\0", 14); + std::string data(16 + 128, '\0'); + data.replace(0, id.size(), id); data.replace(16, title.size(), title); + std::string s(20 + 32, '\0'); + s.replace(0, 4, std::string("\0PSF", 4)); put32(s, 4, 0x101); put32(s, 8, 20 + 32); put32(s, 12, 20 + 32 + 16); put32(s, 16, 2); + // entries: {key off, fmt 0x0204 (utf8)}, len, max, data off + put32(s, 20, 0 | 0x0204u << 16); put32(s, 24, (uint32_t)id.size() + 1); put32(s, 28, 16); put32(s, 32, 0); + put32(s, 36, 8 | 0x0204u << 16); put32(s, 40, (uint32_t)title.size() + 1); put32(s, 44, 128); put32(s, 48, 16); + return s + keys + std::string(2, '\0') + data; +} +// sectors: 16 PVD, 18 root, 19 PSP_GAME, 20 PARAM.SFO, 21 USRDIR, 22 DATA, 23 PACKAGE_INFO.BIN, 24.. PACKAGE.BIN +static std::string make_iso(const std::string& id, const std::string& title, const std::string& pkg_info = "", const std::string& pkg = "") { + const size_t S = 2048, sectors = 24 + (pkg.size() + S - 1) / S + 1; + std::string img(sectors * S, '\0'); + std::string pvd(S, '\0'); + pvd[0] = 1; pvd.replace(1, 5, "CD001"); + pvd.replace(156, 34, dirrec(std::string(1, '\0'), 18, S, true).substr(0, 34)); + img.replace(16 * S, S, pvd); + std::string root = dirrec("PSP_GAME", 19, S, true); + img.replace(18 * S, root.size(), root); + const std::string sf = sfo(id, title); + std::string pg = dirrec("PARAM.SFO;1", 20, (uint32_t)sf.size(), false) + dirrec("USRDIR", 21, S, true); + img.replace(19 * S, pg.size(), pg); + img.replace(20 * S, sf.size(), sf); + std::string ud = dirrec("DATA", 22, S, true); + img.replace(21 * S, ud.size(), ud); + if (!pkg_info.empty()) { + std::string dd = dirrec("PACKAGE_INFO.BIN;1", 23, (uint32_t)pkg_info.size(), false) + dirrec("PACKAGE.BIN;1", 24, (uint32_t)pkg.size(), false); + img.replace(22 * S, dd.size(), dd); + img.replace(23 * S, pkg_info.size(), pkg_info); + img.replace(24 * S, pkg.size(), pkg); + } + return img; +} +// the same image as a CSO (CISO v1): even blocks deflated, odd blocks stored +static std::string make_cso(const std::string& iso) { + const uint32_t B = 2048, n = (uint32_t)((iso.size() + B - 1) / B); + std::string out(24 + 4 * (n + 1), '\0'); + out.replace(0, 4, "CISO"); put32(out, 4, 24); + for (int i = 0; i < 8; i++) out[8 + i] = (char)((uint64_t)iso.size() >> (8 * i)); + put32(out, 16, B); out[20] = 1; out[21] = 0; + for (uint32_t b = 0; b < n; b++) { + put32(out, 24 + 4 * b, (uint32_t)out.size() | (b & 1 ? 0x80000000u : 0)); + const std::string blk = iso.substr((size_t)b * B, B); + if (b & 1) out += blk; + else { + std::string z(B * 2, '\0'); + const size_t zn = tdefl_compress_mem_to_mem(&z[0], z.size(), blk.data(), blk.size(), TDEFL_DEFAULT_MAX_PROBES); + out += z.substr(0, zn); + } + } + put32(out, 24 + 4 * n, (uint32_t)out.size()); + return out; +} +static std::string write_file(const std::filesystem::path& p, const std::string& data) { + std::ofstream(p, std::ios::binary) << data; + return p.string(); +} +// a package index {crc, sector, size} of one entry, and its PACKAGE.BIN +static std::string pkg_index(uint32_t crc, uint32_t size) { + std::string s(16 + 12, '\0'); + put32(s, 0, 0x09200218u); s.replace(4, 4, "pack"); put32(s, 8, 1); + put32(s, 16, crc); put32(s, 20, 0); put32(s, 24, size | 0x10000000u); + return s; +} + +static void disc_checks(const std::filesystem::path& tmp) { + namespace fs = std::filesystem; + std::string id, title; + const std::string d1 = write_file(tmp / "d1.iso", make_iso("ULUS10437", "DISSIDIA FINAL FANTASY")); + CHECK(read_disc_id(d1, &id, &title) && id == "ULUS10437" && title == "DISSIDIA FINAL FANTASY"); + CHECK(d1_disc_problem(d1).empty()); + const std::string d1cso = write_file(tmp / "d1.cso", make_cso(make_iso("ULUS10437", "DISSIDIA FINAL FANTASY"))); + CHECK(read_disc_id(d1cso, &id, &title) && id == "ULUS10437"); + CHECK(d1_disc_problem(d1cso).empty()); + const std::string d012 = write_file(tmp / "012.iso", make_iso("ULUS10566", "DISSIDIA duodecim FINAL FANTASY")); + CHECK(d1_disc_problem(d012).find("Dissidia 012") != std::string::npos); // the reported mistake + const std::string d1eu = write_file(tmp / "d1eu.iso", make_iso("ULES01270", "DISSIDIA FINAL FANTASY")); + CHECK(d1_disc_problem(d1eu).find("another release") != std::string::npos); + const std::string other = write_file(tmp / "other.iso", make_iso("ULUS10391", "Crisis Core")); + CHECK(d1_disc_problem(other).find("not Dissidia") != std::string::npos); + const std::string junk = write_file(tmp / "junk.iso", std::string(40000, 'x')); + CHECK(!read_disc_id(junk, &id, &title)); + CHECK(!d1_disc_problem(junk).empty()); + CHECK(!d1_disc_problem((tmp / "missing.iso").string()).empty()); + + // undub: Dissidia 012 whose voices are not the English ones + CHECK(undub_disc_problem(d1).find("first Dissidia") != std::string::npos); + CHECK(undub_disc_problem(other).find("not Dissidia 012") != std::string::npos); + CHECK(!undub_disc_problem(junk).empty()); + const std::string jpbank(284832, 'j'); + const std::string undub = write_file(tmp / "undub.iso", make_iso("ULUS10566", "DISSIDIA duodecim FINAL FANTASY", pkg_index(0x4300D781u, (uint32_t)jpbank.size()), jpbank)); + CHECK(undub_disc_problem(undub).empty()); + const std::string undubcso = write_file(tmp / "undub.cso", make_cso(make_iso("ULUS10566", "DISSIDIA duodecim FINAL FANTASY", pkg_index(0x4300D781u, (uint32_t)jpbank.size()), jpbank))); + CHECK(undub_disc_problem(undubcso).empty()); + CHECK(undub_disc_problem(d012).find("cannot be read") != std::string::npos); // no PACKAGE.BIN: a broken 012 image + + // real disc images, when given: SETUP_TEST_DISCS="|<012 iso with English voices>|" + if (const char* e = std::getenv("SETUP_TEST_DISCS")) { + std::string v = e; + const size_t a = v.find('|'), b = v.find('|', a + 1); + const std::string rd1 = v.substr(0, a), r012 = v.substr(a + 1, b - a - 1), rundub = v.substr(b + 1); + CHECK(d1_disc_problem(rd1).empty()); + CHECK(d1_disc_problem(r012).find("Dissidia 012") != std::string::npos); + CHECK(d1_disc_problem(rundub).find("Dissidia 012") != std::string::npos); + CHECK(undub_disc_problem(rundub).empty()); + CHECK(undub_disc_problem(r012).find("English voices") != std::string::npos); + CHECK(undub_disc_problem(rd1).find("first Dissidia") != std::string::npos); + std::printf("real discs:\n D1 as undub: %s\n 012 as D1: %s\n 012 as undub: %s\n", undub_disc_problem(rd1).c_str(), + d1_disc_problem(r012).c_str(), undub_disc_problem(r012).c_str()); + } +} + int main() { namespace fs = std::filesystem; fs::path tmp = fs::temp_directory_path() / "psp_setup_test"; @@ -142,6 +269,8 @@ int main() { } #endif + disc_checks(tmp); + fs::remove_all(tmp); std::printf(failures ? "%d FAILED\n" : "all setup tests passed\n", failures); return failures ? 1 : 0; From 40417bfe06982dbc9453eeb49b4dfb844a30cf2a Mon Sep 17 00:00:00 2001 From: Schindler Date: Wed, 18 Nov 2026 21:03:55 -0500 Subject: [PATCH 2/4] Installer: the failure page opens setup.log or shows it in its folder Two rows on the NOT FINISHED page when the run has a setup.log (desktop only): Open setup.log (default program) and Show setup.log in its folder (Windows Explorer /select, macOS Finder -R, Linux xdg-open on the folder). If it cannot be opened, a message gives the path. setup::open_on_desktop in install.cpp. Co-Authored-By: Claude Opus 5.5 --- psprecomp/setup/install.cpp | 32 +++++++++++++++++++++++++ psprecomp/setup/install.h | 4 ++++ psprecomp/setup/setup_main.cpp | 43 +++++++++++++++++++++++++++++----- 3 files changed, 73 insertions(+), 6 deletions(-) diff --git a/psprecomp/setup/install.cpp b/psprecomp/setup/install.cpp index df47f09..99847c7 100644 --- a/psprecomp/setup/install.cpp +++ b/psprecomp/setup/install.cpp @@ -9,6 +9,7 @@ #include #ifdef _WIN32 #include +#include #else #include #endif @@ -302,6 +303,37 @@ std::string undub_disc_problem(const std::string& path) { return ""; // no English bank to compare with (a release that names its files differently): not refused here } +bool open_on_desktop(const std::string& path, bool show_in_folder, std::string* error) { +#if defined(__ANDROID__) + (void)path; (void)show_in_folder; + if (error) *error = "not available on Android"; + return false; +#elif defined(_WIN32) + auto wide = [](const std::string& s) { + std::wstring w(MultiByteToWideChar(CP_UTF8, 0, s.c_str(), -1, nullptr, 0), L'\0'); + if (!w.empty()) MultiByteToWideChar(CP_UTF8, 0, s.c_str(), -1, &w[0], (int)w.size()); + if (!w.empty() && w.back() == L'\0') w.pop_back(); + return w; + }; + std::string p = path; + for (char& ch : p) if (ch == '/') ch = '\\'; + HINSTANCE r; + if (show_in_folder) { + const std::wstring args = L"/select,\"" + wide(p) + L"\""; + r = ShellExecuteW(nullptr, L"open", L"explorer.exe", args.c_str(), nullptr, SW_SHOWNORMAL); + } else { + r = ShellExecuteW(nullptr, L"open", wide(p).c_str(), nullptr, nullptr, SW_SHOWNORMAL); + } + if ((INT_PTR)r <= 32) { if (error) *error = "Windows could not open " + p; return false; } + return true; +#elif defined(__APPLE__) + return spawn_detached(show_in_folder ? std::vector{"open", "-R", path} : std::vector{"open", path}, {}, "", error); +#else + const std::string target = show_in_folder ? fs::path(path).parent_path().string() : path; + return spawn_detached({"xdg-open", target}, {}, "", error); +#endif +} + bool feature_available(const Layout& l, const std::string& name) { std::ifstream f(join(l.game_dir, "features.ini")); std::string line; diff --git a/psprecomp/setup/install.h b/psprecomp/setup/install.h index c36a638..16941d8 100644 --- a/psprecomp/setup/install.h +++ b/psprecomp/setup/install.h @@ -67,6 +67,10 @@ std::string d1_disc_problem(const std::string& path); // "" when `path` is one, else what is wrong with it (another game, the first Dissidia, or a regular English 012). std::string undub_disc_problem(const std::string& path); +// Opens `path` with the desktop's default program (show_in_folder = false), or the folder that holds it with the file +// selected where the platform can (Windows Explorer /select, macOS Finder -R; Linux opens the folder). Not on Android. +bool open_on_desktop(const std::string& path, bool show_in_folder, std::string* error); + std::vector plan_install(const Layout& l, const std::string& iso, const std::string& workdir, const Options& o); // The game build only (rebuild / repair): analysis, recompile, compile (+ the Android build when chosen). // Only the extras of `o` (HD textures unless skip_textures, Japanese voices, saves, the Android build) on an installed game: no rebuild. diff --git a/psprecomp/setup/setup_main.cpp b/psprecomp/setup/setup_main.cpp index 47f0018..06179fa 100644 --- a/psprecomp/setup/setup_main.cpp +++ b/psprecomp/setup/setup_main.cpp @@ -647,12 +647,26 @@ struct App { case Screen::Failed: p.title = "NOT FINISHED"; p.tab = "Stopped"; - p.rows = {row("Try again", fail_msg + " Trying again continues where it stopped."), row("Back to the start"), row("Quit")}; + p.rows = {row("Try again", fail_msg + " Trying again continues where it stopped.")}; + if (can_open_log()) { + p.rows.push_back(row("Open setup.log", "The log of this install: every step's output and why it stopped. " + log_path)); + p.rows.push_back(row("Show setup.log in its folder", "Opens the install folder with setup.log selected, e.g. to send the file. " + log_path)); + } + p.rows.push_back(row("Back to the start")); + p.rows.push_back(row("Quit")); break; } return p; } + bool can_open_log() const { // a desktop with a setup.log to show (Android: the app's storage, not reachable from here) +#ifdef __ANDROID__ + return false; +#else + std::error_code ec; + return !log_path.empty() && std::filesystem::is_regular_file(log_path, ec); +#endif + } std::string cur_workdir() const { return !state.workdir.empty() ? state.workdir : workdir; } bool is_browser() const { return screen == Screen::PickIso || screen == Screen::PickUndub || screen == Screen::BrowseDir; } int row_count() { return is_browser() ? (int)br.list.size() : (int)page().rows.size(); } @@ -745,11 +759,13 @@ struct App { // setup.log in the install folder: every build line plus the failure reason (the wizard has no console on Windows) FILE* logf = nullptr; + std::string log_path; // setup.log of this run (the failure page opens it) void log_open(const std::string& dir) { if (logf) { std::fclose(logf); logf = nullptr; } std::error_code ec; std::filesystem::create_directories(dir, ec); - logf = std::fopen((std::filesystem::path(dir) / "setup.log").string().c_str(), "ab"); + log_path = (std::filesystem::path(dir) / "setup.log").string(); + logf = std::fopen(log_path.c_str(), "ab"); if (logf) { std::fprintf(logf, "\n===== psprecomp_setup run =====\n"); std::fflush(logf); } } void log_line(const std::string& l) { @@ -1363,11 +1379,23 @@ struct App { else if (sel == 1 && setup::android_build_exists(cur_workdir())) start_plan(false, false, true); else quit = true; break; - case Screen::Failed: + case Screen::Failed: { + const int extra = can_open_log() ? 2 : 0; // rows 1 and 2: open setup.log / show it in its folder if (sel == 0) { if (!plan.empty()) start_plan(rebuild_only && !updating && !android_inst, updating, android_inst); else go(Screen::Welcome); } - else if (sel == 1) go(Screen::Welcome); + else if (sel <= extra) { + if (logf) std::fflush(logf); + std::string err; + if (!setup::open_on_desktop(log_path, sel == 2, &err)) { + msg_title = "SETUP LOG"; + msg_text = "Could not open it (" + err + "). The file is " + log_path; + msg_back = Screen::Failed; + go(Screen::Message); + } + } + else if (sel == extra + 1) go(Screen::Welcome); else quit = true; break; + } case Screen::TypePath: break; } @@ -1929,8 +1957,11 @@ int main(int argc, char** argv) { app.prog.start_ms = (long long)now_ms() - 7 * 60 * 1000; app.prog.compile_i = 183; app.prog.compile_n = 474; app.prog.compile_start_ms = (long long)now_ms() - 4 * 60 * 1000; app.go(Screen::Build, 3); shot("5_build"); app.go(Screen::Done); shot("6_done"); - app.fail_msg = "The installation was cancelled."; - app.go(Screen::Failed); shot("7_failed"); + app.fail_msg = "A step failed (exit code 1); see setup.log in the install folder."; + app.log_path = (std::filesystem::path(shots) / "setup.log").string(); // a log to offer (the rows need the file) + { std::ofstream(app.log_path) << "example\n"; } + app.go(Screen::Failed, 1); shot("7_failed"); + { std::error_code ec; std::filesystem::remove(app.log_path, ec); } app.osk.begin(u"Folder path", u"/home/player", 200, true); app.go(Screen::TypePath); shot("8_type_path"); std::printf("%d screenshots in %s\n", n, shots.c_str()); From 5ca021c0e4e5148bb0dd74642ac109982d104146 Mon Sep 17 00:00:00 2001 From: Schindler Date: Wed, 18 Nov 2026 21:08:14 -0500 Subject: [PATCH 3/4] Windows: the D1 moves step crashed writing extra_abilities.json (cp1252), and every retry then failed on the cut file Python's text files default to the ANSI code page on Windows; the D1 help texts have characters cp1252 does not have, so json.dump stopped half way and left a truncated extra_abilities.json; Try again failed on reading it. - d1_moves.py, netpack.py, roster_mainbin.py (+ charmod_objx, voice_bank, package_extract): UTF-8 for every text file - extra_abilities.json is written to .tmp and swapped in; one that does not parse is kept as .bad and written again - the wizard runs its Python steps with PYTHONUTF8=1 Checked with PYTHONWARNDEFAULTENCODING=1 PYTHONWARNINGS=error::EncodingWarning and a truncated file: installs. Co-Authored-By: Claude Opus 5.5 --- .../games/dissidia/tools/charmod_objx.py | 4 +- psprecomp/games/dissidia/tools/d1_moves.py | 37 +++++++++++-------- psprecomp/games/dissidia/tools/netpack.py | 14 +++---- .../games/dissidia/tools/package_extract.py | 2 +- .../games/dissidia/tools/roster_mainbin.py | 4 +- psprecomp/games/dissidia/tools/voice_bank.py | 6 +-- psprecomp/setup/install.cpp | 18 ++++----- 7 files changed, 46 insertions(+), 39 deletions(-) diff --git a/psprecomp/games/dissidia/tools/charmod_objx.py b/psprecomp/games/dissidia/tools/charmod_objx.py index 8555821..69a5f5d 100644 --- a/psprecomp/games/dissidia/tools/charmod_objx.py +++ b/psprecomp/games/dissidia/tools/charmod_objx.py @@ -420,7 +420,7 @@ def hit_encode(d): # ---------------------------------------------------------------- json helpers def wj(path, obj, compact=True): """compact: every item of a top-level list on one line (events, hits, commands ...).""" - with open(path, 'w') as f: + with open(path, 'w', encoding='utf-8') as f: if not compact or not isinstance(obj, dict): json.dump(obj, f, indent=1, ensure_ascii=False) else: @@ -437,7 +437,7 @@ def wj(path, obj, compact=True): def rj(path): - with open(path) as f: + with open(path, encoding='utf-8') as f: return json.load(f) diff --git a/psprecomp/games/dissidia/tools/d1_moves.py b/psprecomp/games/dissidia/tools/d1_moves.py index 951ff37..6c48ada 100644 --- a/psprecomp/games/dissidia/tools/d1_moves.py +++ b/psprecomp/games/dissidia/tools/d1_moves.py @@ -173,7 +173,7 @@ def read_crc(self, crc): def prepare(src, d1, d12): """/d1bartz (D1, 012 layout) and /bartz (retail Bartz) for the Studio.""" os.makedirs(src + '/_template') - with open(src + '/_template/mod.json', 'w') as f: + with open(src + '/_template/mod.json', 'w', encoding="utf-8") as f: f.write('{}') gmo = d1.read('obj/p_fiv100_1p.gmo') os.makedirs(src + '/d1bartz/models') @@ -182,7 +182,7 @@ def prepare(src, d1, d12): co.unpack(d1_objx.convert(d1.read('obj/p_fiv100.objx')), src + '/d1bartz/moveset/p_d1b', bones=G.Model(gmo).bone_names()) names = {'%02X' % (0x60 + i): {'cmd': '0x%X' % c, 'name': n, 'type': t, 'phase': ph} for i, (c, n, t, ph, *_r) in enumerate(MOVES)} - with open(src + '/d1bartz/mod.json', 'w') as f: # based_on = this character: voices and face textures come along + with open(src + '/d1bartz/mod.json', 'w', encoding="utf-8") as f: # based_on = this character: voices and face textures come along json.dump({'character': {'name': 'Bartz (DFF)', 'code': 'd1b', 'based_on': 'fiv100'}, 'move_names': names, 'files': {}}, f, indent=1) os.makedirs(src + '/bartz/models') files = {} @@ -193,13 +193,13 @@ def prepare(src, d1, d12): co.unpack(d12.read('obj/%s.objx' % ms), '%s/bartz/moveset/%s' % (src, ms), bones=G.Model(g).bone_names()) files['obj/%s.objx' % ms] = 'moveset/' + ms files['obj/%s.gmo' % rig] = 'models/%s.gmo' % rig - with open(src + '/bartz/mod.json', 'w') as f: # move_names: the retail abilities, filled in by the page from the ISO + with open(src + '/bartz/mod.json', 'w', encoding="utf-8") as f: # move_names: the retail abilities, filled in by the page from the ISO json.dump({'kind': 'retail', 'character': {'name': 'Bartz', 'code': 'fiv100'}, 'move_names': {}, 'files': files}, f, indent=1) def finish(mod, d1, d12): """Names, help, costs, learn levels; D1's voice lines in the slots 012's bank left empty.""" - with open(mod + '/mod.json') as f: + with open(mod + '/mod.json', encoding="utf-8") as f: m = json.load(f) by_name = {mv[1]: mv for mv in MOVES} helps = d1_help(d1) @@ -214,7 +214,7 @@ def finish(mod, d1, d12): e['_note'] = 'Dissidia (2008) Bartz move, command 0x%X there (tools/d1_moves.py)' % cmd m['display'] = 'Bartz: Dissidia (2008) moves' ms = mod + '/moveset/p_fiv100' - rj = lambda p: json.load(open(p)) # noqa: E731 + rj = lambda p: json.load(open(p, encoding="utf-8")) # noqa: E731 acts = {a['action']: a['script'] for a in rj(ms + '/actions.json')['actions']} rows = rj(ms + '/motion_map.json')['rows'] new_cmds = {int(e['cmd'], 16) for e in m['move_names'].values() if 'copy' in e} @@ -240,7 +240,7 @@ def finish(mod, d1, d12): with open(p, 'wb') as f: f.write(b12.to_bytes()) m['files'][VOICE_012] = 'files/' + VOICE_012 - with open(mod + '/mod.json', 'w') as f: + with open(mod + '/mod.json', 'w', encoding="utf-8") as f: json.dump(m, f, indent=1, ensure_ascii=False) f.write('\n') print('voice lines from Dissidia (2008) in slots %s' % ', '.join(map(str, filled))) @@ -287,8 +287,8 @@ def cmd_patches(a): import netpack as npk npk.BLOCK = 8 d1, d12 = IsoPackage(a.d1), pe.Package(a.disc) - m = json.load(open(os.path.join(REPO, 'mods', 'bartz-dff', 'mod.json'))) - owner = json.load(open(os.path.join(a.mods, 'readme_charmod_retail.json'))) + m = json.load(open(os.path.join(REPO, 'mods', 'bartz-dff', 'mod.json'), encoding="utf-8")) + owner = json.load(open(os.path.join(a.mods, 'readme_charmod_retail.json'), encoding="utf-8")) os.makedirs(PATCH_DIR, exist_ok=True) for f in os.listdir(PATCH_DIR): if f.endswith('.npd'): @@ -314,7 +314,7 @@ def cmd_patches(a): e = {x: v for x, v in e.items() if x != 'help'} e['help_d1_cmd'] = next(mv[0] for mv in MOVES if mv[1] == e['name']) # D1's own text, read at install man['move_names'][k] = e - with open(os.path.join(PATCH_DIR, 'manifest.json'), 'w') as f: + with open(os.path.join(PATCH_DIR, 'manifest.json'), 'w', encoding="utf-8") as f: json.dump(man, f, indent=1) f.write('\n') print('wrote', os.path.normpath(PATCH_DIR)) @@ -325,7 +325,7 @@ def cmd_install(a): byte for byte (SHA-256); the new abilities into main.bin (roster_mainbin.py), the Settings switch (retail_moves.txt) and the online deltas (netpack.py).""" import netpack as npk - man = json.load(open(os.path.join(PATCH_DIR, 'manifest.json'))) + man = json.load(open(os.path.join(PATCH_DIR, 'manifest.json'), encoding="utf-8")) d12 = pe.Package(os.path.join(a.work, 'disc0')) d1 = IsoPackage(a.d1) disc, title = d1.disc_id() # Dissidia 012 has obj/p_fiv100.objx too: the disc's own id tells the games apart @@ -363,22 +363,29 @@ def cmd_install(a): rm.expand_help(e['help'], '') names[k] = e ab_p = os.path.join(mods, 'extra_abilities.json') - ab = json.load(open(ab_p)) if os.path.exists(ab_p) else {} + ab = {} + if os.path.exists(ab_p): + try: + ab = json.load(open(ab_p, encoding="utf-8")) + except ValueError: # cut short by a crash of an earlier install (v0.1.18 on Windows wrote it in cp1252 and stopped) + os.replace(ab_p, ab_p + '.bad') + print('%s could not be read (kept as %s.bad): written again' % (ab_p, os.path.basename(ab_p)), flush=True) ab[man['code']] = names - with open(ab_p, 'w') as fh: + with open(ab_p + '.tmp', 'w', encoding="utf-8") as fh: # whole or not at all json.dump(ab, fh, indent=2, ensure_ascii=False) fh.write('\n') + os.replace(ab_p + '.tmp', ab_p) ids = ','.join('%04X' % ((9 << 8) | int(k, 16)) for k in sorted(names)) line = '%s 9 %s nm=%s' % (man['folder'], ids, man['display'].replace(' ', '_')) rt = os.path.join(mods, 'retail_moves.txt') - lines = open(rt).read().splitlines() if os.path.exists(rt) else [] + lines = open(rt, encoding="utf-8").read().splitlines() if os.path.exists(rt) else [] off = any(l.startswith('#off# ' + man['folder'] + ' ') for l in lines) lines = [l for l in lines if not (l.split() and l.replace('#off# ', '').split()[0] == man['folder'])] lines.append(('#off# ' if off else '') + line) - with open(rt, 'w') as fh: + with open(rt, 'w', encoding="utf-8") as fh: fh.write('\n'.join(lines) + '\n') if not os.path.exists(os.path.join(mods, 'roster.txt')): - open(os.path.join(mods, 'roster.txt'), 'w').close() # (roster_mainbin would make up a character without one) + open(os.path.join(mods, 'roster.txt'), 'w', encoding="utf-8").close() # (roster_mainbin would make up a character without one) print('PROGRESS 75', flush=True) if subprocess.call([sys.executable, os.path.join(HERE, 'roster_mainbin.py'), os.path.join(a.work, 'disc0'), mods]): raise SystemExit('roster_mainbin.py failed') diff --git a/psprecomp/games/dissidia/tools/netpack.py b/psprecomp/games/dissidia/tools/netpack.py index 09db1f9..bbc656b 100644 --- a/psprecomp/games/dissidia/tools/netpack.py +++ b/psprecomp/games/dissidia/tools/netpack.py @@ -114,25 +114,25 @@ def decode(delta, src, max_out=64 << 20): # ---------------------------------------------------------------- the list def load(): - return json.load(open(PACKS_JSON)) if os.path.exists(PACKS_JSON) else {'_info': __doc__.split('\n\n')[1], 'packs': []} + return json.load(open(PACKS_JSON, encoding="utf-8")) if os.path.exists(PACKS_JSON) else {'_info': __doc__.split('\n\n')[1], 'packs': []} def save(j): - with open(PACKS_JSON, 'w') as f: + with open(PACKS_JSON, 'w', encoding="utf-8") as f: json.dump(j, f, indent=1) f.write('\n') def cmd_record(a): mods, disc = a.mods, pe.Package(a.disc) - owner = json.load(open(os.path.join(mods, RETAIL_STATE))) + owner = json.load(open(os.path.join(mods, RETAIL_STATE), encoding="utf-8")) files = sorted(gp for gp, n in owner.items() if n == a.folder) if not files: raise SystemExit('%s installs no retail files in %s (build it first)' % (a.folder, mods)) - abil = json.load(open(os.path.join(mods, ABILITIES))) - mod = json.load(open(os.path.join(HERE, '..', '..', '..', '..', 'mods', a.folder, 'mod.json'))) + abil = json.load(open(os.path.join(mods, ABILITIES), encoding="utf-8")) + mod = json.load(open(os.path.join(HERE, '..', '..', '..', '..', 'mods', a.folder, 'mod.json'), encoding="utf-8")) code = mod['character']['code'] - lines = [l.split() for l in open(os.path.join(mods, 'retail_moves.txt')).read().splitlines()] + lines = [l.split() for l in open(os.path.join(mods, 'retail_moves.txt'), encoding="utf-8").read().splitlines()] char = next(int(l[-3]) for l in lines if a.folder in l) name = a.name or a.folder if len(name) > MAX_NAME: @@ -166,7 +166,7 @@ def cmd_inc(_a=None): for x in p['abilities']: out.append(' {%s, %s, %s, %d, %d},' % (x['id'], x['copy'], x['cmd'], x['type'], x['phase'])) out.append(' }},') - with open(PACKS_INC, 'w') as f: + with open(PACKS_INC, 'w', encoding="utf-8") as f: f.write('\n'.join(out) + '\n') diff --git a/psprecomp/games/dissidia/tools/package_extract.py b/psprecomp/games/dissidia/tools/package_extract.py index b3219f2..45be4c8 100644 --- a/psprecomp/games/dissidia/tools/package_extract.py +++ b/psprecomp/games/dissidia/tools/package_extract.py @@ -74,7 +74,7 @@ def main(argv): cmd = argv[1] if cmd == "list": sub = argv[2].lower() - for line in open(PATHS): + for line in open(PATHS, encoding="utf-8"): p = line.strip() if sub in p.lower(): c = package_crc(p) diff --git a/psprecomp/games/dissidia/tools/roster_mainbin.py b/psprecomp/games/dissidia/tools/roster_mainbin.py index e9eb3ba..36079a7 100644 --- a/psprecomp/games/dissidia/tools/roster_mainbin.py +++ b/psprecomp/games/dissidia/tools/roster_mainbin.py @@ -219,7 +219,7 @@ def parse_arc(d, base): # {tag: (offset, size)} def read_roster(path, min_id=FIRST_NEW): """[(charId, code, like)] for charId >= min_id (default FIRST_NEW) from a roster.txt (' [like] [k=v...]').""" out = [] - for line in open(path): + for line in open(path, encoding="utf-8"): line = line.split("#")[0].split() if len(line) < 2 or not line[0].isdigit(): continue @@ -245,7 +245,7 @@ def parse_objs(spec): def read_roster_objs(path): """(objs, charId of the line that declares the first objs= token) from a roster.txt; objs = every `objs=new:src,...` pair.""" out, char = [], None - for line in open(path): + for line in open(path, encoding="utf-8"): toks = line.split("#")[0].split() for tok in toks: if tok.startswith("objs="): diff --git a/psprecomp/games/dissidia/tools/voice_bank.py b/psprecomp/games/dissidia/tools/voice_bank.py index 9abf2e1..c71192a 100644 --- a/psprecomp/games/dissidia/tools/voice_bank.py +++ b/psprecomp/games/dissidia/tools/voice_bank.py @@ -421,7 +421,7 @@ def cmd_dump(a): man["slots"].append(s) if vold: man["vold"] = vold - json.dump(man, open(os.path.join(a.outdir, "manifest.json"), "w"), indent=1) + json.dump(man, open(os.path.join(a.outdir, "manifest.json"), "w", encoding="utf-8"), indent=1) nd = sum(1 for e in bank.entries if e["kind"] == "data") print("%s: %d slots (%d clips, %d aliases, %d links, %d empty) -> %s" % (src, len(bank.entries), nd, sum(e["kind"] == "alias" for e in bank.entries), @@ -472,7 +472,7 @@ def pcm_for_slot(path, rate_opt, template_rate): def cmd_build(a): d = a.dir mpath = os.path.join(d, "manifest.json") - man = json.load(open(mpath)) if os.path.exists(mpath) else None + man = json.load(open(mpath, encoding="utf-8")) if os.path.exists(mpath) else None like = None if a.like: like, _ = load_bank_file(a.like) @@ -683,7 +683,7 @@ def cmd_alias(a): import package_extract as pe pkg = pe.Package(a.disc) like, code = a.like.lower(), a.code.lower() - paths = [l.strip().lower() for l in open(os.path.join(HERE, "..", "package_paths.txt"))] + paths = [l.strip().lower() for l in open(os.path.join(HERE, "..", "package_paths.txt"), encoding="utf-8")] want = [] for n in range(10): for suf in ("", "_a"): diff --git a/psprecomp/setup/install.cpp b/psprecomp/setup/install.cpp index 99847c7..4b432b8 100644 --- a/psprecomp/setup/install.cpp +++ b/psprecomp/setup/install.cpp @@ -360,7 +360,7 @@ static Step build_step(const Layout& l, const std::string& workdir, bool android b.argv = {l.python, l.build_script, workdir, "--sdk", l.sdk, "--toolchain", l.toolchain, "--psprecomp", l.recompiler, "--game", l.game_dir, "--progress"}; if (android) b.argv.push_back("--android"); - b.env = {{"PYTHONUNBUFFERED", "1"}}; + b.env = {{"PYTHONUNBUFFERED", "1"}, {"PYTHONUTF8", "1"}}; return b; } @@ -394,7 +394,7 @@ static void append_extras(std::vector& p, const Layout& l, const std::stri t.label = ai ? "Downloading the AI upscaled textures" : "Downloading the HD textures"; t.explain = ai ? "Downloads the AI upscaled HD textures (about 1.3 GB)." : "Downloads LiinkPK's HD textures (about 0.8 GB)."; t.argv = {l.python, join(l.game_dir, "tools/fetch_textures.py"), workdir, "--pack", ai ? "ai" : "liink"}; - t.env = {{"PYTHONUNBUFFERED", "1"}}; + t.env = {{"PYTHONUNBUFFERED", "1"}, {"PYTHONUTF8", "1"}}; t.weight = 0.06; p.push_back(t); } @@ -403,7 +403,7 @@ static void append_extras(std::vector& p, const Layout& l, const std::stri j.label = "Extracting the Japanese voices"; j.explain = "Adds the Japanese voices and movies from the undub disc."; j.argv = {l.python, join(l.game_dir, "tools/jp_audio.py"), "--iso", o.undub_iso, "--work", workdir}; - j.env = {{"PYTHONUNBUFFERED", "1"}}; + j.env = {{"PYTHONUNBUFFERED", "1"}, {"PYTHONUTF8", "1"}}; j.weight = 0.05; p.push_back(j); } @@ -412,7 +412,7 @@ static void append_extras(std::vector& p, const Layout& l, const std::stri d.label = "Adding Bartz's Dissidia 1 moves"; d.explain = "Rebuilds the eight moves Bartz had in the first Dissidia from your Dissidia 1 disc and adds them as new abilities. Settings > Mods in the game turns them off and on."; d.argv = {l.python, join(l.game_dir, "tools/d1_moves.py"), "install", "--d1", o.d1_iso, "--work", workdir}; - d.env = {{"PYTHONUNBUFFERED", "1"}}; + d.env = {{"PYTHONUNBUFFERED", "1"}, {"PYTHONUTF8", "1"}}; d.weight = 0.01; p.push_back(d); } @@ -431,7 +431,7 @@ std::vector plan_install(const Layout& l, const std::string& iso, const st x.label = "Reading your game file"; x.explain = "Reads the game's files out of your disc image. Nothing is uploaded."; x.argv = {l.python, l.setup_script, iso, workdir}; - x.env = {{"PSPRECOMP_TOOLCHAIN", l.toolchain}, {"PYTHONUNBUFFERED", "1"}}; + x.env = {{"PSPRECOMP_TOOLCHAIN", l.toolchain}, {"PYTHONUNBUFFERED", "1"}, {"PYTHONUTF8", "1"}}; x.weight = 0.08; std::vector p{x}; for (auto& s : plan_rebuild(l, workdir, o)) p.push_back(s); // weights are renormalised below @@ -468,7 +468,7 @@ std::vector plan_steam(const Layout& l, const std::string& workdir) { st.argv = {l.python, join(l.game_dir, "tools/steam_shortcut.py"), exe, "--name", "Dissidia 012 Duodecim Final Fantasy", "--launch-options=--launch", "--yes", "--icon", join(l.game_dir, "icon/icon.png"), "--pic1", join(workdir, "disc0/PSP_GAME/PIC1.PNG")}; - st.env = {{"PYTHONUNBUFFERED", "1"}}; + st.env = {{"PYTHONUNBUFFERED", "1"}, {"PYTHONUTF8", "1"}}; st.weight = 0.03; st.optional = true; std::vector p{st}; @@ -481,7 +481,7 @@ std::vector plan_android_install(const Layout& l, const std::string& workd x.label = "Installing on your device"; x.explain = "Installs the app over adb and copies the game files (about 2 GB)."; x.argv = {l.python, join(l.game_dir, "tools/android_install.py"), workdir}; - x.env = {{"PYTHONUNBUFFERED", "1"}}; + x.env = {{"PYTHONUNBUFFERED", "1"}, {"PYTHONUTF8", "1"}}; x.weight = 1; std::vector p{x}; finish_plan(p); @@ -519,7 +519,7 @@ std::vector android_verify_argv(const Layout& l, const std::string& std::string android_game_dir() { return std::string("/sdcard/Android/media/") + kGamePackage + "/game"; } EnvList android_env(const Layout& l, const std::string& workdir) { - return {{"PYTHONUNBUFFERED", "1"}, {"PYTHONDONTWRITEBYTECODE", "1"}, {"PYTHONHOME", join(l.root, "python")}, + return {{"PYTHONUNBUFFERED", "1"}, {"PYTHONUTF8", "1"}, {"PYTHONDONTWRITEBYTECODE", "1"}, {"PYTHONHOME", join(l.root, "python")}, {"PSPRECOMP_ANDROID_DEVICE", "1"}, {"PSPRECOMP_ZIG", l.zig}, {"HOME", workdir}, {"TMPDIR", join(workdir, "tmp")}, {"ZIG_LIB_DIR", join(l.android_dir, "zig-lib")}}; } @@ -675,7 +675,7 @@ std::vector plan_update(const Layout& l, const std::string& workdir, const d.explain = "Downloads the new release and replaces the program files. Your saves are untouched."; d.argv = {l.python, l.update_script, "apply", "--root", l.root}; if (const char* m = std::getenv("PSPRECOMP_MANIFEST_URL")) { d.argv.push_back("--manifest"); d.argv.push_back(m); } - d.env = {{"PYTHONUNBUFFERED", "1"}}; + d.env = {{"PYTHONUNBUFFERED", "1"}, {"PYTHONUTF8", "1"}}; d.weight = u.rebuild ? 0.10 : 0.60; std::vector p{d}; if (u.rebuild || u.relink) { From 3f7973ce780332dd94aaa4f92ec3718bd43cb934 Mon Sep 17 00:00:00 2001 From: Schindler Date: Wed, 18 Nov 2026 21:12:01 -0500 Subject: [PATCH 4/4] Installer: drop PYTHONUTF8 again: the Windows release's embedded Python has a ._pth file (isolated mode), which ignores PYTHON* variables; the tools' explicit UTF-8 is the fix Co-Authored-By: Claude Opus 5.5 --- psprecomp/setup/install.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/psprecomp/setup/install.cpp b/psprecomp/setup/install.cpp index 4b432b8..99847c7 100644 --- a/psprecomp/setup/install.cpp +++ b/psprecomp/setup/install.cpp @@ -360,7 +360,7 @@ static Step build_step(const Layout& l, const std::string& workdir, bool android b.argv = {l.python, l.build_script, workdir, "--sdk", l.sdk, "--toolchain", l.toolchain, "--psprecomp", l.recompiler, "--game", l.game_dir, "--progress"}; if (android) b.argv.push_back("--android"); - b.env = {{"PYTHONUNBUFFERED", "1"}, {"PYTHONUTF8", "1"}}; + b.env = {{"PYTHONUNBUFFERED", "1"}}; return b; } @@ -394,7 +394,7 @@ static void append_extras(std::vector& p, const Layout& l, const std::stri t.label = ai ? "Downloading the AI upscaled textures" : "Downloading the HD textures"; t.explain = ai ? "Downloads the AI upscaled HD textures (about 1.3 GB)." : "Downloads LiinkPK's HD textures (about 0.8 GB)."; t.argv = {l.python, join(l.game_dir, "tools/fetch_textures.py"), workdir, "--pack", ai ? "ai" : "liink"}; - t.env = {{"PYTHONUNBUFFERED", "1"}, {"PYTHONUTF8", "1"}}; + t.env = {{"PYTHONUNBUFFERED", "1"}}; t.weight = 0.06; p.push_back(t); } @@ -403,7 +403,7 @@ static void append_extras(std::vector& p, const Layout& l, const std::stri j.label = "Extracting the Japanese voices"; j.explain = "Adds the Japanese voices and movies from the undub disc."; j.argv = {l.python, join(l.game_dir, "tools/jp_audio.py"), "--iso", o.undub_iso, "--work", workdir}; - j.env = {{"PYTHONUNBUFFERED", "1"}, {"PYTHONUTF8", "1"}}; + j.env = {{"PYTHONUNBUFFERED", "1"}}; j.weight = 0.05; p.push_back(j); } @@ -412,7 +412,7 @@ static void append_extras(std::vector& p, const Layout& l, const std::stri d.label = "Adding Bartz's Dissidia 1 moves"; d.explain = "Rebuilds the eight moves Bartz had in the first Dissidia from your Dissidia 1 disc and adds them as new abilities. Settings > Mods in the game turns them off and on."; d.argv = {l.python, join(l.game_dir, "tools/d1_moves.py"), "install", "--d1", o.d1_iso, "--work", workdir}; - d.env = {{"PYTHONUNBUFFERED", "1"}, {"PYTHONUTF8", "1"}}; + d.env = {{"PYTHONUNBUFFERED", "1"}}; d.weight = 0.01; p.push_back(d); } @@ -431,7 +431,7 @@ std::vector plan_install(const Layout& l, const std::string& iso, const st x.label = "Reading your game file"; x.explain = "Reads the game's files out of your disc image. Nothing is uploaded."; x.argv = {l.python, l.setup_script, iso, workdir}; - x.env = {{"PSPRECOMP_TOOLCHAIN", l.toolchain}, {"PYTHONUNBUFFERED", "1"}, {"PYTHONUTF8", "1"}}; + x.env = {{"PSPRECOMP_TOOLCHAIN", l.toolchain}, {"PYTHONUNBUFFERED", "1"}}; x.weight = 0.08; std::vector p{x}; for (auto& s : plan_rebuild(l, workdir, o)) p.push_back(s); // weights are renormalised below @@ -468,7 +468,7 @@ std::vector plan_steam(const Layout& l, const std::string& workdir) { st.argv = {l.python, join(l.game_dir, "tools/steam_shortcut.py"), exe, "--name", "Dissidia 012 Duodecim Final Fantasy", "--launch-options=--launch", "--yes", "--icon", join(l.game_dir, "icon/icon.png"), "--pic1", join(workdir, "disc0/PSP_GAME/PIC1.PNG")}; - st.env = {{"PYTHONUNBUFFERED", "1"}, {"PYTHONUTF8", "1"}}; + st.env = {{"PYTHONUNBUFFERED", "1"}}; st.weight = 0.03; st.optional = true; std::vector p{st}; @@ -481,7 +481,7 @@ std::vector plan_android_install(const Layout& l, const std::string& workd x.label = "Installing on your device"; x.explain = "Installs the app over adb and copies the game files (about 2 GB)."; x.argv = {l.python, join(l.game_dir, "tools/android_install.py"), workdir}; - x.env = {{"PYTHONUNBUFFERED", "1"}, {"PYTHONUTF8", "1"}}; + x.env = {{"PYTHONUNBUFFERED", "1"}}; x.weight = 1; std::vector p{x}; finish_plan(p); @@ -519,7 +519,7 @@ std::vector android_verify_argv(const Layout& l, const std::string& std::string android_game_dir() { return std::string("/sdcard/Android/media/") + kGamePackage + "/game"; } EnvList android_env(const Layout& l, const std::string& workdir) { - return {{"PYTHONUNBUFFERED", "1"}, {"PYTHONUTF8", "1"}, {"PYTHONDONTWRITEBYTECODE", "1"}, {"PYTHONHOME", join(l.root, "python")}, + return {{"PYTHONUNBUFFERED", "1"}, {"PYTHONDONTWRITEBYTECODE", "1"}, {"PYTHONHOME", join(l.root, "python")}, {"PSPRECOMP_ANDROID_DEVICE", "1"}, {"PSPRECOMP_ZIG", l.zig}, {"HOME", workdir}, {"TMPDIR", join(workdir, "tmp")}, {"ZIG_LIB_DIR", join(l.android_dir, "zig-lib")}}; } @@ -675,7 +675,7 @@ std::vector plan_update(const Layout& l, const std::string& workdir, const d.explain = "Downloads the new release and replaces the program files. Your saves are untouched."; d.argv = {l.python, l.update_script, "apply", "--root", l.root}; if (const char* m = std::getenv("PSPRECOMP_MANIFEST_URL")) { d.argv.push_back("--manifest"); d.argv.push_back(m); } - d.env = {{"PYTHONUNBUFFERED", "1"}, {"PYTHONUTF8", "1"}}; + d.env = {{"PYTHONUNBUFFERED", "1"}}; d.weight = u.rebuild ? 0.10 : 0.60; std::vector p{d}; if (u.rebuild || u.relink) {