From d29d35ad2d1de66c18176b7c85feceacdcc126ef Mon Sep 17 00:00:00 2001 From: xylonzinho Date: Sun, 29 Mar 2026 13:48:46 -0300 Subject: [PATCH 1/2] ZFS Support Implementation --- .gitignore | 2 + Makefile | 13 + README.md | 44 +++- config.ini.example | 3 + include/sm_mount_defs.h | 5 + include/sm_types.h | 4 + makezfs.sh | 544 ++++++++++++++++++++++++++++++++++++++++ src/sm_config_mount.c | 39 ++- src/sm_image.c | 26 ++ 9 files changed, 671 insertions(+), 9 deletions(-) create mode 100644 makezfs.sh diff --git a/.gitignore b/.gitignore index 9162c0d..7eb93d4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ notify.txt src/notify_icon_asset.c refs/* docs/* +.DS_Store +/zfs_implementation* diff --git a/Makefile b/Makefile index 7429a16..7b7e046 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,9 @@ PS5_PAYLOAD_SDK ?= /opt/ps5-payload-sdk + +# Allow utility-only targets to run without the PS5 SDK toolchain present. +ifeq ($(filter clean makezfs,$(MAKECMDGOALS)),) include $(PS5_PAYLOAD_SDK)/toolchain/prospero.mk +endif VERSION_TAG := $(shell git describe --abbrev=6 --dirty --always --tags 2>/dev/null || echo unknown) @@ -21,6 +25,15 @@ HEADERS := $(wildcard include/*.h) # Targets all: shadowmountplus.elf +.PHONY: makezfs +makezfs: + @if [ -z "$(SOURCE)" ]; then \ + echo "Usage: make makezfs SOURCE= [OUTPUT=] [ZFS_COMPRESSION=lz4]"; \ + exit 1; \ + fi + ZFS_COMPRESSION="$(if $(ZFS_COMPRESSION),$(ZFS_COMPRESSION),lz4)" \ + ./makezfs.sh "$(SOURCE)" "$(if $(OUTPUT),$(OUTPUT),download0.ffzfs)" + # Build Daemon shadowmountplus.elf: $(OBJS) $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(OBJS) $(LIBS) diff --git a/README.md b/README.md index 310655d..b5950c8 100644 --- a/README.md +++ b/README.md @@ -13,19 +13,21 @@ ## Current image support -`PFS support is experimental.` +`PFS and ZFS support are experimental.` | Extension | Mounted FS | Attach backend | Status | | --- | --- | --- | --- | | `.ffpkg` | `ufs` | `LVD` or `MD` (configurable) | Recommended | | `.exfat` | `exfatfs` | `LVD` or `MD` (configurable) | Compatibility / external-drive-only titles | | `.ffpfs` | `pfs` | `LVD` | Experimental | +| `.ffzfs` | `zfs` | `LVD` or `MD` (configurable) | Experimental - File compression support | Notes: - Backend, read-only mode, and sector size can be configured via `/data/shadowmount/config.ini`. - Debug logging is enabled by default (`debug=1`) and writes to console plus `/data/shadowmount/debug.log` (set `debug=0` to disable). - **UFS (`.ffpkg`) is the recommended image format for normal use.** - **Use exFAT (`.exfat`) only for titles that need external-drive-style compatibility.** +- **ZFS (`.ffzfs`) is experimental; use only if you wanto so benefit from the file compression from ZFS.** - **When building exFAT images manually, keep the cluster size at `64 KB`; smaller clusters can reduce performance.** ## Recommended FS choice @@ -51,6 +53,7 @@ Supported keys (all optional): - `stability_wait_seconds=<0..3600>` (minimum source age before processing; default: `10`) - `exfat_backend=lvd|md` (default: `lvd`) - `ufs_backend=lvd|md` (default: `lvd`) +- `zfs_backend=lvd|md` (default: `lvd`) - `backport_fakelib=1|0` (`1` mounts sandbox `fakelib` overlays for running games; default: `1`) - `kstuff_game_auto_toggle=1|0` (`1` pauses kstuff after tracked game launches and resumes it on stop; default: `1`) - `kstuff_pause_delay_image_seconds=<0..3600>` (delay before pausing kstuff for image-backed launches; default: `20`) @@ -63,9 +66,11 @@ Supported keys (all optional): - `scanpath=` (can be repeated on multiple lines; default: built-in scan path list below) - `lvd_exfat_sector_size=` (default: `512`) - `lvd_ufs_sector_size=` (default: `4096`) +- `lvd_zfs_sector_size=` (default: `4096`) - `lvd_pfs_sector_size=` (default: `32768`) - `md_exfat_sector_size=` (default: `512`) - `md_ufs_sector_size=` (default: `512`) +- `md_zfs_sector_size=` (default: `512`) Per-image mode override behavior: - Match is done by image file name (without path). @@ -125,7 +130,7 @@ Image mountpoints are created under: `/mnt/shadowmnt/_` -Image layout requirement (`.ffpkg`, `.exfat`, `.ffpfs`): +Image layout requirement (`.ffpkg`, `.exfat`, `.ffpfs`, `.ffzfs`): - Game files must be placed at the image root. - Do not add an extra top-level folder inside the image. - Valid example: `/sce_sys/param.json` exists directly from image root. @@ -225,6 +230,37 @@ Windows: - `UFS2Tool.exe newfs -O 2 -b 65536 -f 65536 -m 0 -S 4096 -i 262144 -D ./APPXXXX ./PPSA12345.ffpkg` - For manual builds, use `-i 262144` as the baseline and lower it for images with many small files. +## Creating a ZFS image (`.ffzfs`) + +ZFS support is experimental and intended for advanced users who need ZFS tooling/compression. + +Linux: +- Script: `makezfs.sh` +- Usage: `sudo ./makezfs.sh [output_file]` +- Example: + - `chmod +x makezfs.sh` + - `sudo ./makezfs.sh ./APPXXXX ./PPSA12345.ffzfs` +- Requirements: + - `zpool`, `zfs`, `losetup`, `truncate`, `rsync` +- Notes: + - Source folder must be the game root and contain `eboot.bin`. + - Recommended/default compression is `lz4` for speed and compatibility. + - You can override compression via env var, for example: + - `sudo ZFS_COMPRESSION=zstd ./makezfs.sh ./APPXXXX ./PPSA12345.ffzfs` + +macOS: +- Script: `makezfs.sh` +- Usage: `sudo ./makezfs.sh [output_file]` +- Requirements: + - OpenZFS tools installed and available in PATH (`zpool`, `zfs`) + - `hdiutil`, `mkfile`, `rsync` +- Notes: + - The script uses `hdiutil` raw attachment for image-backed pool creation. + +Windows: +- Recommended path is WSL with OpenZFS tooling and then run `makezfs.sh` from WSL. +- Native Windows ZFS/image workflow is not bundled yet. + ## Installation and usage @@ -262,9 +298,9 @@ If a game is not mounted: - If logs show `source not stable yet`, adjust `stability_wait_seconds` (or wait for source copy/write to finish). - Verify game structure: - folder game: `/sce_sys/param.json`; - - image game (`.ffpkg` / `.exfat` / `.ffpfs`): `sce_sys/param.json` must be at image root (no extra top-level folder). + - image game (`.ffpkg` / `.exfat` / `.ffpfs` / `.ffzfs`): `sce_sys/param.json` must be at image root (no extra top-level folder). - If you see `missing/invalid param.json` for an image, check via FTP that files are present under `/mnt/shadowmnt/_/` and include `sce_sys/param.json`. -- If you see image mount failure, check image integrity and filesystem type (`.ffpkg`=UFS, `.exfat`=exFAT, `.ffpfs`=PFS). +- If you see image mount failure, check image integrity and filesystem type (`.ffpkg`=UFS, `.exfat`=exFAT, `.ffpfs`=PFS, `.ffzfs`=ZFS). - If you see duplicate titleId notification, keep only one source per ``. If a game is mounted but does not start: diff --git a/config.ini.example b/config.ini.example index ba48b08..9db25f6 100644 --- a/config.ini.example +++ b/config.ini.example @@ -51,6 +51,7 @@ stability_wait_seconds=10 # md -> /dev/mdctl -> /dev/mdN exfat_backend=lvd ufs_backend=lvd +zfs_backend=lvd # Sandbox fakelib backport watcher: # 1/true/yes/on -> mount /mnt/sandbox/_XXX/app0/fakelib into common/lib @@ -78,9 +79,11 @@ kstuff_pause_delay_direct_seconds=10 # Backend-specific sector size overrides: lvd_exfat_sector_size=512 lvd_ufs_sector_size=4096 +lvd_zfs_sector_size=4096 lvd_pfs_sector_size=32768 md_exfat_sector_size=512 md_ufs_sector_size=512 +md_zfs_sector_size=512 # Optional custom scan roots (can be repeated): # If at least one scanpath=... is present, only these paths are used. diff --git a/include/sm_mount_defs.h b/include/sm_mount_defs.h index b78ab99..9327eee 100644 --- a/include/sm_mount_defs.h +++ b/include/sm_mount_defs.h @@ -7,6 +7,8 @@ #define EXFAT_ATTACH_USE_MDCTL 0 // 1 = allow mounting .ffpkg images via /dev/mdctl, 0 = keep UFS on LVD. #define UFS_ATTACH_USE_MDCTL 0 +// 1 = allow mounting .ffzfs images via /dev/mdctl, 0 = keep ZFS on LVD. +#define ZFS_ATTACH_USE_MDCTL 0 // --- LVD Definitions --- // Kernel exposes base attach (V0), extended attach (V1/Attach2) and detach. @@ -35,16 +37,19 @@ # #define LVD_SECTOR_SIZE_EXFAT 512u #define LVD_SECTOR_SIZE_UFS 4096u +#define LVD_SECTOR_SIZE_ZFS 4096u #define LVD_SECTOR_SIZE_PFS 4096u #define LVD_SECONDARY_UNIT_SINGLE_IMAGE 0x10000u #define MD_SECTOR_SIZE_EXFAT 512u #define MD_SECTOR_SIZE_UFS 512u +#define MD_SECTOR_SIZE_ZFS 512u // Raw option bits are normalized by sceFsLvdAttachCommon before validation: // raw:0x1->norm:0x08, raw:0x2->norm:0x80, raw:0x4->norm:0x02, raw:0x8->norm:0x10. // The normalized masks are then checked against validator constraints (0x82/0x92). #define LVD_ATTACH_IMAGE_TYPE_SINGLE 0 #define LVD_ATTACH_IMAGE_TYPE_UFS_DOWNLOAD_DATA 7 #define LVD_ATTACH_IMAGE_TYPE_PFS_SAVE_DATA 5 +#define LVD_ATTACH_IMAGE_TYPE_ZFS LVD_ATTACH_IMAGE_TYPE_SINGLE #define LVD_ATTACH_LAYER_COUNT 1 #define LVD_ATTACH_LAYER_ARRAY_SIZE 3 #define LVD_ENTRY_TYPE_FILE 1 diff --git a/include/sm_types.h b/include/sm_types.h index 45689e6..05c21d9 100644 --- a/include/sm_types.h +++ b/include/sm_types.h @@ -116,11 +116,14 @@ typedef struct runtime_config { uint32_t kstuff_pause_delay_direct_seconds; attach_backend_t exfat_backend; attach_backend_t ufs_backend; + attach_backend_t zfs_backend; uint32_t lvd_sector_exfat; uint32_t lvd_sector_ufs; + uint32_t lvd_sector_zfs; uint32_t lvd_sector_pfs; uint32_t md_sector_exfat; uint32_t md_sector_ufs; + uint32_t md_sector_zfs; } runtime_config_t; typedef enum { @@ -128,6 +131,7 @@ typedef enum { IMAGE_FS_UFS, IMAGE_FS_EXFAT, IMAGE_FS_PFS, + IMAGE_FS_ZFS, } image_fs_type_t; typedef struct sm_error { diff --git a/makezfs.sh b/makezfs.sh new file mode 100644 index 0000000..b86e7bb --- /dev/null +++ b/makezfs.sh @@ -0,0 +1,544 @@ +#!/bin/sh +# Create a ZFS image from a application directory +# Usage: makezfs.sh [output_file] [OPTIONS] +# +# OPTIONS: +# --no-optimize Skip two-pass size optimization (write once, add fixed 256MB margin) +# --no-bruteforce Disable second-pass margin probing; use the requested margin once +# --margin N CoW margin percentage (default: 10, must be 1-50) +# Values lower than 10 may risk failure if compression is poor, values higher than 20 will waste space. +# +# ENVIRONMENT: +# ZFS_COMPRESSION Compression algorithm (default: lz4) +# ZFS_ASHIFT ZFS ashift value (default: 12) +# ZFS_RECORD_SIZE Record size in bytes (default: 131072) +# +# WHY MARGIN? +# ZFS uses copy-on-write (CoW), requiring free space during writes. +# The margin reserves space so rsync can complete safely. After the copy, +# the pool will be ~(100-margin)% full with minimal wasted space. +# Two-pass mode measures actual compressed size in pass 1, then sizes the image +# just large enough in pass 2. If that would exceed raw input size, it uses +# raw size instead (no compression benefit, but no risk of failure). +# Bruteforce mode is enabled by default for optimized runs: it first tries the +# requested margin, grows by +1% if needed to get a success, then probes downward +# in 1% steps using .tmp outputs until the first failure to keep the smallest +# working margin. +# +# EXAMPLES: +# sudo ./makezfs.sh ./APPXXXX ./output.ffzfs +# sudo ./makezfs.sh ./APPXXXX ./output.ffzfs --no-optimize +# sudo ./makezfs.sh ./APPXXXX ./output.ffzfs --no-bruteforce +# sudo ./makezfs.sh ./APPXXXX ./output.ffzfs --margin 10 +# ZFS_COMPRESSION=zstd ./makezfs.sh ./APPXXXX ./output.ffzfs +# +# Debian/Ubuntu: sudo apt-get install -y zfsutils-linux rsync +# macOS (Homebrew + OpenZFS): brew install openzfs rsync + +set -e + +if [ -z "$1" ]; then + echo "Usage: $0 [output_file] [--no-optimize] [--no-bruteforce] [--margin N]" + exit 1 +fi + +if [ "$(id -u)" -ne 0 ]; then + echo "Error: this script must run as root (zpool/zfs operations require it)." + exit 1 +fi + +INPUT_DIR="$1" +shift + +OUTPUT="download0.ffzfs" +if [ $# -gt 0 ] && [ "${1#--}" = "$1" ]; then + OUTPUT="$1" + shift +fi + +OPTIMIZE=true +BRUTEFORCE=true +MARGIN=10 + +is_positive_integer() { + case "$1" in + ''|*[!0-9]*) + return 1 + ;; + *) + return 0 + ;; + esac +} + +# Parse optional arguments +while [ $# -gt 0 ]; do + case "$1" in + --no-optimize) + OPTIMIZE=false + shift + ;; + --no-bruteforce) + BRUTEFORCE=false + shift + ;; + --margin) + if [ -z "$2" ]; then + echo "Error: --margin requires a value" + exit 1 + fi + MARGIN="$2" + if ! is_positive_integer "$MARGIN"; then + echo "Error: margin must be a positive integer percent" + exit 1 + fi + if [ "$MARGIN" -lt 1 ] || [ "$MARGIN" -gt 50 ]; then + echo "Error: margin must be between 1 and 50 percent" + exit 1 + fi + shift 2 + ;; + *) + echo "Error: unknown option: $1" + exit 1 + ;; + esac +done + +if [ "$OPTIMIZE" = "false" ]; then + BRUTEFORCE=false +fi + +if [ ! -d "$INPUT_DIR" ]; then + echo "Error: input directory not found: $INPUT_DIR" + exit 1 +fi + +if [ ! -f "$INPUT_DIR/eboot.bin" ]; then + echo "Error: eboot.bin not found in source directory: $INPUT_DIR" + exit 1 +fi + +# ZFS defaults tuned for speed/compatibility. +ZFS_COMPRESSION="${ZFS_COMPRESSION:-lz4}" +ASHIFT="${ZFS_ASHIFT:-12}" +RECORD_SIZE="${ZFS_RECORD_SIZE:-131072}" + +OS_TYPE=$(uname -s) +case "$OS_TYPE" in + Darwin|Linux) ;; + *) + echo "Error: unsupported OS: $OS_TYPE" + exit 1 + ;; +esac + +need_cmd() { + if ! command -v "$1" >/dev/null 2>&1; then + echo "Error: required command not found: $1" + exit 1 + fi +} + +need_cmd find +need_cmd awk +need_cmd grep +need_cmd rsync +need_cmd tee +need_cmd zpool +need_cmd zfs + +RSYNC_PROGRESS_ARGS="--progress" +if rsync --help 2>/dev/null | grep -q -- '--info'; then + RSYNC_PROGRESS_ARGS="--info=progress2" +fi + +rsync_copy() { + # progress2 shows total transfer progress when supported; fallback remains per-file progress. + rsync -r $RSYNC_PROGRESS_ARGS "$1"/ "$2"/ +} + +if [ "$OS_TYPE" = "Darwin" ]; then + need_cmd mkfile + need_cmd hdiutil + STAT_FMT="-f %z" +else + need_cmd truncate + need_cmd losetup + STAT_FMT="-c %s" +fi + +OUTPUT="$(cd "$(dirname "$OUTPUT")" && pwd)/$(basename "$OUTPUT")" + +SIZES_FILE=$(mktemp) +TMP_IMAGE=$(mktemp) +TMP_MOUNT=$(mktemp -d) +FINAL_MOUNT=$(mktemp -d) +TMP_POOL="smpzfs_t$$" +FINAL_POOL="smpzfs_f$$" +TMP_LOOP="" +FINAL_LOOP="" +TMP_POOL_CREATED=0 +FINAL_POOL_CREATED=0 +POOL_ALLOC=0 +MARGIN_BYTES=0 +FIXED_NO_OPT_MARGIN_BYTES=$((256 * 1024 * 1024)) +TMP_OUTPUT="${OUTPUT}.tmp" +SELECTED_MARGIN="$MARGIN" +CANDIDATE_FINAL_BYTES=0 +CANDIDATE_MARGIN_BYTES=0 +CANDIDATE_MB=0 +CANDIDATE_MARGIN_PCT_ACTUAL=0 +LAST_COPY_NO_SPACE=false +RSYNC_ERR_LOG=$(mktemp) +RSYNC_ERR_PIPE="${RSYNC_ERR_LOG}.pipe" +SELECTED_MARGIN_PCT_ACTUAL=0 + +detach_device() { + if [ -z "$1" ]; then + return 0 + fi + + if [ "$OS_TYPE" = "Darwin" ]; then + hdiutil detach "$1" >/dev/null 2>&1 || true + else + losetup -d "$1" >/dev/null 2>&1 || true + fi +} + +attach_image_file() { + image_path="$1" + image_mb="$2" + + if [ "$OS_TYPE" = "Darwin" ]; then + mkfile -n "${image_mb}m" "$image_path" || return 1 + ATTACH_OUTPUT=$(hdiutil attach -imagekey diskimage-class=CRawDiskImage -nomount "$image_path") || return 1 + loop_device=$(printf '%s\n' "$ATTACH_OUTPUT" | awk 'NR==1 {print $1; exit}') + if [ -z "$loop_device" ]; then + echo "Error: failed to determine attached raw device" + return 1 + fi + else + truncate -s "${image_mb}M" "$image_path" || return 1 + loop_device=$(losetup --find --show "$image_path") || return 1 + fi + + printf '%s\n' "$loop_device" +} + +cleanup_final_attempt() { + image_path="$1" + + if [ "$FINAL_POOL_CREATED" -eq 1 ]; then + zpool export "$FINAL_POOL" >/dev/null 2>&1 || true + FINAL_POOL_CREATED=0 + fi + + if [ -n "$FINAL_LOOP" ]; then + detach_device "$FINAL_LOOP" + FINAL_LOOP="" + fi + + rm -f "$image_path" +} + +compute_candidate_size() { + candidate_margin_pct="$1" + + candidate_margin_bytes=$(( (POOL_ALLOC * candidate_margin_pct) / 100 )) + candidate_final_bytes=$(( POOL_ALLOC + candidate_margin_bytes )) + + if [ "$candidate_final_bytes" -gt "$RAW_FILE_BYTES" ]; then + candidate_final_bytes="$RAW_FILE_BYTES" + candidate_margin_bytes=$(( RAW_FILE_BYTES - POOL_ALLOC )) + if [ "$candidate_margin_bytes" -lt 0 ]; then + candidate_margin_bytes=0 + fi + fi + + CANDIDATE_FINAL_BYTES="$candidate_final_bytes" + CANDIDATE_MARGIN_BYTES="$candidate_margin_bytes" + CANDIDATE_MB=$(( (candidate_final_bytes + 1024*1024 - 1) / (1024*1024) )) + if [ "$POOL_ALLOC" -gt 0 ] && [ "$candidate_margin_bytes" -gt 0 ]; then + CANDIDATE_MARGIN_PCT_ACTUAL=$(( (candidate_margin_bytes * 100) / POOL_ALLOC )) + else + CANDIDATE_MARGIN_PCT_ACTUAL=0 + fi +} + +run_final_pass() { + image_path="$1" + image_mb="$2" + stage_prefix="$3" + + echo "$stage_prefix Creating image container (${image_mb}MB)..." + FINAL_LOOP=$(attach_image_file "$image_path" "$image_mb") || return 1 + + echo "$stage_prefix Creating ZFS pool and copying files..." + if ! zpool create -f \ + -o ashift="$ASHIFT" \ + -O compression="$ZFS_COMPRESSION" \ + -O atime=off \ + -O mountpoint="$FINAL_MOUNT" \ + "$FINAL_POOL" "$FINAL_LOOP"; then + cleanup_final_attempt "$image_path" + return 1 + fi + FINAL_POOL_CREATED=1 + + if ! zfs set recordsize="$RECORD_SIZE" "$FINAL_POOL"; then + cleanup_final_attempt "$image_path" + return 1 + fi + + LAST_COPY_NO_SPACE=false + : > "$RSYNC_ERR_LOG" + rm -f "$RSYNC_ERR_PIPE" + if ! mkfifo "$RSYNC_ERR_PIPE"; then + cleanup_final_attempt "$image_path" + return 1 + fi + + tee -a "$RSYNC_ERR_LOG" < "$RSYNC_ERR_PIPE" >&2 & + tee_pid=$! + if rsync_copy "$INPUT_DIR" "$FINAL_MOUNT" 2>"$RSYNC_ERR_PIPE"; then + rsync_status=0 + else + rsync_status=$? + fi + wait "$tee_pid" >/dev/null 2>&1 || true + rm -f "$RSYNC_ERR_PIPE" + + if [ "$rsync_status" -ne 0 ]; then + if grep -qi 'No space left on device' "$RSYNC_ERR_LOG"; then + LAST_COPY_NO_SPACE=true + fi + cleanup_final_attempt "$image_path" + return 1 + fi + + echo "$stage_prefix Finalizing image (exporting pool)..." + if ! zpool export "$FINAL_POOL"; then + cleanup_final_attempt "$image_path" + return 1 + fi + FINAL_POOL_CREATED=0 + + detach_device "$FINAL_LOOP" + FINAL_LOOP="" + return 0 +} + +cleanup() { + if [ "$FINAL_POOL_CREATED" -eq 1 ]; then + zpool export "$FINAL_POOL" >/dev/null 2>&1 || true + fi + if [ "$TMP_POOL_CREATED" -eq 1 ]; then + zpool export "$TMP_POOL" >/dev/null 2>&1 || true + fi + if [ -n "$FINAL_LOOP" ]; then + detach_device "$FINAL_LOOP" + fi + if [ -n "$TMP_LOOP" ]; then + detach_device "$TMP_LOOP" + fi + rm -f "$SIZES_FILE" "$TMP_IMAGE" + rm -f "$RSYNC_ERR_LOG" + rm -f "$RSYNC_ERR_PIPE" + rm -f "$TMP_OUTPUT" + rmdir "$TMP_MOUNT" >/dev/null 2>&1 || true + rmdir "$FINAL_MOUNT" >/dev/null 2>&1 || true +} +trap cleanup EXIT INT TERM + +# Collect raw file sizes to estimate a safe temporary image size. +if [ "$OS_TYPE" = "Darwin" ]; then + find "$INPUT_DIR" -type f -exec stat $STAT_FMT {} \; > "$SIZES_FILE" +else + find "$INPUT_DIR" -type f -exec stat $STAT_FMT {} + > "$SIZES_FILE" +fi + +RAW_FILE_BYTES=$(awk '{s += $1} END {print s+0}' "$SIZES_FILE") +rm -f "$SIZES_FILE" + +echo "Input folder: $INPUT_DIR" +echo "Output file: $OUTPUT" +echo "ZFS profile: compression=$ZFS_COMPRESSION ashift=$ASHIFT recordsize=$RECORD_SIZE" +echo "Input size (raw files): $RAW_FILE_BYTES bytes (~$(( RAW_FILE_BYTES / 1024 / 1024 )) MB)" +if [ "$OPTIMIZE" = "true" ]; then + if [ "$BRUTEFORCE" = "true" ]; then + echo "Optimize mode: two-pass with bruteforce margin probing enabled" + else + echo "Optimize mode: two-pass with single margin attempt" + fi +else + echo "Optimize mode: disabled (--no-optimize)" +fi + +if [ "$OPTIMIZE" = "false" ]; then + # Single-pass: no compression measurement, use raw input size + fixed safety margin + FINAL_BYTES=$((RAW_FILE_BYTES + FIXED_NO_OPT_MARGIN_BYTES)) + MB=$(( (FINAL_BYTES + 1024*1024 - 1) / (1024*1024) )) + MARGIN_BYTES=$FIXED_NO_OPT_MARGIN_BYTES + echo "" + echo "[1/3] Creating image (single-pass mode, no size optimization, +256MB safety margin)..." + + if ! run_final_pass "$OUTPUT" "$MB" "[single-pass]"; then + echo "Error: failed to create image in single-pass mode" + exit 1 + fi + + POOL_ALLOC=0 +else + # Two-pass mode: measure compression, then size optimally + TMP_MB=$(( ((RAW_FILE_BYTES * 5) / 4 + 256*1024*1024 + 1024*1024 - 1) / (1024*1024) )) + + echo "" + echo "Pass 1/${TMP_MB}MB — measuring actual compressed allocation..." + + # ── pass 1: write to oversized temp image ──────────────────────────────────── + echo "[1/6] Creating temporary image for pass 1..." + if [ "$OS_TYPE" = "Darwin" ]; then + mkfile -n "${TMP_MB}m" "$TMP_IMAGE" + TMP_ATTACH=$(hdiutil attach -imagekey diskimage-class=CRawDiskImage -nomount "$TMP_IMAGE") + TMP_LOOP=$(printf '%s\n' "$TMP_ATTACH" | awk 'NR==1 {print $1; exit}') + if [ -z "$TMP_LOOP" ]; then + echo "Error: failed to attach temp image" + exit 1 + fi + else + truncate -s "${TMP_MB}M" "$TMP_IMAGE" + TMP_LOOP=$(losetup --find --show "$TMP_IMAGE") + fi + + echo "[2/6] Creating temporary pool and copying files (pass 1)..." + zpool create -f \ + -o ashift="$ASHIFT" \ + -O compression="$ZFS_COMPRESSION" \ + -O atime=off \ + -O mountpoint="$TMP_MOUNT" \ + "$TMP_POOL" "$TMP_LOOP" + TMP_POOL_CREATED=1 + + zfs set recordsize="$RECORD_SIZE" "$TMP_POOL" + rsync_copy "$INPUT_DIR" "$TMP_MOUNT" + + echo "[3/6] Measuring compressed allocation and closing pass 1..." + # Measure actual bytes allocated in the pool (compressed data + all ZFS internal metadata). + POOL_ALLOC=$(zpool list -Hp -o alloc "$TMP_POOL") + + zpool export "$TMP_POOL" + TMP_POOL_CREATED=0 + + if [ "$OS_TYPE" = "Darwin" ]; then + hdiutil detach "$TMP_LOOP" >/dev/null 2>&1 || true + else + losetup -d "$TMP_LOOP" >/dev/null 2>&1 || true + fi + TMP_LOOP="" + rm -f "$TMP_IMAGE" + + echo "Compressed allocation (pass 1): $POOL_ALLOC bytes (~$(( POOL_ALLOC / 1024 / 1024 )) MB)" + if [ "$BRUTEFORCE" = "true" ]; then + echo "Searching for the smallest working pass-2 margin..." + else + echo "Running pass 2 with requested margin ${MARGIN}%..." + fi + + compute_candidate_size "$MARGIN" + probe_margin="$MARGIN" + probe_success=false + + while [ "$probe_success" = "false" ]; do + echo "[pass2] Trying margin=${probe_margin}% -> ${CANDIDATE_MB}MB" + rm -f "$TMP_OUTPUT" + if run_final_pass "$TMP_OUTPUT" "$CANDIDATE_MB" "[pass2 ${probe_margin}%]"; then + mv -f "$TMP_OUTPUT" "$OUTPUT" + probe_success=true + SELECTED_MARGIN="$probe_margin" + SELECTED_MARGIN_PCT_ACTUAL="$CANDIDATE_MARGIN_PCT_ACTUAL" + MARGIN_BYTES="$CANDIDATE_MARGIN_BYTES" + FINAL_BYTES="$CANDIDATE_FINAL_BYTES" + MB="$CANDIDATE_MB" + else + if [ "$BRUTEFORCE" != "true" ]; then + echo "Error: pass 2 failed at margin ${probe_margin}%" + exit 1 + fi + + if [ "$LAST_COPY_NO_SPACE" != "true" ]; then + echo "Error: pass 2 failed for a reason other than running out of space" + exit 1 + fi + + if [ "$CANDIDATE_FINAL_BYTES" -ge "$RAW_FILE_BYTES" ]; then + echo "Error: pass 2 failed even at the raw-size cap; cannot grow further within current sizing policy" + exit 1 + fi + + probe_margin=$((probe_margin + 1)) + if [ "$probe_margin" -gt 50 ]; then + echo "Error: no working pass-2 margin found up to 50%" + exit 1 + fi + compute_candidate_size "$probe_margin" + fi + done + + if [ "$BRUTEFORCE" = "true" ]; then + probe_margin=$((SELECTED_MARGIN - 1)) + while [ "$probe_margin" -ge 0 ]; do + compute_candidate_size "$probe_margin" + echo "[pass2] Probing smaller margin=${probe_margin}% -> ${CANDIDATE_MB}MB" + rm -f "$TMP_OUTPUT" + if run_final_pass "$TMP_OUTPUT" "$CANDIDATE_MB" "[probe ${probe_margin}%]"; then + mv -f "$TMP_OUTPUT" "$OUTPUT" + SELECTED_MARGIN="$probe_margin" + SELECTED_MARGIN_PCT_ACTUAL="$CANDIDATE_MARGIN_PCT_ACTUAL" + MARGIN_BYTES="$CANDIDATE_MARGIN_BYTES" + FINAL_BYTES="$CANDIDATE_FINAL_BYTES" + MB="$CANDIDATE_MB" + probe_margin=$((probe_margin - 1)) + else + if [ "$LAST_COPY_NO_SPACE" != "true" ]; then + echo "Error: probe at margin ${probe_margin}% failed for a reason other than running out of space" + exit 1 + fi + echo "[pass2] Margin ${probe_margin}% failed; keeping last successful image" + break + fi + done + fi + + MARGIN="$SELECTED_MARGIN" +fi + +# ── output summary ──────────────────────────────────────────────────────────── +FINAL_IMAGE_BYTES=$(( MB * 1024 * 1024 )) +REDUCTION_PCT=0 +if [ "$RAW_FILE_BYTES" -gt 0 ]; then + REDUCTION_PCT=$(( (RAW_FILE_BYTES - FINAL_IMAGE_BYTES) * 100 / RAW_FILE_BYTES )) +fi + +if [ "$POOL_ALLOC" = "0" ]; then + COMPRESSION_PCT="(not measured)" +else + COMPRESSION_PCT=$(( (RAW_FILE_BYTES - POOL_ALLOC) * 100 / RAW_FILE_BYTES )) +fi + +echo "" +echo "Task completed successfully!" +echo "" +echo "Compression Results:" +echo " Input size (raw files): $RAW_FILE_BYTES bytes (~$(( RAW_FILE_BYTES / 1024 / 1024 )) MB)" +if [ "$POOL_ALLOC" != "0" ]; then + echo " Compressed data: $POOL_ALLOC bytes (~$(( POOL_ALLOC / 1024 / 1024 )) MB, $COMPRESSION_PCT% reduction)" +fi +if [ "$MARGIN_BYTES" -gt 0 ]; then + if [ "$OPTIMIZE" = "false" ]; then + echo " Free space margin: $MARGIN_BYTES bytes (~$(( MARGIN_BYTES / 1024 / 1024 )) MB, fixed in --no-optimize mode)" + else + echo " Free space margin: $MARGIN_BYTES bytes (~$(( MARGIN_BYTES / 1024 / 1024 )) MB, ${SELECTED_MARGIN_PCT_ACTUAL}% actual)" + fi +fi +echo " Final image: $FINAL_IMAGE_BYTES bytes (~${MB} MB, $REDUCTION_PCT% reduction)" diff --git a/src/sm_config_mount.c b/src/sm_config_mount.c index 4ce34d4..d6548f7 100644 --- a/src/sm_config_mount.c +++ b/src/sm_config_mount.c @@ -153,6 +153,14 @@ static attach_backend_t default_ufs_backend(void) { #endif } +static attach_backend_t default_zfs_backend(void) { +#if ZFS_ATTACH_USE_MDCTL + return ATTACH_BACKEND_MD; +#else + return ATTACH_BACKEND_LVD; +#endif +} + static void clear_runtime_scan_paths(runtime_config_state_t *state) { state->scan_path_count = 0; memset(state->scan_path_storage, 0, sizeof(state->scan_path_storage)); @@ -223,11 +231,14 @@ static void init_runtime_config_defaults(runtime_config_state_t *state) { DEFAULT_KSTUFF_PAUSE_DELAY_DIRECT_SECONDS; state->cfg.exfat_backend = default_exfat_backend(); state->cfg.ufs_backend = default_ufs_backend(); + state->cfg.zfs_backend = default_zfs_backend(); state->cfg.lvd_sector_exfat = LVD_SECTOR_SIZE_EXFAT; state->cfg.lvd_sector_ufs = LVD_SECTOR_SIZE_UFS; + state->cfg.lvd_sector_zfs = LVD_SECTOR_SIZE_ZFS; state->cfg.lvd_sector_pfs = LVD_SECTOR_SIZE_PFS; state->cfg.md_sector_exfat = MD_SECTOR_SIZE_EXFAT; state->cfg.md_sector_ufs = MD_SECTOR_SIZE_UFS; + state->cfg.md_sector_zfs = MD_SECTOR_SIZE_ZFS; memset(state->image_mode_rules, 0, sizeof(state->image_mode_rules)); clear_kstuff_title_rules(state); init_runtime_scan_paths_defaults(state); @@ -951,6 +962,16 @@ static config_load_status_t load_runtime_config_state(runtime_config_state_t *st continue; } + if (strcasecmp(key, "zfs_backend") == 0) { + if (!parse_backend_ini(value, &backend)) { + log_debug(" [CFG] invalid backend at line %d: %s=%s", line_no, key, + value); + continue; + } + state->cfg.zfs_backend = backend; + continue; + } + if (strcasecmp(key, "scanpath") == 0) { if (!has_custom_scanpaths) { clear_runtime_scan_paths(state); @@ -966,9 +987,11 @@ static config_load_status_t load_runtime_config_state(runtime_config_state_t *st bool is_sector_key = (strcasecmp(key, "lvd_exfat_sector_size") == 0) || (strcasecmp(key, "lvd_ufs_sector_size") == 0) || + (strcasecmp(key, "lvd_zfs_sector_size") == 0) || (strcasecmp(key, "lvd_pfs_sector_size") == 0) || (strcasecmp(key, "md_exfat_sector_size") == 0) || - (strcasecmp(key, "md_ufs_sector_size") == 0); + (strcasecmp(key, "md_ufs_sector_size") == 0) || + (strcasecmp(key, "md_zfs_sector_size") == 0); if (!is_sector_key) { log_debug(" [CFG] unknown key at line %d: %s", line_no, key); @@ -985,12 +1008,16 @@ static config_load_status_t load_runtime_config_state(runtime_config_state_t *st state->cfg.lvd_sector_exfat = u32; } else if (strcasecmp(key, "lvd_ufs_sector_size") == 0) { state->cfg.lvd_sector_ufs = u32; + } else if (strcasecmp(key, "lvd_zfs_sector_size") == 0) { + state->cfg.lvd_sector_zfs = u32; } else if (strcasecmp(key, "lvd_pfs_sector_size") == 0) { state->cfg.lvd_sector_pfs = u32; } else if (strcasecmp(key, "md_exfat_sector_size") == 0) { state->cfg.md_sector_exfat = u32; } else if (strcasecmp(key, "md_ufs_sector_size") == 0) { state->cfg.md_sector_ufs = u32; + } else if (strcasecmp(key, "md_zfs_sector_size") == 0) { + state->cfg.md_sector_zfs = u32; } } @@ -1025,8 +1052,8 @@ static config_load_status_t load_runtime_config_state(runtime_config_state_t *st "legacy_recursive_scan_forced=%d backport_fakelib=%d " "kstuff_game_auto_toggle=%d " "kstuff_pause_delay_image_s=%u kstuff_pause_delay_direct_s=%u " - "exfat_backend=%s ufs_backend=%s " - "lvd_sec(exfat=%u ufs=%u pfs=%u) md_sec(exfat=%u ufs=%u) " + "exfat_backend=%s ufs_backend=%s zfs_backend=%s " + "lvd_sec(exfat=%u ufs=%u zfs=%u pfs=%u) md_sec(exfat=%u ufs=%u zfs=%u) " "scan_interval_s=%u stability_wait_s=%u scan_paths=%d image_rules=%d " "kstuff_no_pause=%d kstuff_delay_rules=%d", state->cfg.debug_enabled ? 1 : 0, state->cfg.quiet_mode ? 1 : 0, @@ -1039,9 +1066,11 @@ static config_load_status_t load_runtime_config_state(runtime_config_state_t *st state->cfg.kstuff_pause_delay_direct_seconds, attach_backend_name(state->cfg.exfat_backend), attach_backend_name(state->cfg.ufs_backend), + attach_backend_name(state->cfg.zfs_backend), state->cfg.lvd_sector_exfat, state->cfg.lvd_sector_ufs, - state->cfg.lvd_sector_pfs, state->cfg.md_sector_exfat, - state->cfg.md_sector_ufs, state->cfg.scan_interval_us / 1000000u, + state->cfg.lvd_sector_zfs, state->cfg.lvd_sector_pfs, + state->cfg.md_sector_exfat, state->cfg.md_sector_ufs, + state->cfg.md_sector_zfs, state->cfg.scan_interval_us / 1000000u, state->cfg.stability_wait_seconds, state->scan_path_count, image_rule_count, state->kstuff_no_pause_title_count, kstuff_delay_rule_count); diff --git a/src/sm_image.c b/src/sm_image.c index 838d1f6..20de2d3 100644 --- a/src/sm_image.c +++ b/src/sm_image.c @@ -19,6 +19,8 @@ static uint32_t get_lvd_sector_size_fallback(image_fs_type_t fs_type) { switch (fs_type) { case IMAGE_FS_UFS: return cfg->lvd_sector_ufs; + case IMAGE_FS_ZFS: + return cfg->lvd_sector_zfs; case IMAGE_FS_PFS: return cfg->lvd_sector_pfs; case IMAGE_FS_EXFAT: @@ -54,6 +56,8 @@ static uint32_t get_md_sector_size(image_fs_type_t fs_type) { switch (fs_type) { case IMAGE_FS_UFS: return cfg->md_sector_ufs; + case IMAGE_FS_ZFS: + return cfg->md_sector_zfs; case IMAGE_FS_EXFAT: default: return cfg->md_sector_exfat; @@ -106,6 +110,8 @@ static uint16_t get_lvd_image_type(image_fs_type_t fs_type) { return LVD_ATTACH_IMAGE_TYPE_UFS_DOWNLOAD_DATA; if (fs_type == IMAGE_FS_PFS) return LVD_ATTACH_IMAGE_TYPE_PFS_SAVE_DATA; + if (fs_type == IMAGE_FS_ZFS) + return LVD_ATTACH_IMAGE_TYPE_ZFS; return LVD_ATTACH_IMAGE_TYPE_SINGLE; } @@ -132,6 +138,8 @@ static image_fs_type_t detect_image_fs_type(const char *name) { return IMAGE_FS_EXFAT; if (strcasecmp(dot, ".ffpfs") == 0) return IMAGE_FS_PFS; + if (strcasecmp(dot, ".ffzfs") == 0) + return IMAGE_FS_ZFS; return IMAGE_FS_UNKNOWN; } @@ -147,6 +155,8 @@ static const char *image_fs_name(image_fs_type_t fs_type) { return "exfatfs"; case IMAGE_FS_PFS: return "pfs"; + case IMAGE_FS_ZFS: + return "zfs"; default: return "unknown"; } @@ -531,6 +541,8 @@ static attach_backend_t select_image_backend(const runtime_config_t *cfg, return cfg->exfat_backend; if (fs_type == IMAGE_FS_UFS) return cfg->ufs_backend; + if (fs_type == IMAGE_FS_ZFS) + return cfg->zfs_backend; return ATTACH_BACKEND_LVD; } @@ -609,6 +621,17 @@ static bool perform_image_nmount(const char *file_path, image_fs_type_t fs_type, IOVEC_ENTRY("errmsg"), {(void *)mount_errmsg, sizeof(mount_errmsg)}, IOVEC_ENTRY("force"), IOVEC_ENTRY(NULL)}; + struct iovec iov_zfs[] = { + IOVEC_ENTRY("from"), IOVEC_ENTRY(devname), + IOVEC_ENTRY("fspath"), IOVEC_ENTRY(mount_point), + IOVEC_ENTRY("fstype"), IOVEC_ENTRY("zfs"), + IOVEC_ENTRY("budgetid"), IOVEC_ENTRY(DEVPFS_BUDGET_GAME), + IOVEC_ENTRY("async"), IOVEC_ENTRY(NULL), + IOVEC_ENTRY("noatime"), IOVEC_ENTRY(NULL), + IOVEC_ENTRY("automounted"), IOVEC_ENTRY(NULL), + IOVEC_ENTRY("errmsg"), {(void *)mount_errmsg, sizeof(mount_errmsg)}, + IOVEC_ENTRY("force"), IOVEC_ENTRY(NULL)}; + if (fs_type == IMAGE_FS_UFS) { iov = iov_ufs; iovlen = (unsigned int)IOVEC_SIZE(iov_ufs) - (force_mount ? 0u : 2u); @@ -622,6 +645,9 @@ static bool perform_image_nmount(const char *file_path, image_fs_type_t fs_type, PFS_MOUNT_BUDGET_ID, PFS_MOUNT_MKEYMODE, sigverify, playgo, disc); iov = iov_pfs; iovlen = (unsigned int)IOVEC_SIZE(iov_pfs) - (force_mount ? 0u : 2u); + } else if (fs_type == IMAGE_FS_ZFS) { + iov = iov_zfs; + iovlen = (unsigned int)IOVEC_SIZE(iov_zfs) - (force_mount ? 0u : 2u); } else { log_debug(" [IMG][%s] unsupported fstype=%s", attach_backend_name(attach_backend), image_fs_name(fs_type)); From a82f408a6520b5b7b685073eeed53175274d3baf Mon Sep 17 00:00:00 2001 From: xylonzinho Date: Sun, 29 Mar 2026 14:01:32 -0300 Subject: [PATCH 2/2] Small fix on the Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7b7e046..0c67f13 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ PS5_PAYLOAD_SDK ?= /opt/ps5-payload-sdk # Allow utility-only targets to run without the PS5 SDK toolchain present. -ifeq ($(filter clean makezfs,$(MAKECMDGOALS)),) +ifneq ($(filter-out clean makezfs,$(MAKECMDGOALS)),) include $(PS5_PAYLOAD_SDK)/toolchain/prospero.mk endif