diff --git a/app/src/main/java/cn/classfun/droidvm/daemon/ipc/vm/ExportHandler.java b/app/src/main/java/cn/classfun/droidvm/daemon/ipc/vm/ExportHandler.java index 28cf88e..dab0c13 100644 --- a/app/src/main/java/cn/classfun/droidvm/daemon/ipc/vm/ExportHandler.java +++ b/app/src/main/java/cn/classfun/droidvm/daemon/ipc/vm/ExportHandler.java @@ -8,6 +8,7 @@ import cn.classfun.droidvm.daemon.server.RequestException; import cn.classfun.droidvm.daemon.server.RequestHandler; import cn.classfun.droidvm.daemon.vm.pkg.VMExportTask; +import cn.classfun.droidvm.lib.pkg.PackageConstants; @AutoService(RequestHandler.class) public final class ExportHandler extends RequestHandler { @@ -25,6 +26,12 @@ public void handle(@NonNull ClientRequest request) throws Exception { throw new RequestException("missing vm_id"); if (destPath.isEmpty()) throw new RequestException("missing dest_path"); + var volumeSize = params.optLong("volume_size", 0); + if (volumeSize > 0 && volumeSize < PackageConstants.MIN_VOLUME_SIZE) + throw new RequestException(String.format( + "volume_size too small: %d (min %d)", + volumeSize, PackageConstants.MIN_VOLUME_SIZE + )); var store = request.getContext().getExportTaskStore(); var server = request.getClient().getServer(); var task = new VMExportTask(server, params); diff --git a/app/src/main/java/cn/classfun/droidvm/daemon/vm/pkg/VMExportTask.java b/app/src/main/java/cn/classfun/droidvm/daemon/vm/pkg/VMExportTask.java index b406749..a5e1e70 100644 --- a/app/src/main/java/cn/classfun/droidvm/daemon/vm/pkg/VMExportTask.java +++ b/app/src/main/java/cn/classfun/droidvm/daemon/vm/pkg/VMExportTask.java @@ -36,6 +36,8 @@ import cn.classfun.droidvm.lib.pkg.PackageConstants; import cn.classfun.droidvm.lib.pkg.PackageManifest; import cn.classfun.droidvm.lib.pkg.Phase; +import cn.classfun.droidvm.lib.pkg.VolumeIndex; +import cn.classfun.droidvm.lib.pkg.VolumeSplitOutputStream; import cn.classfun.droidvm.lib.store.base.DataItem; public final class VMExportTask { @@ -47,6 +49,9 @@ public final class VMExportTask { private final UUID sourceVMId; private final int totalItems; private final String destPath; + private final long volumeSize; + private VolumeSplitOutputStream vout = null; + private int finalVolumeCount = 0; public VMExportTask( @NonNull Server server, @@ -56,6 +61,7 @@ public VMExportTask( destPath = request.optString("dest_path"); if (!destPath.startsWith("/")) throw new IllegalArgumentException("missing dest_path"); + volumeSize = request.optLong("volume_size", 0); var vmId = UUID.fromString(request.optString("vm_id")); sourceVMId = vmId; var vm = server.getContext().getVMs().findById(vmId); @@ -91,6 +97,10 @@ public void startAsync() { try { startWrite(); data.set("done", totalItems); + if (finalVolumeCount > 0) { + data.set("volume_index", finalVolumeCount); + data.set("volume_total", finalVolumeCount); + } emit(data, Phase.DONE); } catch (Exception e) { Log.w(TAG, fmt("Export task %s failed", taskId), e); @@ -105,6 +115,11 @@ public void startAsync() { } private void startWrite() throws Exception { + if (volumeSize > 0) startWriteSplit(); + else startWriteSingle(); + } + + private void startWriteSingle() throws Exception { var outFile = new File(destPath); var manifestBytes = manifest.toJson().toString(4).getBytes(UTF_8); if (manifestBytes.length > 0xffff) @@ -132,6 +147,75 @@ private void startWrite() throws Exception { } } + // Multi-volume path: (1) slice the compressed data stream into .001, + // .002, ... sub-volumes, then (2) write the metadata master holding + // header + manifest + volume index (no data). The picked SAF file is reused + // as the master, so it is not deleted. + private void startWriteSplit() throws Exception { + var manifestBytes = manifest.toJson().toString(4).getBytes(UTF_8); + if (manifestBytes.length > 0xffff) + throw new IOException(fmt("manifest too large: %d bytes", manifestBytes.length)); + var subOut = new VolumeSplitOutputStream(destPath, volumeSize); + vout = subOut; + try { + try ( + var compressed = wrapCompressionOutput(subOut, manifest.compression); + var tar = new TarWriter(compressed) + ) { + tar.entry(PackageConstants.MANIFEST_NAME, manifestBytes); + pack(tar); + } + subOut.finish(); + } catch (Exception e) { + subOut.abort(); + deleteQuietly(subOut.files()); + throw e; + } finally { + vout = null; + } + long dataSize = subOut.dataSize(); + finalVolumeCount = subOut.volumeCount(); + writeMaster(manifestBytes, dataSize, subOut); + // Remove any higher-numbered sub-volumes left by a previous, larger + // export to the same base path, so the on-disk set matches the index. + for (int i = finalVolumeCount + 1; i <= PackageConstants.VOLUME_MAX_COUNT; i++) { + var stale = new File(VolumeSplitOutputStream.volumeName(destPath, i)); + if (!stale.exists()) break; + //noinspection ResultOfMethodCallIgnored + stale.delete(); + } + } + + private static void deleteQuietly(@NonNull java.util.List files) { + for (var f : files) { + //noinspection ResultOfMethodCallIgnored + f.delete(); + } + } + + private void writeMaster( + @NonNull byte[] manifestBytes, + long dataSize, + @NonNull VolumeSplitOutputStream subOut + ) throws Exception { + var index = new VolumeIndex(); + index.dataSize = dataSize; + index.volumes.addAll(subOut.entries()); + var indexBytes = index.toBytes(); + try (var raf = new RandomAccessFile(new File(destPath), "rw")) { + raf.setLength(0); + raf.write(new byte[PackageConstants.HEADER_SIZE]); + writeZero(raf, alignUp(raf.getFilePointer()) - raf.getFilePointer()); + raf.write(manifestBytes); + writeZero(raf, alignUpStrict(raf.getFilePointer()) - raf.getFilePointer()); + raf.write(indexBytes); + raf.seek(0); + raf.write(buildHeader( + manifest, manifestBytes.length, dataSize, finalVolumeCount + )); + } + } + private void pack(@NonNull TarWriter tar) throws Exception { int done; var data = DataItem.newObject(); @@ -209,6 +293,9 @@ private void emit(@NonNull DataItem item, @NonNull Phase phase) { data.put("phase", phase.name().toLowerCase()); data.put("vm_id", sourceVMId.toString()); data.put("vm_name", manifest.vm.getName()); + var out = vout; + if (out != null && !data.has("volume_index")) + data.put("volume_index", out.currentIndex()); var ev = new JSONObject(); ev.put("type", "event"); ev.put("data", data); diff --git a/app/src/main/java/cn/classfun/droidvm/daemon/vm/pkg/VMExportUtils.java b/app/src/main/java/cn/classfun/droidvm/daemon/vm/pkg/VMExportUtils.java index 2ed8a79..83f3822 100644 --- a/app/src/main/java/cn/classfun/droidvm/daemon/vm/pkg/VMExportUtils.java +++ b/app/src/main/java/cn/classfun/droidvm/daemon/vm/pkg/VMExportUtils.java @@ -58,6 +58,16 @@ public static byte[] buildHeader( @NonNull PackageManifest manifest, int manifestSize, long dataSize + ) throws IOException { + return buildHeader(manifest, manifestSize, dataSize, 0); + } + + @NonNull + public static byte[] buildHeader( + @NonNull PackageManifest manifest, + int manifestSize, + long dataSize, + int volumeCount ) throws IOException { var hdr = new byte[PackageConstants.HEADER_SIZE]; var magic = PackageConstants.MAGIC.getBytes(UTF_8); @@ -66,6 +76,7 @@ public static byte[] buildHeader( putUInt16LE(hdr, 8, BuildConfig.VERSION_CODE); putUInt16LE(hdr, 10, manifestSize); putUInt16LE(hdr, 12, manifest.compression.type); + putUInt16LE(hdr, 14, volumeCount); putInt64LE(hdr, 16, dataSize); return hdr; } diff --git a/app/src/main/java/cn/classfun/droidvm/daemon/vm/pkg/VMImportTask.java b/app/src/main/java/cn/classfun/droidvm/daemon/vm/pkg/VMImportTask.java index dcf56b7..9f79260 100644 --- a/app/src/main/java/cn/classfun/droidvm/daemon/vm/pkg/VMImportTask.java +++ b/app/src/main/java/cn/classfun/droidvm/daemon/vm/pkg/VMImportTask.java @@ -4,6 +4,7 @@ import static cn.classfun.droidvm.daemon.vm.pkg.VMImportUtils.remapDiskPaths; import static cn.classfun.droidvm.daemon.vm.pkg.VMImportUtils.uniqueFile; import static cn.classfun.droidvm.lib.pkg.PackageConstants.BUFFER; +import static cn.classfun.droidvm.lib.utils.BinaryUtils.readFully; import static cn.classfun.droidvm.lib.utils.JsonUtils.listToJSONArray; import static cn.classfun.droidvm.lib.utils.StringUtils.basename; import static cn.classfun.droidvm.lib.utils.StringUtils.fmt; @@ -31,9 +32,12 @@ import cn.classfun.droidvm.lib.archive.TarReader; import cn.classfun.droidvm.lib.pkg.BootFile; import cn.classfun.droidvm.lib.pkg.DiskEntry; +import cn.classfun.droidvm.lib.pkg.PackageConstants; +import cn.classfun.droidvm.lib.pkg.PackageHeader; import cn.classfun.droidvm.lib.pkg.PackageInput; import cn.classfun.droidvm.lib.pkg.PackageManifest; import cn.classfun.droidvm.lib.pkg.Phase; +import cn.classfun.droidvm.lib.pkg.VolumeSet; import cn.classfun.droidvm.lib.store.base.DataItem; import cn.classfun.droidvm.lib.store.network.NetworkConfig; import cn.classfun.droidvm.lib.store.vm.VMConfig; @@ -47,6 +51,7 @@ public final class VMImportTask { private final File targetDir; private final String networkMode; private int totalItems; + private int volumeTotal = 0; public VMConfig importedVM = null; public PackageManifest importedManifest = null; public final ArrayList placedDisks = new ArrayList<>(); @@ -125,22 +130,24 @@ private void onTarItem( private void unpack() throws Exception { if (!targetDir.exists() && !targetDir.mkdirs()) throw new IOException(fmt("Cannot create target dir: %s", targetDir)); - try ( - var in = new FileInputStream(srcPath); - var pkg = PackageInput.open(in) - ) { - importedManifest = pkg.manifest; - totalItems = importedManifest.disks.size() + importedManifest.boots.size(); - var data = DataItem.newObject(); - data.set("done", 0); - data.set("total", totalItems); - emit(data, Phase.PACK); - var tar = new TarReader(pkg.data); - tar.forEach(this::onTarItem); - var buf = new byte[BUFFER]; - //noinspection StatementWithEmptyBody - while (pkg.data.read(buf) >= 0); - pkg.validateDataConsumed(); + // Any picked path maps to its metadata master (strip a .NNN suffix). + var masterPath = VolumeSet.masterOf(srcPath); + if (readVolumeCount(masterPath) > 0) { + var set = VolumeSet.discover(masterPath); + volumeTotal = set.count(); + try ( + var in = set.openLogicalStream(); + var pkg = PackageInput.open(in, set.dataSize()) + ) { + extract(pkg); + } + } else { + try ( + var in = new FileInputStream(masterPath); + var pkg = PackageInput.open(in) + ) { + extract(pkg); + } } var vm = new VMConfig(importedManifest.vm.toJson()); vm.setId(UUID.randomUUID()); @@ -149,6 +156,29 @@ private void unpack() throws Exception { importedVM = vm; } + private int readVolumeCount(@NonNull String masterPath) throws Exception { + try (var in = new FileInputStream(masterPath)) { + var hdr = new byte[PackageConstants.HEADER_SIZE]; + readFully(in, hdr); + return PackageHeader.fromBytes(hdr).volumeCount; + } + } + + private void extract(@NonNull PackageInput pkg) throws Exception { + importedManifest = pkg.manifest; + totalItems = importedManifest.disks.size() + importedManifest.boots.size(); + var data = DataItem.newObject(); + data.set("done", 0); + data.set("total", totalItems); + emit(data, Phase.PACK); + var tar = new TarReader(pkg.data); + tar.forEach(this::onTarItem); + var buf = new byte[BUFFER]; + //noinspection StatementWithEmptyBody + while (pkg.data.read(buf) >= 0); + pkg.validateDataConsumed(); + } + @NonNull private JSONArray importNetworks( @NonNull VMConfig vm, @@ -281,6 +311,7 @@ private void emit(@NonNull DataItem item, @NonNull Phase phase) { data.put("event", "vm_import_status"); data.put("task_id", taskId.toString()); data.put("phase", phase.name().toLowerCase()); + if (volumeTotal > 0) data.put("volume_total", volumeTotal); var ev = new JSONObject(); ev.put("type", "event"); ev.put("data", data); diff --git a/app/src/main/java/cn/classfun/droidvm/lib/archive/ZeroInputStream.java b/app/src/main/java/cn/classfun/droidvm/lib/archive/ZeroInputStream.java new file mode 100644 index 0000000..00e899a --- /dev/null +++ b/app/src/main/java/cn/classfun/droidvm/lib/archive/ZeroInputStream.java @@ -0,0 +1,31 @@ +package cn.classfun.droidvm.lib.archive; + +import androidx.annotation.NonNull; + +import java.io.InputStream; +import java.util.Arrays; + +/** A bounded stream of zero bytes, used to reproduce trailing alignment padding. */ +public final class ZeroInputStream extends InputStream { + private long remaining; + + public ZeroInputStream(long count) { + this.remaining = Math.max(0, count); + } + + @Override + public int read() { + if (remaining <= 0) return -1; + remaining--; + return 0; + } + + @Override + public int read(@NonNull byte[] b, int off, int len) { + if (remaining <= 0) return -1; + int n = (int) Math.min(len, remaining); + Arrays.fill(b, off, off + n, (byte) 0); + remaining -= n; + return n; + } +} diff --git a/app/src/main/java/cn/classfun/droidvm/lib/pkg/PackageConstants.java b/app/src/main/java/cn/classfun/droidvm/lib/pkg/PackageConstants.java index bca0dad..c45a8af 100644 --- a/app/src/main/java/cn/classfun/droidvm/lib/pkg/PackageConstants.java +++ b/app/src/main/java/cn/classfun/droidvm/lib/pkg/PackageConstants.java @@ -12,6 +12,17 @@ public final class PackageConstants { public static final String MANIFEST_NAME = "manifest.json"; public static final Compression DEFAULT_COMPRESSION = Compression.ZSTD; + // Multi-volume split. The picked file stays as a small metadata + // master (header + manifest + volume index); the compressed data stream is + // sliced across .001, .002, ... sub-volumes. volume_size <= 0 + // keeps the legacy single self-contained file (volumeCount == 0). + public static final long DEFAULT_VOLUME_SIZE = 500L * 1024 * 1024; + public static final long MIN_VOLUME_SIZE = 16L * 1024 * 1024; + public static final String VOLUME_MAGIC = "VMPV"; + public static final int VOLUME_VERSION = 1; + public static final int VOLUME_MAX_COUNT = 999; + public static final int VOLUME_SUFFIX_DIGITS = 3; + private PackageConstants() { } } diff --git a/app/src/main/java/cn/classfun/droidvm/lib/pkg/PackageHeader.java b/app/src/main/java/cn/classfun/droidvm/lib/pkg/PackageHeader.java index 08d934b..64956cf 100644 --- a/app/src/main/java/cn/classfun/droidvm/lib/pkg/PackageHeader.java +++ b/app/src/main/java/cn/classfun/droidvm/lib/pkg/PackageHeader.java @@ -18,6 +18,9 @@ public final class PackageHeader { public int appVersionCode = 0; public int manifestSize = 0; public int compression = 0; + // 0 = single self-contained file; > 0 = split package whose data lives in + // that many .NNN sub-volumes (this file is the metadata master). + public int volumeCount = 0; public long dataSize = 0; public PackageHeader() { @@ -32,6 +35,7 @@ public void parseFromData(@NonNull byte[] hdr) throws IOException { appVersionCode = getUInt16LE(hdr, 8); manifestSize = getUInt16LE(hdr, 10); compression = getUInt16LE(hdr, 12); + volumeCount = getUInt16LE(hdr, 14); dataSize = getInt64LE(hdr, 16); } diff --git a/app/src/main/java/cn/classfun/droidvm/lib/pkg/PackageInput.java b/app/src/main/java/cn/classfun/droidvm/lib/pkg/PackageInput.java index 7a35024..397c2b1 100644 --- a/app/src/main/java/cn/classfun/droidvm/lib/pkg/PackageInput.java +++ b/app/src/main/java/cn/classfun/droidvm/lib/pkg/PackageInput.java @@ -55,6 +55,39 @@ public void close() throws IOException { @NonNull public static PackageInput open(@NonNull InputStream raw) throws Exception { + return open(raw, -1); + } + + /** + * Reads just the header and manifest for preview, without setting up the + * data/decompression stream. Safe on a metadata master (whose bytes after + * the manifest are the volume index, not compressed data). + */ + @NonNull + public static PackageManifest peekManifest(@NonNull InputStream raw) throws Exception { + var pkgHeader = PackageHeader.fromStream(raw); + var manifest = PackageManifest.fromStream(raw, pkgHeader); + if (pkgHeader.manifestVersion != manifest.manifestVersion) + throw new IOException("vmpkg manifest_version mismatch"); + if (pkgHeader.appVersionCode != manifest.appVersionCode) + throw new IOException("vmpkg app_version_code mismatch"); + if (pkgHeader.compression != manifest.compression.type) + throw new IOException("vmpkg compression mismatch"); + return manifest; + } + + /** + * Opens a package, optionally overriding the data-region size. Multi-volume + * packages write a placeholder {@code dataSize} of 0 in the inline header + * (it is only known after streaming) and carry the real value in the volume + * trailer; pass it here as {@code dataSizeOverride}. Use -1 for the legacy + * single-file layout that patches its header in place. + */ + @NonNull + public static PackageInput open( + @NonNull InputStream raw, + long dataSizeOverride + ) throws Exception { var pkgHeader = PackageHeader.fromStream(raw); var manifest = PackageManifest.fromStream(raw, pkgHeader); if (pkgHeader.manifestVersion != manifest.manifestVersion) @@ -63,6 +96,7 @@ public static PackageInput open(@NonNull InputStream raw) throws Exception { throw new IOException("vmpkg app_version_code mismatch"); if (pkgHeader.compression != manifest.compression.type) throw new IOException("vmpkg compression mismatch"); + if (dataSizeOverride >= 0) pkgHeader.dataSize = dataSizeOverride; long pos = alignUp(PackageConstants.HEADER_SIZE) + pkgHeader.manifestSize; readZeroPadding(raw, alignUpStrict(pos) - pos); var stream = new LimitedInputStream(raw, pkgHeader.dataSize); diff --git a/app/src/main/java/cn/classfun/droidvm/lib/pkg/VolumeIndex.java b/app/src/main/java/cn/classfun/droidvm/lib/pkg/VolumeIndex.java new file mode 100644 index 0000000..3a81fc0 --- /dev/null +++ b/app/src/main/java/cn/classfun/droidvm/lib/pkg/VolumeIndex.java @@ -0,0 +1,90 @@ +package cn.classfun.droidvm.lib.pkg; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static cn.classfun.droidvm.lib.utils.StringUtils.fmt; + +import androidx.annotation.NonNull; + +import org.json.JSONArray; +import org.json.JSONObject; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Volume list embedded in the metadata master ({@code .vmpkg}) right + * after the manifest. It records the total data size and, for each sub-volume, + * its logical size and CRC32 so a reader can verify the set is complete and + * intact before unpacking. Plain JSON -- its length is master-file-size minus + * the data-region offset, so no framing/magic footer is needed. + */ +public final class VolumeIndex { + public static final class Entry { + public final int index; + public final long size; + public final long crc32; + + public Entry(int index, long size, long crc32) { + this.index = index; + this.size = size; + this.crc32 = crc32; + } + } + + public int version = PackageConstants.VOLUME_VERSION; + public long dataSize = 0; + public final List volumes = new ArrayList<>(); + + @NonNull + public byte[] toBytes() throws IOException { + var arr = new JSONArray(); + try { + for (var v : volumes) { + var o = new JSONObject(); + o.put("index", v.index); + o.put("size", v.size); + o.put("crc32", v.crc32); + arr.put(o); + } + var root = new JSONObject(); + root.put("magic", PackageConstants.VOLUME_MAGIC); + root.put("version", version); + root.put("data_size", dataSize); + root.put("volumes", arr); + return root.toString().getBytes(UTF_8); + } catch (Exception e) { + throw new IOException("failed to build volume index", e); + } + } + + @NonNull + public static VolumeIndex parse(@NonNull byte[] json) throws IOException { + try { + var root = new JSONObject(new String(json, UTF_8)); + if (!PackageConstants.VOLUME_MAGIC.equals(root.optString("magic"))) + throw new IOException("missing volume index magic"); + var idx = new VolumeIndex(); + idx.version = root.optInt("version", 0); + idx.dataSize = root.optLong("data_size", 0); + var arr = root.optJSONArray("volumes"); + if (arr == null || arr.length() == 0) + throw new IOException("volume index has no volumes"); + for (int i = 0; i < arr.length(); i++) { + var o = arr.getJSONObject(i); + idx.volumes.add(new Entry( + o.optInt("index", i + 1), + o.optLong("size", 0), + o.optLong("crc32", 0) + )); + } + if (idx.version != PackageConstants.VOLUME_VERSION) + throw new IOException(fmt("unsupported volume version: %d", idx.version)); + return idx; + } catch (IOException e) { + throw e; + } catch (Exception e) { + throw new IOException("failed to parse volume index", e); + } + } +} diff --git a/app/src/main/java/cn/classfun/droidvm/lib/pkg/VolumeSet.java b/app/src/main/java/cn/classfun/droidvm/lib/pkg/VolumeSet.java new file mode 100644 index 0000000..4f90600 --- /dev/null +++ b/app/src/main/java/cn/classfun/droidvm/lib/pkg/VolumeSet.java @@ -0,0 +1,212 @@ +package cn.classfun.droidvm.lib.pkg; + +import static cn.classfun.droidvm.lib.utils.BinaryUtils.alignUp; +import static cn.classfun.droidvm.lib.utils.BinaryUtils.alignUpStrict; +import static cn.classfun.droidvm.lib.utils.StringUtils.fmt; + +import androidx.annotation.NonNull; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Vector; +import java.util.zip.CRC32; + +import cn.classfun.droidvm.lib.archive.ZeroInputStream; + +/** + * Reader-side view of a multi-volume vmpkg. Given the metadata master + * ({@code .vmpkg}), it parses the embedded {@link VolumeIndex}, locates + * and validates the {@code .001 ..} sub-volumes, then rebuilds the exact + * single-file byte stream -- {@code [master head][sub-volume data][tail padding]} + * -- so the unchanged {@link PackageInput} can consume it. Sub-volume CRC32s are + * verified as their boundaries are crossed. + */ +public final class VolumeSet { + private final byte[] masterHead; + private final long dataStart; + private final VolumeIndex index; + private final List subVolumes; + + private VolumeSet( + @NonNull byte[] masterHead, + long dataStart, + @NonNull VolumeIndex index, + @NonNull List subVolumes + ) { + this.masterHead = masterHead; + this.dataStart = dataStart; + this.index = index; + this.subVolumes = subVolumes; + } + + /** True when {@code path} looks like a sub-volume member ({@code ....NNN}). */ + public static boolean isSubVolume(@NonNull String path) { + return path.matches(fmt(".*\\.\\d{%d}", PackageConstants.VOLUME_SUFFIX_DIGITS)); + } + + /** Maps any picked path to its metadata master: strips a {@code .NNN} + * sub-volume suffix, or returns the path unchanged for the master itself. */ + @NonNull + public static String masterOf(@NonNull String path) { + if (!isSubVolume(path)) return path; + return path.substring(0, path.length() - PackageConstants.VOLUME_SUFFIX_DIGITS - 1); + } + + public long dataSize() { + return index.dataSize; + } + + public int count() { + return subVolumes.size(); + } + + /** + * Parses and validates the whole set from the master path. Throws with a + * clear message when a sub-volume is missing or a size does not match. + */ + @NonNull + public static VolumeSet discover(@NonNull String masterPath) throws IOException { + var master = new File(masterPath); + var all = readAllBytes(master); + if (all.length < PackageConstants.HEADER_SIZE) + throw new IOException("master too small"); + PackageHeader hdr; + try { + hdr = PackageHeader.fromBytes(Arrays.copyOf(all, PackageConstants.HEADER_SIZE)); + } catch (IOException e) { + throw e; + } catch (Exception e) { + throw new IOException("invalid master header", e); + } + if (hdr.volumeCount <= 0) + throw new IOException("not a multi-volume package"); + long dataStart = alignUpStrict(alignUp(PackageConstants.HEADER_SIZE) + hdr.manifestSize); + if (all.length < dataStart) + throw new IOException("truncated master"); + var indexJson = Arrays.copyOfRange(all, (int) dataStart, all.length); + var index = VolumeIndex.parse(indexJson); + if (index.volumes.size() != hdr.volumeCount) + throw new IOException(fmt( + "volume count mismatch: header %d, index %d", + hdr.volumeCount, index.volumes.size() + )); + var subs = new ArrayList(); + for (int i = 0; i < index.volumes.size(); i++) { + var entry = index.volumes.get(i); + var f = new File(VolumeSplitOutputStream.volumeName(masterPath, i + 1)); + if (!f.isFile()) + throw new IOException(fmt("missing sub-volume %d (%s)", i + 1, f.getName())); + if (f.length() != entry.size) + throw new IOException(fmt( + "sub-volume %d size mismatch: on-disk %d, expected %d", + i + 1, f.length(), entry.size + )); + subs.add(f); + } + var head = Arrays.copyOf(all, (int) dataStart); + return new VolumeSet(head, dataStart, index, subs); + } + + /** + * Rebuilds the single-file byte stream: master head (header + manifest + + * padding) ++ concatenated sub-volume data (CRC-checked) ++ trailing + * alignment zeros. Feed to {@link PackageInput#open(InputStream, long)}. + */ + @NonNull + public InputStream openLogicalStream() { + long dataEnd = dataStart + index.dataSize; + long tailPad = alignUp(dataEnd) - dataEnd; + var streams = new Vector(); + streams.add(new ByteArrayInputStream(masterHead)); + streams.add(new SubVolumeConcatStream()); + streams.add(new ZeroInputStream(tailPad)); + return new SequenceInputStream(streams.elements()); + } + + @NonNull + private static byte[] readAllBytes(@NonNull File f) throws IOException { + long len = f.length(); + if (len <= 0 || len > (16L << 20)) + throw new IOException(fmt("unexpected master size: %d", len)); + var buf = new byte[(int) len]; + try (var in = new FileInputStream(f)) { + int off = 0; + while (off < buf.length) { + int n = in.read(buf, off, buf.length - off); + if (n < 0) throw new IOException("short read on master"); + off += n; + } + } + return buf; + } + + private final class SubVolumeConcatStream extends InputStream { + private int idx = -1; + private InputStream cur; + private CRC32 crc; + private long remaining; + + private boolean advance() throws IOException { + if (cur != null) { + verify(); + cur.close(); + cur = null; + } + idx++; + if (idx >= subVolumes.size()) return false; + cur = new BufferedInputStream( + new FileInputStream(subVolumes.get(idx)), PackageConstants.BUFFER + ); + crc = new CRC32(); + remaining = index.volumes.get(idx).size; + return true; + } + + private void verify() throws IOException { + if (remaining != 0) + throw new IOException(fmt("sub-volume %d truncated", idx + 1)); + if (crc.getValue() != index.volumes.get(idx).crc32) + throw new IOException(fmt("sub-volume %d checksum mismatch", idx + 1)); + } + + @Override + public int read() throws IOException { + var one = new byte[1]; + int n = read(one, 0, 1); + return n < 0 ? -1 : (one[0] & 0xff); + } + + @Override + public int read(@NonNull byte[] b, int off, int len) throws IOException { + while (true) { + if (cur == null || remaining == 0) { + if (!advance()) return -1; + continue; + } + int toRead = (int) Math.min(len, remaining); + int n = cur.read(b, off, toRead); + if (n < 0) + throw new IOException(fmt("sub-volume %d ended early", idx + 1)); + crc.update(b, off, n); + remaining -= n; + return n; + } + } + + @Override + public void close() throws IOException { + if (cur != null) { + cur.close(); + cur = null; + } + } + } +} diff --git a/app/src/main/java/cn/classfun/droidvm/lib/pkg/VolumeSplitOutputStream.java b/app/src/main/java/cn/classfun/droidvm/lib/pkg/VolumeSplitOutputStream.java new file mode 100644 index 0000000..77a216f --- /dev/null +++ b/app/src/main/java/cn/classfun/droidvm/lib/pkg/VolumeSplitOutputStream.java @@ -0,0 +1,134 @@ +package cn.classfun.droidvm.lib.pkg; + +import static cn.classfun.droidvm.lib.utils.StringUtils.fmt; + +import androidx.annotation.NonNull; + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.zip.CRC32; + +/** + * Slices the compressed data stream across fixed-size sub-volumes + * {@code .001, .002, ...}. Pure data: no header, manifest or magic + * -- those live in the metadata master ({@link VolumeIndex}). Tracks each + * sub-volume's size and CRC32. {@link #close()} is a no-op so an upstream + * compression stream can be closed without ending the volume set; finalize + * explicitly with {@link #finish()} and read {@link #entries()}/{@link #dataSize()}. + */ +public final class VolumeSplitOutputStream extends OutputStream { + private final String basePath; + private final long volumeSize; + private final List entries = new ArrayList<>(); + private final List files = new ArrayList<>(); + private OutputStream current; + private CRC32 currentCrc; + private long currentBytes; + private int index; + private long position; + private boolean finished; + + public VolumeSplitOutputStream(@NonNull String basePath, long volumeSize) { + this.basePath = basePath; + this.volumeSize = volumeSize; + } + + @NonNull + public static String volumeName(@NonNull String basePath, int index) { + return fmt("%s.%0" + PackageConstants.VOLUME_SUFFIX_DIGITS + "d", basePath, index); // concat-ok + } + + @Override + public void write(int b) throws IOException { + write(new byte[]{(byte) b}, 0, 1); + } + + @Override + public void write(@NonNull byte[] b, int off, int len) throws IOException { + if (finished) throw new IOException("volume stream already finished"); + while (len > 0) { + if (current == null || currentBytes >= volumeSize) rollVolume(); + int chunk = (int) Math.min(len, volumeSize - currentBytes); + current.write(b, off, chunk); + currentCrc.update(b, off, chunk); + currentBytes += chunk; + position += chunk; + off += chunk; + len -= chunk; + } + } + + private void rollVolume() throws IOException { + if (current != null) closeCurrent(); + index++; + if (index > PackageConstants.VOLUME_MAX_COUNT) + throw new IOException(fmt( + "too many volumes (> %d); use a larger volume size", + PackageConstants.VOLUME_MAX_COUNT + )); + var file = new File(volumeName(basePath, index)); + files.add(file); + current = new BufferedOutputStream(new FileOutputStream(file), PackageConstants.BUFFER); + currentCrc = new CRC32(); + currentBytes = 0; + } + + private void closeCurrent() throws IOException { + entries.add(new VolumeIndex.Entry(index, currentBytes, currentCrc.getValue())); + current.flush(); + current.close(); + current = null; + } + + /** Closes the final sub-volume and records its entry. */ + public void finish() throws IOException { + if (finished) return; + if (current != null) closeCurrent(); + finished = true; + } + + /** Total bytes written across all sub-volumes (the compressed data size). */ + public long dataSize() { + return position; + } + + public int currentIndex() { + return index; + } + + public int volumeCount() { + return entries.size(); + } + + @NonNull + public List entries() { + return entries; + } + + @NonNull + public List files() { + return files; + } + + /** No-op: mirrors RandomAccessFileOutputStream so upstream close() cannot + * end the volume set prematurely. Use {@link #finish()} to finalize. */ + @Override + public void close() { + } + + /** Closes any open sub-volume without recording it, for error cleanup. */ + public void abort() { + if (current != null) { + try { + current.close(); + } catch (IOException ignored) { + } + current = null; + } + } +} diff --git a/app/src/main/java/cn/classfun/droidvm/lib/utils/BinaryUtils.java b/app/src/main/java/cn/classfun/droidvm/lib/utils/BinaryUtils.java index 38a7c7d..6f08986 100644 --- a/app/src/main/java/cn/classfun/droidvm/lib/utils/BinaryUtils.java +++ b/app/src/main/java/cn/classfun/droidvm/lib/utils/BinaryUtils.java @@ -45,6 +45,15 @@ public static void writeZero(@NonNull RandomAccessFile raf, long count) throws I } } + public static void writeZero(@NonNull java.io.OutputStream out, long count) throws IOException { + long rem = count; + while (rem > 0) { + int n = (int) Math.min(ZERO.length, rem); + out.write(ZERO, 0, n); + rem -= n; + } + } + @NonNull public static String decodeUtf8(@NonNull byte[] bytes) throws IOException { var decoder = UTF_8.newDecoder() diff --git a/app/src/main/java/cn/classfun/droidvm/ui/vm/pkg/exports/VMPkgExportActivity.java b/app/src/main/java/cn/classfun/droidvm/ui/vm/pkg/exports/VMPkgExportActivity.java index 0a88f75..7465015 100644 --- a/app/src/main/java/cn/classfun/droidvm/ui/vm/pkg/exports/VMPkgExportActivity.java +++ b/app/src/main/java/cn/classfun/droidvm/ui/vm/pkg/exports/VMPkgExportActivity.java @@ -54,6 +54,7 @@ import cn.classfun.droidvm.lib.utils.ShareUtils; import cn.classfun.droidvm.ui.widgets.container.CollapsibleContainer; import cn.classfun.droidvm.ui.widgets.row.ChooseRowWidget; +import cn.classfun.droidvm.ui.widgets.row.TextInputRowWidget; public final class VMPkgExportActivity extends AppCompatActivity implements DaemonConnection.EventListener { @@ -70,6 +71,7 @@ public final class VMPkgExportActivity extends AppCompatActivity private TextView tvDiskSummary; private ChooseRowWidget chooseCompression; private TextView tvCompressionDesc; + private TextInputRowWidget inputVolumeSize; private LinearProgressIndicator pbRun; private TextView tvStatus; private TextView tvFileName; @@ -78,9 +80,11 @@ public final class VMPkgExportActivity extends AppCompatActivity private ExtendedFloatingActionButton fabShare; private VMConfig config; private Compression compression = PackageConstants.DEFAULT_COMPRESSION; + private long volumeSize = 0; private VMPkgExportDiskAdapter adapter; private String pendingTaskId = null; private String path = null; + private int exportVolumeIndex = 0; private boolean exporting = false; private Phase phase = Phase.IDLE; private final ActivityResultLauncher createDocLauncher = @@ -108,6 +112,7 @@ protected void onCreate(@Nullable Bundle savedInstanceState) { tvDiskSummary = findViewById(R.id.tv_disk_summary); chooseCompression = findViewById(R.id.choose_compression); tvCompressionDesc = findViewById(R.id.tv_compression_desc); + inputVolumeSize = findViewById(R.id.input_volume_size); pbRun = findViewById(R.id.pb_run); tvStatus = findViewById(R.id.tv_status); tvFileName = findViewById(R.id.tv_filename); @@ -144,6 +149,9 @@ private void initialize() { containerDisks.setAdapter(adapter); bindDisks(); bindCompression(); + // Volume split defaults off: leave the input empty so export produces a + // single self-contained file unless the user enters a size. + inputVolumeSize.setText(""); var handler = new VMPkgExportOnBackPressedCallback(); toolbar.setNavigationOnClickListener(v -> handler.handleOnBackPressed()); getOnBackPressedDispatcher().addCallback(this, handler); @@ -303,6 +311,23 @@ private boolean validateSelectedDisks() { private void onDocPicked(@Nullable Uri uri) { if (uri == null || exporting) return; + // Clear any stale volume index from a previous export so a following + // single-file export does not inherit its "volume N" status. + exportVolumeIndex = 0; + // Read the (empty = single file) volume size on the UI thread before + // handing off to the worker pool. + var vsText = inputVolumeSize.getText().trim(); + if (vsText.isEmpty()) { + volumeSize = 0; + } else if (!inputVolumeSize.isInputValid()) { + showToast(R.string.vmpkg_export_failed, getString( + R.string.vmpkg_export_volume_size_invalid, + PackageConstants.MIN_VOLUME_SIZE / (1024 * 1024) + )); + return; + } else { + volumeSize = inputVolumeSize.getValue(); + } runOnPool(() -> startExport(uri)); } @@ -344,6 +369,7 @@ private void startExport(@NonNull Uri uri) { .put("disks", diskIndices) .put("dest_path", destPath) .put("compression", compression) + .put("volume_size", volumeSize) .onResponse(onExport) .onUnsuccessful(f) .onError(err) @@ -384,6 +410,7 @@ public void onDaemonEvent(JSONObject msg) { var bytesTotal = data.optLong("bytes_total"); var file = data.optString("file"); var message = data.optString("message"); + exportVolumeIndex = data.optInt("volume_index", exportVolumeIndex); mainHandler.post(() -> onProgress( phase, done, total, bytesDone, bytesTotal, file, message )); @@ -424,7 +451,10 @@ private void onProgress( applyProgress(done, total, bytesDone, bytesTotal); break; case PACK: - tvStatus.setText(R.string.vmpkg_export_pack); + if (exportVolumeIndex > 0) tvStatus.setText(getString( + R.string.vmpkg_export_pack_volume, exportVolumeIndex + )); + else tvStatus.setText(R.string.vmpkg_export_pack); applyProgress(done, total, bytesDone, bytesTotal); applyProgressDetail(file, bytesDone, bytesTotal); break; @@ -471,6 +501,7 @@ private void setExporting(boolean exporting) { pbRun.setVisibility(exporting ? VISIBLE : GONE); pbRun.setIndeterminate(exporting); chooseCompression.setEnabled(!exporting); + inputVolumeSize.setEnabled(!exporting); containerDisks.setEnabled(!exporting); adapter.setEnabled(!exporting); } diff --git a/app/src/main/java/cn/classfun/droidvm/ui/vm/pkg/imports/VMPkgImportActivity.java b/app/src/main/java/cn/classfun/droidvm/ui/vm/pkg/imports/VMPkgImportActivity.java index e3283c0..9689b55 100644 --- a/app/src/main/java/cn/classfun/droidvm/ui/vm/pkg/imports/VMPkgImportActivity.java +++ b/app/src/main/java/cn/classfun/droidvm/ui/vm/pkg/imports/VMPkgImportActivity.java @@ -41,6 +41,7 @@ import org.json.JSONObject; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.text.DateFormat; import java.util.Date; @@ -51,6 +52,7 @@ import cn.classfun.droidvm.lib.pkg.PackageInput; import cn.classfun.droidvm.lib.pkg.PackageManifest; import cn.classfun.droidvm.lib.pkg.Phase; +import cn.classfun.droidvm.lib.pkg.VolumeSet; import cn.classfun.droidvm.lib.store.disk.DiskConfig; import cn.classfun.droidvm.lib.store.disk.DiskStore; import cn.classfun.droidvm.lib.store.network.NetworkConfig; @@ -86,10 +88,14 @@ public final class VMPkgImportActivity extends AppCompatActivity private TextView tvProgressDetail; private View groupSummary; private Uri pickedUri; + // Real filesystem path of the metadata master, resolved at pick time when a + // sub-volume (.NNN) is chosen so preview/import target the master directly. + private String masterRealPath; private PackageManifest preview; private VMPkgImportDiskAdapter diskAdapter; private VMPkgImportNetworkAdapter networkAdapter; private String pendingTaskId = null; + private int importVolumeTotal = 0; private boolean importing = false; private NetworkImportMode networkMode = NetworkImportMode.AUTO; private final ActivityResultLauncher openDocLauncher = @@ -188,16 +194,15 @@ protected void onStop() { private void onDocPicked(@Nullable Uri uri) { if (uri == null || importing) return; pickedUri = uri; + masterRealPath = null; + importVolumeTotal = 0; tvError.setVisibility(GONE); groupSummary.setVisibility(GONE); btnImport.setEnabled(false); tvStatus.setText(R.string.vmpkg_import_reading); runOnPool(() -> { - try (var is = getContentResolver().openInputStream(uri)) { - if (is == null) throw new RuntimeException("Cannot open file"); - try (var input = PackageInput.open(is)) { - preview = input.manifest; - } + try { + preview = readPreviewManifest(uri); mainHandler.post(this::showPreview); } catch (Exception e) { mainHandler.post(() -> showError(getString(R.string.vmpkg_import_invalid, e.getMessage()))); @@ -205,6 +210,26 @@ private void onDocPicked(@Nullable Uri uri) { }); } + @NonNull + private PackageManifest readPreviewManifest(@NonNull Uri uri) throws Exception { + // A picked sub-volume (.NNN) has no header/manifest; redirect to its + // metadata master in the same folder (needs MANAGE_EXTERNAL_STORAGE). + var real = resolveUriPath(this, uri); + if (real != null && !real.isEmpty() && VolumeSet.isSubVolume(real)) { + masterRealPath = VolumeSet.masterOf(real); + try (var is = new FileInputStream(masterRealPath)) { + return PackageInput.peekManifest(is); + } + } + // Master or single file: SAF stream is enough. Remember the resolved + // master path (may be null) so import can reuse it. + masterRealPath = real == null || real.isEmpty() ? null : VolumeSet.masterOf(real); + try (var is = getContentResolver().openInputStream(uri)) { + if (is == null) throw new RuntimeException("Cannot open file"); + return PackageInput.peekManifest(is); + } + } + @SuppressLint("NotifyDataSetChanged") private void showPreview() { if (preview == null) return; @@ -252,7 +277,10 @@ private void showPreview() { private void doImport() { if (pickedUri == null || preview == null || importing) return; - var srcPath = resolveUriPath(this, pickedUri); + // Prefer the master path resolved at pick time; the daemon maps any + // .NNN sub-volume to its master anyway. + var srcPath = masterRealPath; + if (srcPath == null || srcPath.isEmpty()) srcPath = resolveUriPath(this, pickedUri); if (srcPath == null || srcPath.isEmpty()) { showError(getString( R.string.vmpkg_import_failed, @@ -296,6 +324,7 @@ public void onDaemonEvent(JSONObject msg) { var phase = optEnum(data, "phase", Phase.SCAN); var done = data.optInt("done", 0); var total = data.optInt("total", 0); + importVolumeTotal = data.optInt("volume_total", importVolumeTotal); var file = data.optString("file", ""); var bytesDone = data.optLong("bytes_done", -1); var bytesTotal = data.optLong("bytes_total", -1); @@ -332,7 +361,10 @@ private void onProgress( ) { switch (phase) { case PACK: - tvStatus.setText(R.string.vmpkg_import_running); + if (importVolumeTotal > 0) tvStatus.setText(getString( + R.string.vmpkg_import_running_volume, importVolumeTotal + )); + else tvStatus.setText(R.string.vmpkg_import_running); applyProgress(done, total, bytesDone, bytesTotal); applyProgressDetail(file, bytesDone, bytesTotal); break; diff --git a/app/src/main/res/layout/activity_vmpkg_export.xml b/app/src/main/res/layout/activity_vmpkg_export.xml index f7a850b..7d47d8e 100644 --- a/app/src/main/res/layout/activity_vmpkg_export.xml +++ b/app/src/main/res/layout/activity_vmpkg_export.xml @@ -153,6 +153,27 @@ android:textAppearance="?attr/textAppearanceBodySmall" android:textColor="?android:attr/textColorSecondary" /> + + + + 正在扫描磁盘\u2026 正在打包磁盘\u2026 %1$s / %2$s %3$d%% + 分卷大小 + 将包拆分为固定大小的分卷(.001、.002…)。留空则导出为单个文件。 + 分卷大小不能小于 %1$d MB + 正在打包磁盘(分卷 %1$d)… + 正在导入(共 %1$d 卷)… 包已创建 包已创建:%1$s 导出失败:%1$s diff --git a/app/src/main/res/values-zh-rTW/strings.xml b/app/src/main/res/values-zh-rTW/strings.xml index d6d5a09..785db36 100644 --- a/app/src/main/res/values-zh-rTW/strings.xml +++ b/app/src/main/res/values-zh-rTW/strings.xml @@ -1056,6 +1056,11 @@ 正在掃描磁碟\u2026 正在打包磁碟\u2026 %1$s / %2$s %3$d%% + 分卷大小 + 將封裝包拆分為固定大小的分卷(.001、.002…)。留空則匯出為單一檔案。 + 分卷大小不能小於 %1$d MB + 正在打包磁碟(分卷 %1$d)… + 正在匯入(共 %1$d 卷)… 包已建立 包已建立:%1$s 匯出失敗:%1$s diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 85f08a3..954a15f 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -1089,9 +1089,13 @@ Gzip is fast with low CPU usage but produces larger archives. XZ (LZMA2) is slowest and CPU-heavy but produces the smallest archives. Zstandard offers the best balance: fast compression with good ratios. + Volume size + Split the package into fixed-size volumes (.001, .002, \u2026). Leave empty for a single file. + Volume size must be at least %1$d MB Submitting to daemon\u2026 Scanning disks\u2026 Packaging disks\u2026 + Packaging disks (volume %1$d)\u2026 %1$s / %2$s %3$d%% Package created Package created: %1$s @@ -1110,6 +1114,7 @@ Import Reading package\u2026 Importing\u2026 + Importing (%1$d volumes)\u2026 Disks Networks No disks in this package.