Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import cn.classfun.droidvm.daemon.server.RequestException;
import cn.classfun.droidvm.daemon.server.RequestHandler;
import cn.classfun.droidvm.daemon.vm.pkg.VMExportTask;
import cn.classfun.droidvm.lib.pkg.PackageConstants;

@AutoService(RequestHandler.class)
public final class ExportHandler extends RequestHandler {
Expand All @@ -25,6 +26,12 @@ public void handle(@NonNull ClientRequest request) throws Exception {
throw new RequestException("missing vm_id");
if (destPath.isEmpty())
throw new RequestException("missing dest_path");
var volumeSize = params.optLong("volume_size", 0);
if (volumeSize > 0 && volumeSize < PackageConstants.MIN_VOLUME_SIZE)
throw new RequestException(String.format(
"volume_size too small: %d (min %d)",
volumeSize, PackageConstants.MIN_VOLUME_SIZE
));
var store = request.getContext().getExportTaskStore();
var server = request.getClient().getServer();
var task = new VMExportTask(server, params);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@
import cn.classfun.droidvm.lib.pkg.PackageConstants;
import cn.classfun.droidvm.lib.pkg.PackageManifest;
import cn.classfun.droidvm.lib.pkg.Phase;
import cn.classfun.droidvm.lib.pkg.VolumeIndex;
import cn.classfun.droidvm.lib.pkg.VolumeSplitOutputStream;
import cn.classfun.droidvm.lib.store.base.DataItem;

public final class VMExportTask {
Expand All @@ -47,6 +49,9 @@ public final class VMExportTask {
private final UUID sourceVMId;
private final int totalItems;
private final String destPath;
private final long volumeSize;
private VolumeSplitOutputStream vout = null;
private int finalVolumeCount = 0;

public VMExportTask(
@NonNull Server server,
Expand All @@ -56,6 +61,7 @@ public VMExportTask(
destPath = request.optString("dest_path");
if (!destPath.startsWith("/"))
throw new IllegalArgumentException("missing dest_path");
volumeSize = request.optLong("volume_size", 0);
var vmId = UUID.fromString(request.optString("vm_id"));
sourceVMId = vmId;
var vm = server.getContext().getVMs().findById(vmId);
Expand Down Expand Up @@ -91,6 +97,10 @@ public void startAsync() {
try {
startWrite();
data.set("done", totalItems);
if (finalVolumeCount > 0) {
data.set("volume_index", finalVolumeCount);
data.set("volume_total", finalVolumeCount);
}
emit(data, Phase.DONE);
} catch (Exception e) {
Log.w(TAG, fmt("Export task %s failed", taskId), e);
Expand All @@ -105,6 +115,11 @@ public void startAsync() {
}

private void startWrite() throws Exception {
if (volumeSize > 0) startWriteSplit();
else startWriteSingle();
}

private void startWriteSingle() throws Exception {
var outFile = new File(destPath);
var manifestBytes = manifest.toJson().toString(4).getBytes(UTF_8);
if (manifestBytes.length > 0xffff)
Expand Down Expand Up @@ -132,6 +147,75 @@ private void startWrite() throws Exception {
}
}

// Multi-volume path: (1) slice the compressed data stream into <dest>.001,
// .002, ... sub-volumes, then (2) write the metadata master <dest> holding
// header + manifest + volume index (no data). The picked SAF file is reused
// as the master, so it is not deleted.
private void startWriteSplit() throws Exception {
var manifestBytes = manifest.toJson().toString(4).getBytes(UTF_8);
if (manifestBytes.length > 0xffff)
throw new IOException(fmt("manifest too large: %d bytes", manifestBytes.length));
var subOut = new VolumeSplitOutputStream(destPath, volumeSize);
vout = subOut;
try {
try (
var compressed = wrapCompressionOutput(subOut, manifest.compression);
var tar = new TarWriter(compressed)
) {
tar.entry(PackageConstants.MANIFEST_NAME, manifestBytes);
pack(tar);
}
subOut.finish();
} catch (Exception e) {
subOut.abort();
deleteQuietly(subOut.files());
throw e;
} finally {
vout = null;
}
long dataSize = subOut.dataSize();
finalVolumeCount = subOut.volumeCount();
writeMaster(manifestBytes, dataSize, subOut);
// Remove any higher-numbered sub-volumes left by a previous, larger
// export to the same base path, so the on-disk set matches the index.
for (int i = finalVolumeCount + 1; i <= PackageConstants.VOLUME_MAX_COUNT; i++) {
var stale = new File(VolumeSplitOutputStream.volumeName(destPath, i));
if (!stale.exists()) break;
//noinspection ResultOfMethodCallIgnored
stale.delete();
}
}

private static void deleteQuietly(@NonNull java.util.List<File> files) {
for (var f : files) {
//noinspection ResultOfMethodCallIgnored
f.delete();
}
}

private void writeMaster(
@NonNull byte[] manifestBytes,
long dataSize,
@NonNull VolumeSplitOutputStream subOut
) throws Exception {
var index = new VolumeIndex();
index.dataSize = dataSize;
index.volumes.addAll(subOut.entries());
var indexBytes = index.toBytes();
try (var raf = new RandomAccessFile(new File(destPath), "rw")) {
raf.setLength(0);
raf.write(new byte[PackageConstants.HEADER_SIZE]);
writeZero(raf, alignUp(raf.getFilePointer()) - raf.getFilePointer());
raf.write(manifestBytes);
writeZero(raf, alignUpStrict(raf.getFilePointer()) - raf.getFilePointer());
raf.write(indexBytes);
raf.seek(0);
raf.write(buildHeader(
manifest, manifestBytes.length, dataSize, finalVolumeCount
));
}
}

private void pack(@NonNull TarWriter tar) throws Exception {
int done;
var data = DataItem.newObject();
Expand Down Expand Up @@ -209,6 +293,9 @@ private void emit(@NonNull DataItem item, @NonNull Phase phase) {
data.put("phase", phase.name().toLowerCase());
data.put("vm_id", sourceVMId.toString());
data.put("vm_name", manifest.vm.getName());
var out = vout;
if (out != null && !data.has("volume_index"))
data.put("volume_index", out.currentIndex());
var ev = new JSONObject();
ev.put("type", "event");
ev.put("data", data);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,16 @@ public static byte[] buildHeader(
@NonNull PackageManifest manifest,
int manifestSize,
long dataSize
) throws IOException {
return buildHeader(manifest, manifestSize, dataSize, 0);
}

@NonNull
public static byte[] buildHeader(
@NonNull PackageManifest manifest,
int manifestSize,
long dataSize,
int volumeCount
) throws IOException {
var hdr = new byte[PackageConstants.HEADER_SIZE];
var magic = PackageConstants.MAGIC.getBytes(UTF_8);
Expand All @@ -66,6 +76,7 @@ public static byte[] buildHeader(
putUInt16LE(hdr, 8, BuildConfig.VERSION_CODE);
putUInt16LE(hdr, 10, manifestSize);
putUInt16LE(hdr, 12, manifest.compression.type);
putUInt16LE(hdr, 14, volumeCount);
putInt64LE(hdr, 16, dataSize);
return hdr;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import static cn.classfun.droidvm.daemon.vm.pkg.VMImportUtils.remapDiskPaths;
import static cn.classfun.droidvm.daemon.vm.pkg.VMImportUtils.uniqueFile;
import static cn.classfun.droidvm.lib.pkg.PackageConstants.BUFFER;
import static cn.classfun.droidvm.lib.utils.BinaryUtils.readFully;
import static cn.classfun.droidvm.lib.utils.JsonUtils.listToJSONArray;
import static cn.classfun.droidvm.lib.utils.StringUtils.basename;
import static cn.classfun.droidvm.lib.utils.StringUtils.fmt;
Expand Down Expand Up @@ -31,9 +32,12 @@
import cn.classfun.droidvm.lib.archive.TarReader;
import cn.classfun.droidvm.lib.pkg.BootFile;
import cn.classfun.droidvm.lib.pkg.DiskEntry;
import cn.classfun.droidvm.lib.pkg.PackageConstants;
import cn.classfun.droidvm.lib.pkg.PackageHeader;
import cn.classfun.droidvm.lib.pkg.PackageInput;
import cn.classfun.droidvm.lib.pkg.PackageManifest;
import cn.classfun.droidvm.lib.pkg.Phase;
import cn.classfun.droidvm.lib.pkg.VolumeSet;
import cn.classfun.droidvm.lib.store.base.DataItem;
import cn.classfun.droidvm.lib.store.network.NetworkConfig;
import cn.classfun.droidvm.lib.store.vm.VMConfig;
Expand All @@ -47,6 +51,7 @@ public final class VMImportTask {
private final File targetDir;
private final String networkMode;
private int totalItems;
private int volumeTotal = 0;
public VMConfig importedVM = null;
public PackageManifest importedManifest = null;
public final ArrayList<DiskEntry> placedDisks = new ArrayList<>();
Expand Down Expand Up @@ -125,22 +130,24 @@ private void onTarItem(
private void unpack() throws Exception {
if (!targetDir.exists() && !targetDir.mkdirs())
throw new IOException(fmt("Cannot create target dir: %s", targetDir));
try (
var in = new FileInputStream(srcPath);
var pkg = PackageInput.open(in)
) {
importedManifest = pkg.manifest;
totalItems = importedManifest.disks.size() + importedManifest.boots.size();
var data = DataItem.newObject();
data.set("done", 0);
data.set("total", totalItems);
emit(data, Phase.PACK);
var tar = new TarReader(pkg.data);
tar.forEach(this::onTarItem);
var buf = new byte[BUFFER];
//noinspection StatementWithEmptyBody
while (pkg.data.read(buf) >= 0);
pkg.validateDataConsumed();
// Any picked path maps to its metadata master (strip a .NNN suffix).
var masterPath = VolumeSet.masterOf(srcPath);
if (readVolumeCount(masterPath) > 0) {
var set = VolumeSet.discover(masterPath);
volumeTotal = set.count();
try (
var in = set.openLogicalStream();
var pkg = PackageInput.open(in, set.dataSize())
) {
extract(pkg);
}
} else {
try (
var in = new FileInputStream(masterPath);
var pkg = PackageInput.open(in)
) {
extract(pkg);
}
}
var vm = new VMConfig(importedManifest.vm.toJson());
vm.setId(UUID.randomUUID());
Expand All @@ -149,6 +156,29 @@ private void unpack() throws Exception {
importedVM = vm;
}

private int readVolumeCount(@NonNull String masterPath) throws Exception {
try (var in = new FileInputStream(masterPath)) {
var hdr = new byte[PackageConstants.HEADER_SIZE];
readFully(in, hdr);
return PackageHeader.fromBytes(hdr).volumeCount;
}
}

private void extract(@NonNull PackageInput pkg) throws Exception {
importedManifest = pkg.manifest;
totalItems = importedManifest.disks.size() + importedManifest.boots.size();
var data = DataItem.newObject();
data.set("done", 0);
data.set("total", totalItems);
emit(data, Phase.PACK);
var tar = new TarReader(pkg.data);
tar.forEach(this::onTarItem);
var buf = new byte[BUFFER];
//noinspection StatementWithEmptyBody
while (pkg.data.read(buf) >= 0);
pkg.validateDataConsumed();
}

@NonNull
private JSONArray importNetworks(
@NonNull VMConfig vm,
Expand Down Expand Up @@ -281,6 +311,7 @@ private void emit(@NonNull DataItem item, @NonNull Phase phase) {
data.put("event", "vm_import_status");
data.put("task_id", taskId.toString());
data.put("phase", phase.name().toLowerCase());
if (volumeTotal > 0) data.put("volume_total", volumeTotal);
var ev = new JSONObject();
ev.put("type", "event");
ev.put("data", data);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package cn.classfun.droidvm.lib.archive;

import androidx.annotation.NonNull;

import java.io.InputStream;
import java.util.Arrays;

/** A bounded stream of zero bytes, used to reproduce trailing alignment padding. */
public final class ZeroInputStream extends InputStream {
private long remaining;

public ZeroInputStream(long count) {
this.remaining = Math.max(0, count);
}

@Override
public int read() {
if (remaining <= 0) return -1;
remaining--;
return 0;
}

@Override
public int read(@NonNull byte[] b, int off, int len) {
if (remaining <= 0) return -1;
int n = (int) Math.min(len, remaining);
Arrays.fill(b, off, off + n, (byte) 0);
remaining -= n;
return n;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,17 @@ public final class PackageConstants {
public static final String MANIFEST_NAME = "manifest.json";
public static final Compression DEFAULT_COMPRESSION = Compression.ZSTD;

// Multi-volume split. The picked file <base> stays as a small metadata
// master (header + manifest + volume index); the compressed data stream is
// sliced across <base>.001, <base>.002, ... sub-volumes. volume_size <= 0
// keeps the legacy single self-contained file (volumeCount == 0).
public static final long DEFAULT_VOLUME_SIZE = 500L * 1024 * 1024;
public static final long MIN_VOLUME_SIZE = 16L * 1024 * 1024;
public static final String VOLUME_MAGIC = "VMPV";
public static final int VOLUME_VERSION = 1;
public static final int VOLUME_MAX_COUNT = 999;
public static final int VOLUME_SUFFIX_DIGITS = 3;

private PackageConstants() {
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@ public final class PackageHeader {
public int appVersionCode = 0;
public int manifestSize = 0;
public int compression = 0;
// 0 = single self-contained file; > 0 = split package whose data lives in
// that many <base>.NNN sub-volumes (this file is the metadata master).
public int volumeCount = 0;
public long dataSize = 0;

public PackageHeader() {
Expand All @@ -32,6 +35,7 @@ public void parseFromData(@NonNull byte[] hdr) throws IOException {
appVersionCode = getUInt16LE(hdr, 8);
manifestSize = getUInt16LE(hdr, 10);
compression = getUInt16LE(hdr, 12);
volumeCount = getUInt16LE(hdr, 14);
dataSize = getInt64LE(hdr, 16);
}

Expand Down
Loading
Loading