diff --git a/switch/include/io.hpp b/switch/include/io.hpp index fce0fc01..0a020113 100644 --- a/switch/include/io.hpp +++ b/switch/include/io.hpp @@ -42,13 +42,20 @@ #define BUFFER_SIZE 0x80000 +// safety margin left free in the save journal when committing partway through a file +#define JOURNAL_COMMIT_MARGIN 0x100000 + +// extra headroom added on top of the backup size when extending the save data partition +#define SAVE_EXTEND_MARGIN 0x500000 + namespace io { std::tuple backup(size_t index, AccountUid uid, size_t cellIndex); std::tuple restore(size_t index, AccountUid uid, size_t cellIndex, const std::string& nameFromCell); size_t countFiles(const std::string& path); - Result copyDirectory(const std::string& srcPath, const std::string& dstPath); - void copyFile(const std::string& srcPath, const std::string& dstPath); + u64 getDirectorySize(const std::string& path); + Result copyDirectory(const std::string& srcPath, const std::string& dstPath, u64 commitWriteLimit = 0); + void copyFile(const std::string& srcPath, const std::string& dstPath, u64 commitWriteLimit = 0); Result createDirectory(const std::string& path); Result deleteFolderRecursively(const std::string& path); bool directoryExists(const std::string& path); diff --git a/switch/include/title.hpp b/switch/include/title.hpp index 59d2465f..b8d26826 100644 --- a/switch/include/title.hpp +++ b/switch/include/title.hpp @@ -60,6 +60,10 @@ class Title { void refreshDirectories(void); u64 saveId(); void saveId(u64 id); + u64 journalSize(); + void journalSize(u64 size); + u64 journalSizeMax(); + void journalSizeMax(u64 size); std::vector saves(void); u8 saveDataType(void); u8 saveDataSpaceId(void); @@ -69,6 +73,8 @@ class Title { private: u64 mId; u64 mSaveId; + u64 mJournalSize; + u64 mJournalSizeMax; AccountUid mUserId; std::string mUserName; std::string mName; diff --git a/switch/source/io.cpp b/switch/source/io.cpp index 7e138de3..2f7b986e 100644 --- a/switch/source/io.cpp +++ b/switch/source/io.cpp @@ -50,7 +50,28 @@ size_t io::countFiles(const std::string& path) return count; } -void io::copyFile(const std::string& srcPath, const std::string& dstPath) +u64 io::getDirectorySize(const std::string& path) +{ + u64 size = 0; + Directory items(path); + if (!items.good()) { + return 0; + } + for (size_t i = 0, sz = items.size(); i < sz; i++) { + if (items.folder(i)) { + size += io::getDirectorySize(path + items.entry(i) + "/"); + } + else { + struct stat st; + if (stat((path + items.entry(i)).c_str(), &st) == 0) { + size += st.st_size; + } + } + } + return size; +} + +void io::copyFile(const std::string& srcPath, const std::string& dstPath, u64 commitWriteLimit) { g_isTransferringFile = true; @@ -78,6 +99,12 @@ void io::copyFile(const std::string& srcPath, const std::string& dstPath) g_currentFileSize = sz; g_currentFileOffset = 0; + // when writing to the save device, the journal can only hold a limited amount of + // uncommitted data: commit partway through large files so a single file bigger than + // the journal doesn't overflow it at commit time. + const bool toSaveDevice = dstPath.rfind("save:/", 0) == 0; + u64 journalPending = 0; + while (offset < sz) { u32 count = fread((char*)buf, 1, BUFFER_SIZE, src); if (count == 0) { @@ -88,6 +115,24 @@ void io::copyFile(const std::string& srcPath, const std::string& dstPath) offset += count; g_currentFileOffset = offset; + if (toSaveDevice && commitWriteLimit > 0 && offset < sz) { + journalPending += count; + if (journalPending >= commitWriteLimit) { + journalPending = 0; + fflush(dst); + fclose(dst); + Result cres = fsdevCommitDevice("save"); + if (R_FAILED(cres)) { + Logging::error("Failed mid-file commit of {} at offset {}/{} with result 0x{:08X}.", dstPath, offset, sz, cres); + } + dst = fopen(dstPath.c_str(), "ab"); + if (dst == NULL) { + Logging::error("Failed to reopen destination file {} after commit with errno {}. Aborting copy.", dstPath, errno); + break; + } + } + } + // avoid freezing the UI // this will be made less horrible next time... g_screen->draw(); @@ -96,19 +141,20 @@ void io::copyFile(const std::string& srcPath, const std::string& dstPath) delete[] buf; fclose(src); - fclose(dst); + if (dst != NULL) { + fclose(dst); + } g_copyCount++; // commit each file to the save - if (dstPath.rfind("save:/", 0) == 0) { - Logging::error("Committing file {} to the save archive.", dstPath); + if (toSaveDevice) { fsdevCommitDevice("save"); } g_isTransferringFile = false; } -Result io::copyDirectory(const std::string& srcPath, const std::string& dstPath) +Result io::copyDirectory(const std::string& srcPath, const std::string& dstPath, u64 commitWriteLimit) { Result res = 0; bool quit = false; @@ -127,14 +173,14 @@ Result io::copyDirectory(const std::string& srcPath, const std::string& dstPath) if (R_SUCCEEDED(res)) { newsrc += "/"; newdst += "/"; - res = io::copyDirectory(newsrc, newdst); + res = io::copyDirectory(newsrc, newdst, commitWriteLimit); } else { quit = true; } } else { - io::copyFile(newsrc, newdst); + io::copyFile(newsrc, newdst, commitWriteLimit); } } @@ -177,6 +223,20 @@ Result io::deleteFolderRecursively(const std::string& path) return 0; } +static Result mountTitleSave(Title& title, FsFileSystem* fileSystem) +{ + switch (title.saveDataType()) { + case FsSaveDataType_Bcat: + return FileSystem::mountBcatSave(fileSystem, title.id()); + case FsSaveDataType_Device: + return FileSystem::mountDeviceSave(fileSystem, title.id()); + case FsSaveDataType_System: + return FileSystem::mountSystemSave(fileSystem, title.id(), title.saveDataSpaceId()); + default: + return FileSystem::mountSave(fileSystem, title.id(), title.userId()); + } +} + std::tuple io::backup(size_t index, AccountUid uid, size_t cellIndex) { const bool isNewFolder = cellIndex == 0; @@ -340,6 +400,41 @@ std::tuple io::restore(size_t index, AccountUid uid, std::string srcPath = title.fullPath(cellIndex) + "/"; std::string dstPath = "save:/"; + // if the backup is larger than the currently allocated save data, grow the partition + // before restoring (a save can outgrow its initial allocation as the game progresses). + u64 journalSizeMax = title.journalSizeMax(); + if (journalSizeMax > 0) { + u64 backupSize = io::getDirectorySize(srcPath); + s64 totalSpace = 0; + res = fsFsGetTotalSpace(fsdevGetDeviceFileSystem("save"), "/", &totalSpace); + if (R_SUCCEEDED(res) && backupSize > (u64)totalSpace) { + FileSystem::unmountDevice(); + + s64 newDataSize = (s64)(backupSize + SAVE_EXTEND_MARGIN); + res = fsExtendSaveDataFileSystem((FsSaveDataSpaceId)title.saveDataSpaceId(), title.saveId(), newDataSize, (s64)journalSizeMax); + if (R_FAILED(res)) { + Logging::error("Failed to extend save data to {} bytes with result 0x{:08X}. Title id: 0x{:016X}.", newDataSize, res, title.id()); + return std::make_tuple(false, res, "Failed to extend the save data\nto fit the backup."); + } + Logging::info("Extended save data of title 0x{:016X} to {} bytes to fit backup of {} bytes.", title.id(), newDataSize, backupSize); + + // remount the now larger save + res = mountTitleSave(title, &fileSystem); + if (R_SUCCEEDED(res)) { + int rc = FileSystem::mountDevice(fileSystem); + if (rc == -1) { + FileSystem::unmountDevice(); + Logging::error("Failed to remount filesystem after extend. Title id: 0x{:016X}.", title.id()); + return std::make_tuple(false, -2, "Failed to mount save."); + } + } + else { + Logging::error("Failed to remount filesystem after extend with result 0x{:08X}. Title id: 0x{:016X}.", res, title.id()); + return std::make_tuple(false, res, "Failed to mount save."); + } + } + } + res = io::deleteFolderRecursively(dstPath.c_str()); if (R_FAILED(res)) { FileSystem::unmountDevice(); @@ -347,10 +442,24 @@ std::tuple io::restore(size_t index, AccountUid uid, return std::make_tuple(false, res, "Failed to delete save."); } + // commit the wipe before writing so the deletions don't add to the journal pressure + // accumulated while restoring the backup. + res = fsdevCommitDevice("save"); + if (R_FAILED(res)) { + FileSystem::unmountDevice(); + Logging::error("Failed to commit save wipe with result 0x{:08X}.", res); + return std::make_tuple(false, res, "Failed to commit to save device."); + } + + // leave a safety margin under the journal size so the in-flight commits never overflow it. + // a journal size of 0 (unknown, e.g. system saves) disables mid-file commits. + u64 journalSize = title.journalSize(); + u64 commitWriteLimit = journalSize > JOURNAL_COMMIT_MARGIN ? journalSize - JOURNAL_COMMIT_MARGIN : journalSize; + g_copyCount = 0; g_copyTotal = io::countFiles(srcPath); g_transferMode = "Restore"; - res = io::copyDirectory(srcPath, dstPath); + res = io::copyDirectory(srcPath, dstPath, commitWriteLimit); if (R_FAILED(res)) { FileSystem::unmountDevice(); Logging::error("Failed to copy directory {} to {} with result 0x{:08X}. Skipping...", srcPath, dstPath, res); diff --git a/switch/source/title.cpp b/switch/source/title.cpp index fa20aa2b..a96f17cd 100644 --- a/switch/source/title.cpp +++ b/switch/source/title.cpp @@ -78,6 +78,8 @@ void Title::init(u8 saveDataType, u64 id, AccountUid userID, const std::string& mUserId = userID; mSaveDataType = saveDataType; mSaveDataSpaceId = spaceId; + mJournalSize = 0; + mJournalSizeMax = 0; if (mSaveDataType == FsSaveDataType_Bcat) { mUserName = "BCAT"; @@ -144,6 +146,26 @@ void Title::saveId(u64 saveId) mSaveId = saveId; } +u64 Title::journalSize(void) +{ + return mJournalSize; +} + +void Title::journalSize(u64 journalSize) +{ + mJournalSize = journalSize; +} + +u64 Title::journalSizeMax(void) +{ + return mJournalSizeMax; +} + +void Title::journalSizeMax(u64 journalSizeMax) +{ + mJournalSizeMax = journalSizeMax; +} + AccountUid Title::userId(void) { return mUserId; @@ -299,6 +321,9 @@ void loadTitles(void) Title title; title.init(info.save_data_type, tid, uid, std::string(nle->name), std::string(nle->author)); title.saveId(sid); + title.journalSize(nsacd->nacp.user_account_save_data_journal_size); + title.journalSizeMax(std::max( + nsacd->nacp.user_account_save_data_journal_size_max, nsacd->nacp.user_account_save_data_journal_size)); // load play statistics PdmPlayStatistics stats; @@ -339,6 +364,8 @@ void loadTitles(void) AccountUid bcatUid = {0}; title.init(FsSaveDataType_Bcat, tid, bcatUid, std::string(nle->name), std::string(nle->author)); title.saveId(sid); + title.journalSize(nsacd->nacp.bcat_delivery_cache_storage_size); + title.journalSizeMax(nsacd->nacp.bcat_delivery_cache_storage_size); loadIcon(tid, nsacd, outsize - sizeof(nsacd->nacp)); bcatTitles.push_back(title); @@ -359,6 +386,9 @@ void loadTitles(void) AccountUid deviceUid = {0}; title.init(FsSaveDataType_Device, tid, deviceUid, std::string(nle->name), std::string(nle->author)); title.saveId(sid); + title.journalSize(nsacd->nacp.device_save_data_journal_size); + title.journalSizeMax( + std::max(nsacd->nacp.device_save_data_journal_size_max, nsacd->nacp.device_save_data_journal_size)); loadIcon(tid, nsacd, outsize - sizeof(nsacd->nacp)); deviceTitles.push_back(title);