diff --git a/clean/wgshared/include/wg_state.h b/clean/wgshared/include/wg_state.h index 8fc60083..9b0045bd 100644 --- a/clean/wgshared/include/wg_state.h +++ b/clean/wgshared/include/wg_state.h @@ -44,7 +44,8 @@ // // Writes are atomic (WriteAtomic): a new temporary file in the same folder (.wgtmp--, created only where // nothing is), written in checked chunks, flushed and closed (both checked), then MoveFileExW(REPLACE_EXISTING | -// WRITE_THROUGH) over the file. A reader sees the old file or the new one, never part of one. On a failure the temporary file +// WRITE_THROUGH) over the file (MoveIntoPlace: retried for up to kMoveRetryMs while another program, an antivirus or indexer +// scan, briefly holds the file or the temporary one). A reader sees the old file or the new one, never part of one. On a failure the temporary file // is deleted when possible; the result (WriteResult, and SetOutcome for SetSettings) carries the first failure AND every // secondary one: a close that failed after an earlier failure, a cleanup that failed (its error and the file left). words() // says all of them. State files are small: more than kMaxStateBytes is refused. @@ -89,11 +90,47 @@ constexpr size_t kMaxStateBytes = 1u << 20; // no state file is larger constexpr DWORD kChunk = 64u * 1024u; // Test seam (tests only): called before each step of WriteAtomic ("write", "flush", "close", "move") and of its cleanup -// ("delete"); returning true makes that step fail as a failed call would (error ERROR_WRITE_FAULT). nullptr: never. +// ("delete"); returning true makes that step fail as a failed call would (error ERROR_WRITE_FAULT). "move-busy", before each +// attempt of MoveIntoPlace: true makes that attempt fail as if another program held the file (ERROR_ACCESS_DENIED). +// nullptr: never. using FaultHook = bool (*)(const char* step); inline FaultHook& TestFault() { static FaultHook hook = nullptr; return hook; } inline bool Faulted(const char* step) { return TestFault() && TestFault()(step); } +// A state file's rename into place (MoveFileExW with `flags`). Another program can hold the file or the new temporary file for +// a moment (an antivirus or indexer scan of a file just written, typically while a build writes many files): the rename then +// fails with ERROR_ACCESS_DENIED, ERROR_SHARING_VIOLATION or ERROR_LOCK_VIOLATION. Those are retried, with short pauses, for up +// to kMoveRetryMs; any other error, or the same one after that time, is the result (false, *err). The caller holds the state +// lock, so no WrongGPU program can hold the file meanwhile. `unchanged` (optional) is asked before every attempt after the +// first: the program that held the file may have written it meanwhile; false stops without replacing (false, *err 0, +// *changed true), so that program's bytes stay. +constexpr DWORD kMoveRetryMs = 2000; +inline bool MoveIntoPlace(const std::wstring& from, const std::wstring& to, DWORD flags, DWORD* err, + const std::function& unchanged = nullptr, bool* changed = nullptr) { + const ULONGLONG start = GetTickCount64(); + DWORD pause = 5; + if (changed) *changed = false; + for (bool first = true;; first = false) { + if (!first && unchanged && !unchanged()) { + *err = 0; + if (changed) *changed = true; + return false; + } + DWORD e = 0; + if (Faulted("move-busy")) e = ERROR_ACCESS_DENIED; + else if (MoveFileExW(from.c_str(), to.c_str(), flags)) return true; + else e = GetLastError(); + const bool held = e == ERROR_ACCESS_DENIED || e == ERROR_SHARING_VIOLATION || e == ERROR_LOCK_VIOLATION; + if (!held || GetTickCount64() - start >= kMoveRetryMs) { + *err = e; + SetLastError(e); + return false; + } + Sleep(pause); + if (pause < 100) pause *= 2; + } +} + inline std::wstring JoinPath(std::wstring a, const std::wstring& b) { if (!a.empty() && a.back() != L'\\' && a.back() != L'/') a.push_back(L'\\'); return a + b; @@ -367,7 +404,7 @@ class OperationGuard { } if (!e && !FlushFileBuffers(t)) e = GetLastError(); if (!CloseHandle(t) && !e) e = GetLastError(); - if (!e && !MoveFileExW(tmp.c_str(), target.c_str(), MOVEFILE_WRITE_THROUGH)) e = GetLastError(); + if (!e) MoveIntoPlace(tmp, target, MOVEFILE_WRITE_THROUGH, &e); if (e) DeleteFileW(tmp.c_str()); return e; } @@ -468,7 +505,7 @@ inline WriteResult WriteAtomic(const std::wstring& path, const std::string& byte } if (ok) { if (Faulted("move")) { ok = false; r.error = ERROR_WRITE_FAULT; } - else if (!MoveFileExW(tmp.c_str(), target.c_str(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) { ok = false; r.error = GetLastError(); } + else if (!MoveIntoPlace(tmp, target, MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH, &r.error)) ok = false; } if (ok) { r.ok = true; r.error = 0; return r; } const bool faultDelete = Faulted("delete"); @@ -862,9 +899,12 @@ inline SetOutcome UpdateStateFile(const std::wstring& gameDir, const wchar_t* fi if (existed) { CloseHandle(held.h); held.h = INVALID_HANDLE_VALUE; } Faulted("replace-check"); // (tests: another program changes or makes the file in this moment) if (existed ? !StillIdentity(path, before) : false) { dropTemp(); return fail(SetResult::Changed, 0); } - if (!MoveFileExW(tmp.c_str(), target.c_str(), (existed ? MOVEFILE_REPLACE_EXISTING : 0) | MOVEFILE_WRITE_THROUGH)) { - const DWORD me = GetLastError(); + DWORD me = 0; + bool changedMeanwhile = false; // a retried replace: the file must still be the one decided on + const auto unchanged = [&]() { return !existed || StillIdentity(path, before); }; // (absent: one that appears refuses the move itself) + if (!MoveIntoPlace(tmp, target, (existed ? MOVEFILE_REPLACE_EXISTING : 0) | MOVEFILE_WRITE_THROUGH, &me, unchanged, &changedMeanwhile)) { dropTemp(); + if (changedMeanwhile) return fail(SetResult::Changed, 0); // the program that held the file wrote it: its bytes stay if (!existed && (me == ERROR_ALREADY_EXISTS || me == ERROR_FILE_EXISTS)) return fail(SetResult::Changed, 0); // it appeared: the other program's o.write.error = me; return fail(me == ERROR_ACCESS_DENIED ? SetResult::AccessDenied : SetResult::WriteFailed, me); diff --git a/clean/wgshared/tests/test_wg_state.cpp b/clean/wgshared/tests/test_wg_state.cpp index 6ed6d33e..219715ad 100644 --- a/clean/wgshared/tests/test_wg_state.cpp +++ b/clean/wgshared/tests/test_wg_state.cpp @@ -101,6 +101,11 @@ static std::wstring Q(const std::wstring& s) { while (tail < s.size() && s[s.size() - 1 - tail] == L'\\') ++tail; return L"\"" + s + std::wstring(tail, L'\\') + L"\""; } +// A setting's value, or "(none)" when the key is missing (a failed write or read fails the check, never crashes the test). +static std::string Val(const Settings& s, const char* key) { + const std::string* v = Get(s, key); + return v ? *v : std::string("(none)"); +} static unsigned long Counter(const std::wstring& game) { const Settings s = ReadSettings(game); const std::string* v = Get(s, "counter"); @@ -117,16 +122,24 @@ static int Child(int argc, wchar_t** argv) { const std::wstring mode = argv[1], game = argv[2]; if (mode == L"--child-add" && argc >= 4) { // n locked read-modify-write increments of one counter const int n = _wtoi(argv[3]); + const std::wstring diag = argc >= 5 ? argv[4] : L""; // why it stopped, for the parent's failure line + auto stop = [&](int code, const std::string& why) { + if (!diag.empty()) WriteText(diag, why); + return code; + }; for (int i = 0; i < n; ++i) { StateLock lock(game, 30000); - if (!lock.held()) return 3; + if (!lock.held()) return stop(3, "lock not held at increment " + std::to_string(i) + " (error " + std::to_string(lock.error()) + ")"); const Settings s = ReadSettings(game); // re-read after the lock is taken + if (!s.why.empty()) return stop(5, "settings.ini " + s.why + " at increment " + std::to_string(i)); // never count from 0 const std::string* v = Get(s, "counter"); - const unsigned long c = v ? std::strtoul(v->c_str(), nullptr, 10) : 0; + if (!v) return stop(6, "no counter in settings.ini at increment " + std::to_string(i)); + const unsigned long c = std::strtoul(v->c_str(), nullptr, 10); Settings w = s; Set(w, "counter", std::to_string(c + 1)); Sleep(0); // widen the window a lost update would need - if (!WriteAtomic(SettingsPathOf(game), RenderSettings(w)).ok) return 4; + const WriteResult r = WriteAtomic(SettingsPathOf(game), RenderSettings(w)); + if (!r.ok) return stop(4, "write " + r.words() + " at increment " + std::to_string(i) + " (counter was " + std::to_string(c) + ")"); } return 0; } @@ -208,7 +221,7 @@ int wmain(int argc, wchar_t** argv) { // ---- the format: tolerant, everything kept; corrupt only for text that cannot be kept { Settings s = ParseSettings("; comment\r\n\r\nrr_mode=fast\r\nrr_enabled=1\n"); - check(s.present && !s.corrupt && s.kv.size() == 2 && *Get(s, "rr_mode") == "fast" && *Get(s, "rr_enabled") == "1", "parse: comments, blank lines, CRLF and LF"); + check(s.present && !s.corrupt && s.kv.size() == 2 && Val(s, "rr_mode") == "fast" && Val(s, "rr_enabled") == "1", "parse: comments, blank lines, CRLF and LF"); struct { const char* text; const char* key; const char* value; } kept[] = { {"rr_mode = fast\r\n", "rr_mode", "fast"}, {"[wronggpu]\r\nrr_mode=fast\r\n", "rr_mode", "fast"}, {"RR_MODE=fast\r\n", nullptr, nullptr}, {"rr_mode=fast\r\nrr_mode=accurate\r\n", "rr_mode", "fast"}, @@ -263,7 +276,7 @@ int wmain(int argc, wchar_t** argv) { WriteText(JoinPath(state, L"install.record"), std::string("WRONGGPU-RECORD 2\r\nversion\t0.6.5\r\ninstall-id\t") + kId + "\r\n"); check(InstallIdOf(game) == kId, "InstallIdOf: the record's install-id line"); const std::wstring f = SettingsPathOf(game); - check(SetSettings(game, {{"rr_mode", "fast"}}).ok() && *Get(ReadSettings(game), "rr_mode") == "fast", "SetSettings writes the file"); + check(SetSettings(game, {{"rr_mode", "fast"}}).ok() && Val(ReadSettings(game), "rr_mode") == "fast", "SetSettings writes the file"); const std::string before = ReadText(f); WriteResult w = WriteAtomic(f, std::string(kMaxStateBytes + 1, 'x')); check(!w.ok && w.error == ERROR_FILE_TOO_LARGE && ReadText(f) == before, "over kMaxStateBytes: refused, the file unchanged"); @@ -293,6 +306,44 @@ int wmain(int argc, wchar_t** argv) { w.words().find("could not be deleted") != std::string::npos && ReadText(f) == before, "a cleanup that fails: its error and the file left are reported (words say so), the file unchanged"); if (!w.tempLeft.empty()) DeleteFileW(w.tempLeft.c_str()); + // another program holds the file for a moment (an antivirus or indexer scan): the replace is retried for up to kMoveRetryMs + static int busy = 0, tries = 0; + TestFault() = [](const char* s) { return std::strcmp(s, "move-busy") == 0 && ++tries <= busy; }; + busy = 3, tries = 0; + w = WriteAtomic(f, "rr_mode=exact\r\n"); + check(w.ok && tries == 4 && Val(ReadSettings(game), "rr_mode") == "exact" && List(state).size() == 1, + "the first 3 replaces refused (file held): retried, the 4th writes the file"); + busy = 1000000, tries = 0; + ULONGLONG t0 = GetTickCount64(); + w = WriteAtomic(f, "rr_mode=fast\r\n"); + ULONGLONG took = GetTickCount64() - t0; + check(!w.ok && w.error == ERROR_ACCESS_DENIED && took >= kMoveRetryMs - 100 && took < kMoveRetryMs + 3000 && w.tempLeft.empty() && + Val(ReadSettings(game), "rr_mode") == "exact" && List(state).size() == 1, + "refused for longer than kMoveRetryMs: fails with its error (5) after the bound, the file unchanged, no temporary file left"); + busy = 1000000, tries = 0; + SetOutcome so = SetSettings(game, {{"rr_mode", "fast"}}); + check(!so.ok() && so.result == SetResult::AccessDenied && tries > 1 && Val(ReadSettings(game), "rr_mode") == "exact" && List(state).size() == 1, + "SetSettings: the replace refused for longer than the bound is reported (AccessDenied), the file unchanged"); + TestFault() = nullptr; + // the same with a real handle: settings.ini opened for reading only, sharing read (as a scanner does) + HANDLE held = CreateFileW(f.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + static HANDLE heldCopy = INVALID_HANDLE_VALUE; + heldCopy = held; + HANDLE closer = CreateThread(nullptr, 0, [](LPVOID) -> DWORD { Sleep(300); CloseHandle(heldCopy); return 0; }, nullptr, 0, nullptr); + w = held != INVALID_HANDLE_VALUE && closer ? WriteAtomic(f, "rr_mode=fast\r\n") : WriteResult{}; + if (closer) { WaitForSingleObject(closer, 10000); CloseHandle(closer); } + check(held != INVALID_HANDLE_VALUE && w.ok && Val(ReadSettings(game), "rr_mode") == "fast" && List(state).size() == 1, + "settings.ini held by another handle for 300 ms: the write waits it out and succeeds"); + held = CreateFileW(f.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + t0 = GetTickCount64(); + w = held != INVALID_HANDLE_VALUE ? WriteAtomic(f, "rr_mode=exact\r\n") : WriteResult{}; + took = GetTickCount64() - t0; + if (held != INVALID_HANDLE_VALUE) CloseHandle(held); + if (!w.ok && w.error != ERROR_ACCESS_DENIED) std::printf(" (held write: error %lu after %llu ms)\n", w.error, took); + check(held != INVALID_HANDLE_VALUE && !w.ok && (w.error == ERROR_ACCESS_DENIED || w.error == ERROR_SHARING_VIOLATION) && took >= kMoveRetryMs - 100 && + w.tempLeft.empty() && Val(ReadSettings(game), "rr_mode") == "fast" && List(state).size() == 1, + "settings.ini held by another handle past the bound: fails visibly with its error, the file unchanged, no temporary file left"); + check(WriteAtomic(f, before).ok, "the stamped file put back for the checks below"); w = WriteAtomic(JoinPath(JoinPath(root, L"missing-folder"), L"x.ini"), "x"); check(!w.ok && w.error != 0, "a write into a missing folder fails with its error"); // the stamp: the file WrongGPU created is stamped and stays so while only WrongGPU writes it @@ -302,7 +353,7 @@ int wmain(int argc, wchar_t** argv) { WriteText(f, ReadText(f) + "player_key = 7\r\n; the player's note \xc3\xa4\r\n; wronggpu-written a player's line\r\n"); SetOutcome o = SetSettings(game, {{"rr_mode", "accurate"}}); std::string t = ReadText(f); - check(o.ok() && o.written && *Get(ReadSettings(game), "rr_mode") == "accurate" && t.find("player_key = 7\r\n") != std::string::npos && + check(o.ok() && o.written && Val(ReadSettings(game), "rr_mode") == "accurate" && t.find("player_key = 7\r\n") != std::string::npos && t.find("; the player's note \xc3\xa4\r\n; wronggpu-written a player's line\r\n") != std::string::npos && !StampedByWrongGPU(t, "settings.ini", kId), "a file the player edited: the menu's save keeps the player's key and note as they are and writes no stamp (the player's file now)"); WriteText(f, "\xEF\xBB\xBF# made by the player\r\nRR Mode = Fast\r\n"); @@ -405,7 +456,7 @@ int wmain(int argc, wchar_t** argv) { } std::wstring dn; CreateDirectoryW(ExtendedPath(StateDirOf(deep)).c_str(), nullptr); - check(LockNameFor(deep, &dn) && Narrow(dn) != names[0] && SetSettings(deep, {{"k", "v"}}).ok() && *Get(ReadSettings(deep), "k") == "v", + check(LockNameFor(deep, &dn) && Narrow(dn) != names[0] && SetSettings(deep, {{"k", "v"}}).ok() && Val(ReadSettings(deep), "k") == "v", "a game folder deeper than MAX_PATH: lock name, write and read work"); const std::wstring dout = JoinPath(root, L"name-deep"); HANDLE k = Spawn(L"--child-lockname " + Q(deep) + L" " + Q(dout)); @@ -425,10 +476,28 @@ int wmain(int argc, wchar_t** argv) { { WriteText(SettingsPathOf(game), "counter=0\r\n"); std::vector kids; - for (int i = 0; i < 4; ++i) kids.push_back(Spawn(L"--child-add " + Q(aliases[(i * 2 + 1) % aliases.size()]) + L" 50")); + std::vector diags; + for (int i = 0; i < 4; ++i) { + diags.push_back(JoinPath(root, L"add-diag-" + std::to_wstring(i))); + DeleteFileW(diags.back().c_str()); + kids.push_back(Spawn(L"--child-add " + Q(aliases[(i * 2 + 1) % aliases.size()]) + L" 50 " + Q(diags.back()))); + } bool ok = true; - for (HANDLE k : kids) ok = k && WaitExit(k, 120000) == 0 && ok; - check(ok && Counter(game) == 200, "4 processes, 4 different aliases, 50 locked increments each: 200 (one lock for all spellings)"); + std::vector codes; + for (HANDLE k : kids) { + codes.push_back(k ? WaitExit(k, 120000) : 998); // 998: not started, 999: no exit in time + ok = codes.back() == 0 && ok; + } + const unsigned long count = Counter(game); + if (!ok || count != 200) { // which way it failed: a child that stopped (and why), or increments lost + std::printf(" (count %lu; exit codes", count); + for (DWORD c : codes) std::printf(" %lu", static_cast(c)); + std::printf(")\n"); + for (size_t i = 0; i < diags.size(); ++i) + if (Exists(diags[i])) std::printf(" (child %zu, alias %s: %s)\n", i, Narrow(aliases[(i * 2 + 1) % aliases.size()]).c_str(), ReadText(diags[i]).c_str()); + } + for (auto& d : diags) DeleteFileW(d.c_str()); + check(ok && count == 200, "4 processes, 4 different aliases, 50 locked increments each: 200 (one lock for all spellings)"); check(List(state).size() == 1, "no temporary file stays after concurrent writes"); } @@ -450,7 +519,7 @@ int wmain(int argc, wchar_t** argv) { CreateDirectoryW(state.c_str(), nullptr); DeleteFileW(SettingsPathOf(game).c_str()); check(SetSettings(game, {{"rr_mode", "fast"}}).ok(), "(interleaving: the file starts with fast)"); - const std::string staleRead = *Get(ReadSettings(game), "rr_mode"); // a decision would rest on this read: "fast" + const std::string staleRead = Val(ReadSettings(game), "rr_mode"); // a decision would rest on this read: "fast" HANDLE held = CreateEventW(nullptr, TRUE, FALSE, nullptr); std::thread menu([&] { // the game's menu: takes the lock, saves accurate a moment later, lets go StateLock lock(game, 2000); @@ -463,13 +532,13 @@ int wmain(int argc, wchar_t** argv) { WaitForSingleObject(held, 5000); std::string seen; const SetOutcome o = UpdateSettings(game, [&](const Settings& now, std::vector>* out) { - seen = Get(now, "rr_mode") ? *Get(now, "rr_mode") : ""; + seen = Get(now, "rr_mode") ? Val(now, "rr_mode") : ""; if (seen.empty()) out->push_back({"rr_mode", staleRead}); // only when there is none: a saved mode stays out->push_back({"rr_exact_allowed", seen == "exact" ? "1" : "0"}); }, 5000); menu.join(); CloseHandle(held); - check(staleRead == "fast" && o.ok() && seen == "accurate" && *Get(ReadSettings(game), "rr_mode") == "accurate" && StampedByWrongGPU(ReadText(SettingsPathOf(game)), "settings.ini", kId), + check(staleRead == "fast" && o.ok() && seen == "accurate" && Val(ReadSettings(game), "rr_mode") == "accurate" && StampedByWrongGPU(ReadText(SettingsPathOf(game)), "settings.ini", kId), "UpdateSettings: decide() sees the mode another thread saved while this one waited for the lock (accurate), not the older read " "(fast); the newer mode stays"); const std::string before = ReadText(SettingsPathOf(game)); @@ -500,7 +569,7 @@ int wmain(int argc, wchar_t** argv) { }; SetOutcome o = SetSettings(game, {{"rr_mode", "accurate"}}); TestFault() = nullptr; - check(g_editorWriteError == ERROR_SHARING_VIOLATION && g_editorMoveError != 0 && o.ok() && o.written && *Get(ReadSettings(game), "rr_mode") == "accurate", + check(g_editorWriteError == ERROR_SHARING_VIOLATION && g_editorMoveError != 0 && o.ok() && o.written && Val(ReadSettings(game), "rr_mode") == "accurate", "while WrongGPU holds the file, an editor's write and an editor's replace are refused by Windows (its bytes never land); WrongGPU's save completes"); // 2. the last moment (the file let go, before the move): another program changes the file: WrongGPU's save is refused WriteText(f, "rr_mode=fast\r\n"); @@ -513,6 +582,22 @@ int wmain(int argc, wchar_t** argv) { check(o.result == SetResult::Changed && !o.written && ReadText(f) == "rr_mode=exact\r\n; the player's save\r\n" && List(state).size() == 1 && o.words().find("nothing was saved") != std::string::npos, "a change in the last moment before the move: WrongGPU's save is refused (Changed, in words), the player's bytes stay, no temporary file"); + // 2b. the replace refused while another program holds the file, which it writes and lets go during the retries: refused + WriteText(f, "rr_mode=fast\r\n"); + static int busyCalls = 0; + busyCalls = 0; + TestFault() = [](const char* step) { + if (std::strcmp(step, "move-busy") != 0) return false; + if (++busyCalls > 1) return false; + Sleep(20); + WriteText(g_hookFile, "rr_mode=exact\r\n; the editor's save\r\n"); + return true; + }; + o = SetSettings(game, {{"rr_mode", "accurate"}}); + TestFault() = nullptr; + check(o.result == SetResult::Changed && !o.written && busyCalls == 1 && ReadText(f) == "rr_mode=exact\r\n; the editor's save\r\n" && + List(state).size() == 1, + "the replace refused, the holder writes the file meanwhile: the retry stops (Changed), the holder's bytes stay, no temporary file"); // 3. the file was not there: another program makes it before the move: refused, its bytes stay DeleteFileW(f.c_str()); TestFault() = [](const char* step) { @@ -540,12 +625,12 @@ int wmain(int argc, wchar_t** argv) { { StateLock lock(game, 5000); const Settings s = ReadSettings(game); // re-read after the lock - check(up && keep && lock.held() && lock.abandoned() && !s.corrupt && *Get(s, "counter") == "7", + check(up && keep && lock.held() && lock.abandoned() && !s.corrupt && Val(s, "counter") == "7", "a holder that died mid-write: the lock is taken (abandoned), the state re-read is the old whole file"); std::wstring temp; for (auto& n : List(state)) if (IsTempName(n)) temp = n; check(!temp.empty(), "the half-written temporary file is there and named as a temporary file (never read as state)"); - check(SetSettings(game, {{"counter", "8"}}).ok() && *Get(ReadSettings(game), "counter") == "8", "a write under the abandoned-then-taken lock works"); + check(SetSettings(game, {{"counter", "8"}}).ok() && Val(ReadSettings(game), "counter") == "8", "a write under the abandoned-then-taken lock works"); if (!temp.empty()) DeleteFileW(JoinPath(state, temp).c_str()); } if (keep) CloseHandle(keep);