From 2f18d014842e2eda5a8a3cb50e1fbf43f3136849 Mon Sep 17 00:00:00 2001 From: realtasty <147815293+realtasty@users.noreply.github.com> Date: Thu, 5 Mar 2026 07:00:20 -0500 Subject: [PATCH 1/3] song loading performance overhaul w/ binary cache - added a binary cache (BinaryCache.cs) that stores hashes, durations, level IDs, and info.dat contents in a single file instead of the three separate json caches, auto-migrates from the old format on first run - in the main loading loop, if a song folder's timestamp matches what we have cached we just reconstruct the BeatmapLevel from memory instead of touching the disk at all - capped the parallelism to 8 threads max since on higher core count machines it was starving the main thread and causing the game to hang - made the cache save synchronous on the background thread so it doesn't get lost if the game exits right after loading - fixed a bug where the cache was gated behind the fullRefresh flag, meaning it was actually never used on startup since fullRefresh defaults to true - simplified GetDirectoryHash to just use the folder's last write timestamp instead of enumerating every file --- source/SongCore/Loader.cs | 224 ++++++++++++++---- source/SongCore/Utilities/BinaryCache.cs | 283 +++++++++++++++++++++++ source/SongCore/Utilities/Hashing.cs | 136 +++++------ 3 files changed, 519 insertions(+), 124 deletions(-) create mode 100644 source/SongCore/Utilities/BinaryCache.cs diff --git a/source/SongCore/Loader.cs b/source/SongCore/Loader.cs index b67ed0e..a5fcdf8 100644 --- a/source/SongCore/Loader.cs +++ b/source/SongCore/Loader.cs @@ -146,7 +146,7 @@ private void HandleSceneTransitionDidFinish(SceneTransitionType sceneTransitionT defaultCoverImage = _levelPackDetailViewController._defaultCoverSprite; beatmapCharacteristicCollection = _beatmapCharacteristicCollection; - if (Hashing.cachedSongHashData.Count == 0) + if (BinaryCache.Count == 0 && Hashing.cachedSongHashData.Count == 0) { Hashing.ReadCachedSongHashes(); Hashing.ReadCachedAudioData(); @@ -337,6 +337,8 @@ private async void RetrieveAllSongs(bool fullRefresh) #endregion + var fastPathCount = 0; + var slowPathCount = 0; ConcurrentDictionary foundSongPaths = fullRefresh ? new ConcurrentDictionary() : new ConcurrentDictionary(Hashing.cachedSongHashData.Keys.ToDictionary(Hashing.GetAbsolutePath, _ => false)); @@ -446,9 +448,10 @@ void AddOfficialBeatmapLevelsRepository(BeatmapLevelsRepository levelsRepository .Select(d => d.FullName) .ToArray(); var songFoldersCount = songFolders.Length; + // cap parallelism so we don't starve the main thread var parallelOptions = new ParallelOptions { - MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount / 2 - 1), + MaxDegreeOfParallelism = Math.Min(8, Math.Max(2, Environment.ProcessorCount / 2)), CancellationToken = _loadingTaskCancellationTokenSource.Token }; var processedSongsCount = 0; @@ -481,48 +484,84 @@ void AddOfficialBeatmapLevelsRepository(BeatmapLevelsRepository levelsRepository Parallel.ForEach(songFolders, parallelOptions, folder => { - string[] results; try { - results = Directory.GetFiles(folder, CustomLevelPathHelper.kStandardLevelInfoFilename, SearchOption.TopDirectoryOnly); - } - catch (Exception ex) - { - Plugin.Log.Warn($"Skipping missing or corrupt folder: '{folder}'"); - Plugin.Log.Warn(ex); - return; - } + var songPath = folder; + if (Directory.GetParent(songPath)?.Name == "Backups") + { + return; + } - if (results.Length == 0) - { - Plugin.Log.Warn($"Folder: '{folder}' is missing {CustomLevelPathHelper.kStandardLevelInfoFilename} file!"); - return; - } + if (!fullRefresh && (CustomLevels.ContainsKey(songPath) || CustomWIPLevels.ContainsKey(songPath))) + { + LoadingProgress = (float) Interlocked.Increment(ref processedSongsCount) / songFoldersCount; + return; + } - foreach (var result in results) - { + Hashing.TryGetRelativePath(songPath, out var relativePath); + long dirTimestamp; try { - var songPath = Path.GetDirectoryName(result)!; - if (Directory.GetParent(songPath)?.Name == "Backups") - { - continue; - } + dirTimestamp = Directory.GetLastWriteTimeUtc(songPath).ToFileTimeUtc(); + } + catch + { + Plugin.Log.Warn($"Skipping missing or corrupt folder: '{folder}'"); + LoadingProgress = (float) Interlocked.Increment(ref processedSongsCount) / songFoldersCount; + return; + } - if (!fullRefresh && (CustomLevels.ContainsKey(songPath) || CustomWIPLevels.ContainsKey(songPath))) - { - continue; - } - var wip = songPath.Contains("CustomWIPLevels"); - var customLevel = LoadCustomLevel(songPath); - if (!customLevel.HasValue) + BinaryCache.CacheEntry cached = null; + bool hasCacheHit = + (BinaryCache.TryGetValid(relativePath, dirTimestamp, out cached) || + BinaryCache.TryGet(relativePath, out cached)); + + if (hasCacheHit && cached != null + && !string.IsNullOrEmpty(cached.SongHash) && !string.IsNullOrEmpty(cached.InfoDatJson)) + { + + Interlocked.Increment(ref fastPathCount); + var reconstructed = ReconstructFromCache(songPath, cached); + if (reconstructed.HasValue) { - Plugin.Log.Error($"Failed to load custom level: {folder}"); - continue; + var (_, level) = reconstructed.Value; + var wip = songPath.Contains("CustomWIPLevels"); + if (!wip) + { + CustomLevelsById[level.levelID] = level; + CustomLevels[songPath] = level; + } + else + { + CustomWIPLevels[songPath] = level; + } + foundSongPaths.TryAdd(songPath, false); + LoadingProgress = (float) Interlocked.Increment(ref processedSongsCount) / songFoldersCount; + return; } + } + + + if (!File.Exists(Path.Combine(songPath, CustomLevelPathHelper.kStandardLevelInfoFilename))) + { + Plugin.Log.Warn($"Folder: '{folder}' is missing {CustomLevelPathHelper.kStandardLevelInfoFilename} file!"); + LoadingProgress = (float) Interlocked.Increment(ref processedSongsCount) / songFoldersCount; + return; + } + + Interlocked.Increment(ref slowPathCount); + var customLevel = LoadCustomLevel(songPath); + if (!customLevel.HasValue) + { + Plugin.Log.Error($"Failed to load custom level: {folder}"); + LoadingProgress = (float) Interlocked.Increment(ref processedSongsCount) / songFoldersCount; + return; + } + { var (_, level) = customLevel.Value; + var wip = songPath.Contains("CustomWIPLevels"); if (!wip) { CustomLevelsById[level.levelID] = level; @@ -532,14 +571,13 @@ void AddOfficialBeatmapLevelsRepository(BeatmapLevelsRepository levelsRepository { CustomWIPLevels[songPath] = level; } - foundSongPaths.TryAdd(songPath, false); } - catch (Exception e) - { - Plugin.Log.Error($"Failed to load song folder: {result}"); - Plugin.Log.Error(e); - } + } + catch (Exception e) + { + Plugin.Log.Error($"Failed to load song folder: {folder}"); + Plugin.Log.Error(e); } LoadingProgress = (float) Interlocked.Increment(ref processedSongsCount) / songFoldersCount; @@ -677,7 +715,7 @@ void AddOfficialBeatmapLevelsRepository(BeatmapLevelsRepository levelsRepository int folderCount = songCount - songCountWSF; string songOrSongs = songCount == 1 ? "song" : "songs"; string folderOrFolders = folderCount == 1 ? "folder" : "folders"; - Plugin.Log.Info($"Loaded {songCount} new {songOrSongs} ({songCountWSF}) in CustomLevels | {folderCount} in separate {folderOrFolders}) in {stopwatch.Elapsed.TotalSeconds} seconds"); + Plugin.Log.Info($"Loaded {songCount} new {songOrSongs} ({songCountWSF}) in CustomLevels | {folderCount} in separate {folderOrFolders}) in {stopwatch.Elapsed.TotalSeconds} seconds (fast:{fastPathCount} slow:{slowPathCount})"); try { #region AddSeparateFolderBeatmapsToRespectivePacks @@ -745,9 +783,12 @@ await UnityMainThreadTaskScheduler.Factory.StartNew(() => _loadingTask = null; await UnityMainThreadTaskScheduler.Factory.StartNew(() => SongsLoadedEvent?.Invoke(this, CustomLevels)); - // Write our cached hash info and + Hashing.UpdateCachedHashesInternal(foundSongPaths.Keys); Hashing.UpdateCachedAudioDataInternal(foundSongPaths.Keys); + + + BinaryCache.SaveAndPrune(foundSongPaths.Keys); await Collections.SaveCustomLevelSongDataAsync(); }; @@ -1063,6 +1104,109 @@ private bool AssignBeatmapToSeparateFolder( return false; } + + private (string hash, BeatmapLevel beatmapLevel)? ReconstructFromCache(string songPath, BinaryCache.CacheEntry cached) + { + try + { + var directoryInfo = new DirectoryInfo(songPath); + var json = cached.InfoDatJson; + var customLevelFolderInfo = new CustomLevelFolderInfo(directoryInfo.FullName, directoryInfo.Name, json); + + CustomLevelLoader.LoadedSaveData loadedSaveData; + BeatmapLevel? beatmapLevel; + + var version = BeatmapSaveDataHelpers.GetVersion(json); + if (version < BeatmapSaveDataHelpers.version4) + { + var standardLevelInfoSaveData = StandardLevelInfoSaveData.DeserializeFromJSONString(json); + if (standardLevelInfoSaveData == null) return null; + + loadedSaveData = new CustomLevelLoader.LoadedSaveData { customLevelFolderInfo = customLevelFolderInfo, standardLevelInfoSaveData = standardLevelInfoSaveData }; + beatmapLevel = _customLevelLoader.CreateBeatmapLevelFromV3(customLevelFolderInfo, standardLevelInfoSaveData); + } + else + { + var beatmapLevelSaveData = JsonConvert.DeserializeObject(json, JsonSettings.readableWithDefault); + if (beatmapLevelSaveData == null) return null; + BeatmapLevelSaveDataUtils.MigrateBeatmapLevelSaveData(beatmapLevelSaveData); + loadedSaveData = new CustomLevelLoader.LoadedSaveData { customLevelFolderInfo = customLevelFolderInfo, beatmapLevelSaveData = beatmapLevelSaveData }; + beatmapLevel = _customLevelLoader.CreateBeatmapLevelFromV4(customLevelFolderInfo, beatmapLevelSaveData); + } + + var hash = cached.SongHash; + var wip = songPath.Contains("CustomWIPLevels"); + + string levelID = CustomLevelLoader.kCustomLevelPrefixId + hash; + string folderName = directoryInfo.Name; + while (!Collections.LevelHashDictionary.TryAdd(levelID + (wip ? " WIP" : ""), hash)) + { + levelID += $"_{folderName}"; + } + + if (wip) + { + levelID += " WIP"; + } + + Collections.HashLevelDictionary.AddOrUpdate(hash, new List { levelID }, (_, levels) => + { + lock (levels) + { + levels.Add(levelID); + } + return levels; + }); + + Accessors.LevelIDAccessor(ref beatmapLevel) = levelID; + + if (cached.Duration > 0) + { + Accessors.SongDurationAccessor(ref beatmapLevel) = cached.Duration; + } + else + { + GetSongDuration(loadedSaveData, beatmapLevel); + } + + _customLevelLoader._loadedBeatmapSaveData[levelID] = loadedSaveData; + LoadedBeatmapSaveData.TryAdd(levelID, loadedSaveData); + + Hashing.TryGetRelativePath(songPath, out var cacheRelPath); + Hashing.cachedSongHashData[cacheRelPath] = new SongHashData(cached.DirTimestamp, hash); + + if (cached.Duration > 0) + { + Hashing.cachedAudioData[cacheRelPath] = new AudioCacheData(levelID, cached.Duration); + } + + if (!string.IsNullOrEmpty(cached.SongDataJson)) + { + try + { + var songData = JsonConvert.DeserializeObject(cached.SongDataJson); + if (songData != null) + { + Collections.CustomSongsData.TryAdd(levelID, songData); + } + } + catch { } + } + + if (!Collections.CustomSongsData.ContainsKey(levelID)) + { + Collections.CreateCustomLevelSongData(levelID, loadedSaveData); + } + + return (hash, beatmapLevel); + } + catch (Exception ex) + { + Plugin.Log.Warn($"Cache reconstruction failed for '{songPath}': {ex.Message}"); + return null; + } + } + public static (string hash, BeatmapLevel beatmapLevel)? LoadCustomLevel(string customLevelPath, SongFolderEntry? entry = null) { return Instance.LoadCustomLevelInternal(customLevelPath, entry); diff --git a/source/SongCore/Utilities/BinaryCache.cs b/source/SongCore/Utilities/BinaryCache.cs new file mode 100644 index 0000000..e166c8f --- /dev/null +++ b/source/SongCore/Utilities/BinaryCache.cs @@ -0,0 +1,283 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Newtonsoft.Json; +using SongCore.Data; + +namespace SongCore.Utilities +{ + internal static class BinaryCache + { + private const string Magic = "SC02"; + private const int FormatVersion = 2; + + internal static readonly string CachePath = Path.Combine( + IPA.Utilities.UnityGame.UserDataPath, nameof(SongCore), "SongCoreCache.bin"); + + + internal class CacheEntry + { + public string RelativePath; + public long DirTimestamp; + public string SongHash; + public float Duration; + public string LevelId; + public string InfoDatJson; + public string SongDataJson; + } + + + private static ConcurrentDictionary _entries = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + + internal static int Count => _entries.Count; + + internal static void Load() + { + _entries.Clear(); + + if (File.Exists(CachePath)) + { + try + { + LoadBinary(); + Plugin.Log.Info($"Loaded binary cache: {_entries.Count} entries from {CachePath}"); + return; + } + catch (Exception ex) + { + Plugin.Log.Warn($"Failed to load binary cache, will rebuild: {ex.Message}"); + _entries.Clear(); + } + } + + + LoadLegacyJsonCaches(); + } + + internal static bool TryGetValid(string relativePath, long currentDirTimestamp, out CacheEntry entry) + { + if (_entries.TryGetValue(relativePath, out entry) && entry.DirTimestamp == currentDirTimestamp) + { + return true; + } + + entry = null; + return false; + } + + internal static bool TryGet(string relativePath, out CacheEntry entry) + { + return _entries.TryGetValue(relativePath, out entry); + } + + internal static void Set(string relativePath, CacheEntry entry) + { + entry.RelativePath = relativePath; + _entries[relativePath] = entry; + } + + internal static bool Remove(string relativePath) + { + return _entries.TryRemove(relativePath, out _); + } + + internal static void SaveAndPrune(ICollection activePaths) + { + + var activeSet = new HashSet(activePaths, StringComparer.OrdinalIgnoreCase); + foreach (var key in _entries.Keys) + { + var absolutePath = Hashing.GetAbsolutePath(key); + if (!activeSet.Contains(absolutePath) && !activeSet.Contains(key)) + { + _entries.TryRemove(key, out _); + } + } + + try + { + SaveBinary(); + Plugin.Log.Info($"Saved binary cache: {_entries.Count} entries to {CachePath}"); + } + catch (Exception ex) + { + Plugin.Log.Error($"Failed to save binary cache: {ex.Message}"); + Plugin.Log.Error(ex); + } + } + + internal static IEnumerable> GetAllEntries() + { + return _entries; + } + + #region Binary Format I/O + + private static void LoadBinary() + { + using var fs = new FileStream(CachePath, FileMode.Open, FileAccess.Read, FileShare.Read, 65536); + using var reader = new BinaryReader(fs, Encoding.UTF8, leaveOpen: false); + + + var magic = reader.ReadString(); + if (magic != Magic) + { + throw new InvalidDataException($"Invalid cache magic: expected '{Magic}', got '{magic}'"); + } + + var version = reader.ReadInt32(); + if (version != FormatVersion) + { + throw new InvalidDataException($"Unsupported cache version: {version}"); + } + + var count = reader.ReadInt32(); + + for (int i = 0; i < count; i++) + { + var entry = new CacheEntry + { + RelativePath = reader.ReadString(), + DirTimestamp = reader.ReadInt64(), + SongHash = reader.ReadString(), + Duration = reader.ReadSingle(), + LevelId = reader.ReadString(), + InfoDatJson = reader.ReadString(), + SongDataJson = reader.ReadString() + }; + + _entries[entry.RelativePath] = entry; + } + } + + private static void SaveBinary() + { + var tempPath = CachePath + ".tmp"; + using (var fs = new FileStream(tempPath, FileMode.Create, FileAccess.Write, FileShare.None, 65536)) + using (var writer = new BinaryWriter(fs, Encoding.UTF8, leaveOpen: false)) + { + writer.Write(Magic); + writer.Write(FormatVersion); + writer.Write(_entries.Count); + + foreach (var kvp in _entries) + { + var entry = kvp.Value; + writer.Write(entry.RelativePath ?? string.Empty); + writer.Write(entry.DirTimestamp); + writer.Write(entry.SongHash ?? string.Empty); + writer.Write(entry.Duration); + writer.Write(entry.LevelId ?? string.Empty); + writer.Write(entry.InfoDatJson ?? string.Empty); + writer.Write(entry.SongDataJson ?? string.Empty); + } + } + + if (File.Exists(CachePath)) + { + File.Delete(CachePath); + } + File.Move(tempPath, CachePath); + } + + #endregion + + #region Legacy JSON Migration + + private static void LoadLegacyJsonCaches() + { + int migrated = 0; + + + if (File.Exists(Hashing.cachedHashDataPath)) + { + try + { + using var reader = new JsonTextReader(new StreamReader(Hashing.cachedHashDataPath)); + var serializer = JsonSerializer.CreateDefault(); + var hashData = serializer.Deserialize>(reader); + if (hashData != null) + { + foreach (var kvp in hashData) + { + var entry = GetOrCreate(kvp.Key); + entry.DirTimestamp = kvp.Value.directoryHash; + entry.SongHash = kvp.Value.songHash; + migrated++; + } + } + } + catch (Exception ex) + { + Plugin.Log.Warn($"Failed to migrate legacy hash cache: {ex.Message}"); + } + } + + + if (File.Exists(Hashing.cachedAudioDataPath)) + { + try + { + using var reader = new JsonTextReader(new StreamReader(Hashing.cachedAudioDataPath)); + var serializer = JsonSerializer.CreateDefault(); + var audioData = serializer.Deserialize>(reader); + if (audioData != null) + { + foreach (var kvp in audioData) + { + var entry = GetOrCreate(kvp.Key); + entry.Duration = kvp.Value.duration; + entry.LevelId = kvp.Value.id; + } + } + } + catch (Exception ex) + { + Plugin.Log.Warn($"Failed to migrate legacy duration cache: {ex.Message}"); + } + } + + if (migrated > 0) + { + Plugin.Log.Info($"Migrated {migrated} entries from legacy JSON caches to binary format."); + } + } + + private static CacheEntry GetOrCreate(string relativePath) + { + return _entries.GetOrAdd(relativePath, _ => new CacheEntry { RelativePath = relativePath }); + } + + #endregion + + #region Backward Compatibility Helpers + + internal static void PopulateLegacyHashDictionary(ConcurrentDictionary target) + { + target.Clear(); + foreach (var kvp in _entries) + { + if (!string.IsNullOrEmpty(kvp.Value.SongHash)) + { + target[kvp.Key] = new SongHashData(kvp.Value.DirTimestamp, kvp.Value.SongHash); + } + } + } + + internal static void PopulateLegacyAudioDictionary(ConcurrentDictionary target) + { + target.Clear(); + foreach (var kvp in _entries) + { + if (kvp.Value.Duration > 0) + { + target[kvp.Key] = new AudioCacheData(kvp.Value.LevelId ?? string.Empty, kvp.Value.Duration); + } + } + } + + #endregion + } +} diff --git a/source/SongCore/Utilities/Hashing.cs b/source/SongCore/Utilities/Hashing.cs index 40e85c3..d460471 100644 --- a/source/SongCore/Utilities/Hashing.cs +++ b/source/SongCore/Utilities/Hashing.cs @@ -20,23 +20,9 @@ public class Hashing public static void ReadCachedSongHashes() { - if (File.Exists(cachedHashDataPath)) - { - try - { - var songHashData = JsonConvert.DeserializeObject>(File.ReadAllText(cachedHashDataPath)); - if (songHashData != null) - { - cachedSongHashData = songHashData; - Plugin.Log.Info($"Finished loading cached hashes for {cachedSongHashData.Count} songs."); - } - } - catch (Exception ex) - { - Plugin.Log.Error($"Error loading cached song hashes: {ex.Message}"); - Plugin.Log.Error(ex); - } - } + BinaryCache.Load(); + BinaryCache.PopulateLegacyHashDictionary(cachedSongHashData); + Plugin.Log.Info($"Finished loading cached hashes for {cachedSongHashData.Count} songs."); } public static void UpdateCachedHashes(HashSet currentSongPaths) @@ -44,52 +30,33 @@ public static void UpdateCachedHashes(HashSet currentSongPaths) UpdateCachedHashesInternal(currentSongPaths); } - /// - /// Intended for use in the Loader - /// - /// internal static void UpdateCachedHashesInternal(ICollection currentSongPaths) { - foreach (var levelPath in cachedSongHashData.Keys) + + foreach (var kvp in cachedSongHashData) { - var absolutePath = GetAbsolutePath(levelPath); - if (!currentSongPaths.Contains(absolutePath) || (absolutePath == levelPath && IsInInstallPath(levelPath))) + if (BinaryCache.TryGet(kvp.Key, out var existing)) { - cachedSongHashData.TryRemove(levelPath, out _); + existing.SongHash = kvp.Value.songHash; + existing.DirTimestamp = kvp.Value.directoryHash; + } + else + { + BinaryCache.Set(kvp.Key, new BinaryCache.CacheEntry + { + RelativePath = kvp.Key, + DirTimestamp = kvp.Value.directoryHash, + SongHash = kvp.Value.songHash + }); } - } - - try - { - Plugin.Log.Info($"Saving cached hashes for {cachedSongHashData.Count} songs."); - File.WriteAllText(cachedHashDataPath, JsonConvert.SerializeObject(cachedSongHashData)); - } - catch (Exception ex) - { - Plugin.Log.Error($"Error saving cached song hashes: {ex.Message}"); - Plugin.Log.Error(ex); } } public static void ReadCachedAudioData() { - if (File.Exists(cachedAudioDataPath)) - { - try - { - var audioData = JsonConvert.DeserializeObject>(File.ReadAllText(cachedAudioDataPath)); - if (audioData != null) - { - cachedAudioData = audioData; - Plugin.Log.Info($"Finished loading cached durations for {cachedAudioData.Count} songs."); - } - } - catch (Exception ex) - { - Plugin.Log.Error($"Error loading cached song durations: {ex.Message}"); - Plugin.Log.Error(ex); - } - } + + BinaryCache.PopulateLegacyAudioDictionary(cachedAudioData); + Plugin.Log.Info($"Finished loading cached durations for {cachedAudioData.Count} songs."); } public static void UpdateCachedAudioData(HashSet currentSongPaths) @@ -97,46 +64,21 @@ public static void UpdateCachedAudioData(HashSet currentSongPaths) UpdateCachedAudioDataInternal(currentSongPaths); } - /// - /// Intended for use in the Loader - /// - /// internal static void UpdateCachedAudioDataInternal(ICollection currentSongPaths) { - foreach (var levelPath in cachedAudioData.Keys) + foreach (var kvp in cachedAudioData) { - var absolutePath = GetAbsolutePath(levelPath); - if (!currentSongPaths.Contains(absolutePath) || (absolutePath == levelPath && IsInInstallPath(levelPath))) + if (BinaryCache.TryGet(kvp.Key, out var existing)) { - cachedAudioData.TryRemove(levelPath, out _); + existing.Duration = kvp.Value.duration; + existing.LevelId = kvp.Value.id; } } - - try - { - Plugin.Log.Info($"Saving cached durations for {cachedAudioData.Count} songs."); - File.WriteAllText(cachedAudioDataPath, JsonConvert.SerializeObject(cachedAudioData)); - } - catch (Exception ex) - { - Plugin.Log.Error($"Error saving cached song durations: {ex.Message}"); - Plugin.Log.Error(ex); - } } private static long GetDirectoryHash(string directory) { - long hash = 0; - DirectoryInfo directoryInfo = new DirectoryInfo(directory); - foreach (FileInfo f in directoryInfo.GetFiles()) - { - hash ^= f.CreationTimeUtc.ToFileTimeUtc(); - hash ^= f.LastWriteTimeUtc.ToFileTimeUtc(); - hash ^= f.Name.GetHashCode(); - hash ^= f.Length; - } - - return hash; + return Directory.GetLastWriteTimeUtc(directory).ToFileTimeUtc(); } private static bool GetCachedSongData(string customLevelPath, out long directoryHash, out string cachedSongHash) @@ -144,7 +86,25 @@ private static bool GetCachedSongData(string customLevelPath, out long directory directoryHash = GetDirectoryHash(customLevelPath); TryGetRelativePath(customLevelPath, out var relativePath); - if (cachedSongHashData.TryGetValue(relativePath, out var cachedSong) && cachedSong.directoryHash == directoryHash) + + + if (BinaryCache.TryGetValid(relativePath, directoryHash, out var cachedEntry) && + !string.IsNullOrEmpty(cachedEntry.SongHash)) + { + cachedSongHash = cachedEntry.SongHash; + return true; + } + + if (BinaryCache.TryGet(relativePath, out var anyEntry) && + !string.IsNullOrEmpty(anyEntry.SongHash)) + { + cachedSongHash = anyEntry.SongHash; + return true; + } + + + if (cachedSongHashData.TryGetValue(relativePath, out var cachedSong) && + !string.IsNullOrEmpty(cachedSong.songHash)) { cachedSongHash = cachedSong.songHash; return true; @@ -221,6 +181,14 @@ public static string ComputeCustomLevelHash(CustomLevelFolderInfo customLevelFol string hash = CreateSha1FromFilesWithPrependBytes(prependBytes, files); TryGetRelativePath(customLevelFolderInfo.folderPath, out var relativePath); cachedSongHashData[relativePath] = new SongHashData(directoryHash, hash); + + + var entry = BinaryCache.TryGet(relativePath, out var existing) ? existing : new BinaryCache.CacheEntry(); + entry.RelativePath = relativePath; + entry.DirTimestamp = directoryHash; + entry.SongHash = hash; + BinaryCache.Set(relativePath, entry); + return hash; } From fbd3b1c7f360c6ec6bbf03a6aff7139a8a120881 Mon Sep 17 00:00:00 2001 From: realtasty <> Date: Fri, 6 Mar 2026 13:08:52 -0500 Subject: [PATCH 2/3] fix binary cache fast-path, stale hash fallback, and thread safety --- source/SongCore/Loader.cs | 17 +++++++++++------ source/SongCore/Utilities/Hashing.cs | 28 +++++++++++----------------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/source/SongCore/Loader.cs b/source/SongCore/Loader.cs index a5fcdf8..c5ed03f 100644 --- a/source/SongCore/Loader.cs +++ b/source/SongCore/Loader.cs @@ -511,11 +511,8 @@ void AddOfficialBeatmapLevelsRepository(BeatmapLevelsRepository levelsRepository return; } - BinaryCache.CacheEntry cached = null; - bool hasCacheHit = - (BinaryCache.TryGetValid(relativePath, dirTimestamp, out cached) || - BinaryCache.TryGet(relativePath, out cached)); + bool hasCacheHit = BinaryCache.TryGetValid(relativePath, dirTimestamp, out cached); if (hasCacheHit && cached != null && !string.IsNullOrEmpty(cached.SongHash) && !string.IsNullOrEmpty(cached.InfoDatJson)) @@ -542,7 +539,6 @@ void AddOfficialBeatmapLevelsRepository(BeatmapLevelsRepository levelsRepository } } - if (!File.Exists(Path.Combine(songPath, CustomLevelPathHelper.kStandardLevelInfoFilename))) { Plugin.Log.Warn($"Folder: '{folder}' is missing {CustomLevelPathHelper.kStandardLevelInfoFilename} file!"); @@ -951,6 +947,16 @@ private void DeleteSingleSong(string folderPath, bool deleteFolder) Accessors.LevelIDAccessor(ref beatmapLevel) = levelID; GetSongDuration(loadedSaveData, beatmapLevel); + + Hashing.TryGetRelativePath(loadedSaveData.customLevelFolderInfo.folderPath, out var cacheRelPath); + var cacheEntry = BinaryCache.TryGet(cacheRelPath, out var existing) ? existing : new BinaryCache.CacheEntry(); + cacheEntry.InfoDatJson = loadedSaveData.customLevelFolderInfo.levelInfoJsonString; + cacheEntry.LevelId = levelID; + if (Collections.CustomSongsData.TryGetValue(levelID, out var cachedSongData)) + { + cacheEntry.SongDataJson = JsonConvert.SerializeObject(cachedSongData); + } + BinaryCache.Set(cacheRelPath, cacheEntry); } catch (Exception e) { @@ -1169,7 +1175,6 @@ private bool AssignBeatmapToSeparateFolder( GetSongDuration(loadedSaveData, beatmapLevel); } - _customLevelLoader._loadedBeatmapSaveData[levelID] = loadedSaveData; LoadedBeatmapSaveData.TryAdd(levelID, loadedSaveData); Hashing.TryGetRelativePath(songPath, out var cacheRelPath); diff --git a/source/SongCore/Utilities/Hashing.cs b/source/SongCore/Utilities/Hashing.cs index d460471..fe6628b 100644 --- a/source/SongCore/Utilities/Hashing.cs +++ b/source/SongCore/Utilities/Hashing.cs @@ -78,7 +78,17 @@ internal static void UpdateCachedAudioDataInternal(ICollection currentSo private static long GetDirectoryHash(string directory) { - return Directory.GetLastWriteTimeUtc(directory).ToFileTimeUtc(); + long hash = 0; + DirectoryInfo directoryInfo = new DirectoryInfo(directory); + foreach (FileInfo f in directoryInfo.GetFiles()) + { + hash ^= f.CreationTimeUtc.ToFileTimeUtc(); + hash ^= f.LastWriteTimeUtc.ToFileTimeUtc(); + hash ^= f.Name.GetHashCode(); + hash ^= f.Length; + } + + return hash; } private static bool GetCachedSongData(string customLevelPath, out long directoryHash, out string cachedSongHash) @@ -87,7 +97,6 @@ private static bool GetCachedSongData(string customLevelPath, out long directory TryGetRelativePath(customLevelPath, out var relativePath); - if (BinaryCache.TryGetValid(relativePath, directoryHash, out var cachedEntry) && !string.IsNullOrEmpty(cachedEntry.SongHash)) { @@ -95,21 +104,6 @@ private static bool GetCachedSongData(string customLevelPath, out long directory return true; } - if (BinaryCache.TryGet(relativePath, out var anyEntry) && - !string.IsNullOrEmpty(anyEntry.SongHash)) - { - cachedSongHash = anyEntry.SongHash; - return true; - } - - - if (cachedSongHashData.TryGetValue(relativePath, out var cachedSong) && - !string.IsNullOrEmpty(cachedSong.songHash)) - { - cachedSongHash = cachedSong.songHash; - return true; - } - cachedSongHash = string.Empty; return false; } From 9501b9cd542d285861d4e961fde14a8f8cbbcab0 Mon Sep 17 00:00:00 2001 From: realtasty <> Date: Fri, 6 Mar 2026 13:16:32 -0500 Subject: [PATCH 3/3] skip cache save on incremental refresh --- NuGet.Config | 6 ++---- source/SongCore/Loader.cs | 9 +++++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/NuGet.Config b/NuGet.Config index 3228f75..73f2d67 100644 --- a/NuGet.Config +++ b/NuGet.Config @@ -1,8 +1,6 @@ - + - - - \ No newline at end of file + diff --git a/source/SongCore/Loader.cs b/source/SongCore/Loader.cs index c5ed03f..8f426e0 100644 --- a/source/SongCore/Loader.cs +++ b/source/SongCore/Loader.cs @@ -779,13 +779,14 @@ await UnityMainThreadTaskScheduler.Factory.StartNew(() => _loadingTask = null; await UnityMainThreadTaskScheduler.Factory.StartNew(() => SongsLoadedEvent?.Invoke(this, CustomLevels)); - Hashing.UpdateCachedHashesInternal(foundSongPaths.Keys); Hashing.UpdateCachedAudioDataInternal(foundSongPaths.Keys); - - BinaryCache.SaveAndPrune(foundSongPaths.Keys); - await Collections.SaveCustomLevelSongDataAsync(); + if (fullRefresh) + { + BinaryCache.SaveAndPrune(foundSongPaths.Keys); + await Collections.SaveCustomLevelSongDataAsync(); + } }; try