Skip to content

performance overhaul #181

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
realtasty wants to merge 3 commits into
base: master
Choose a base branch
from
Open

performance overhaul #181

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2f18d01
song loading performance overhaul w/ binary cache
realtasty Mar 5, 2026
fbd3b1c
fix binary cache fast-path, stale hash fallback, and thread safety
Mar 6, 2026
9501b9c
skip cache save on incremental refresh
Mar 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions NuGet.Config
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<?xml version="1.0" encoding="utf-8"?>
<configuration>
<packageSources>
<add key="Atlas-Rhythm GH Packages" value="https://nuget.pkg.github.com/Atlas-Rhythm/index.json" />
<add key="ErisApps GH Packages" value="https://nuget.pkg.github.com/ErisApps/index.json" />
<add key="nuget.org" value="https://api.nuget.org/v3/index.json" protocolVersion="3" />
</packageSources>
</configuration>
</configuration>
232 changes: 191 additions & 41 deletions source/SongCore/Loader.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ private void HandleSceneTransitionDidFinish(SceneTransitionType sceneTransitionT
defaultCoverImage = _levelPackDetailViewController._defaultCoverSprite;
beatmapCharacteristicCollection = _beatmapCharacteristicCollection;

if (Hashing.cachedSongHashData.Count == 0)
if (BinaryCache.Count == 0 && Hashing.cachedSongHashData.Count == 0)
{
Hashing.ReadCachedSongHashes();
Hashing.ReadCachedAudioData();
Expand Down Expand Up @@ -337,6 +337,8 @@ private async void RetrieveAllSongs(bool fullRefresh)

#endregion

var fastPathCount = 0;
var slowPathCount = 0;
ConcurrentDictionary<string, bool> foundSongPaths = fullRefresh
? new ConcurrentDictionary<string, bool>()
: new ConcurrentDictionary<string, bool>(Hashing.cachedSongHashData.Keys.ToDictionary(Hashing.GetAbsolutePath, _ => false));
Expand Down Expand Up @@ -446,9 +448,10 @@ void AddOfficialBeatmapLevelsRepository(BeatmapLevelsRepository levelsRepository
.Select(d => d.FullName)
.ToArray();
var songFoldersCount = songFolders.Length;
// cap parallelism so we don't starve the main thread
var parallelOptions = new ParallelOptions
{
MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount / 2 - 1),
MaxDegreeOfParallelism = Math.Min(8, Math.Max(2, Environment.ProcessorCount / 2)),
CancellationToken = _loadingTaskCancellationTokenSource.Token
};
var processedSongsCount = 0;
Expand Down Expand Up @@ -481,48 +484,80 @@ void AddOfficialBeatmapLevelsRepository(BeatmapLevelsRepository levelsRepository

Parallel.ForEach(songFolders, parallelOptions, folder =>
{
string[] results;
try
{
results = Directory.GetFiles(folder, CustomLevelPathHelper.kStandardLevelInfoFilename, SearchOption.TopDirectoryOnly);
}
catch (Exception ex)
{
Plugin.Log.Warn($"Skipping missing or corrupt folder: '{folder}'");
Plugin.Log.Warn(ex);
return;
}
var songPath = folder;
if (Directory.GetParent(songPath)?.Name == "Backups")
{
return;
}

if (results.Length == 0)
{
Plugin.Log.Warn($"Folder: '{folder}' is missing {CustomLevelPathHelper.kStandardLevelInfoFilename} file!");
return;
}
if (!fullRefresh && (CustomLevels.ContainsKey(songPath) || CustomWIPLevels.ContainsKey(songPath)))
{
LoadingProgress = (float) Interlocked.Increment(ref processedSongsCount) / songFoldersCount;
return;
}

foreach (var result in results)
{
Hashing.TryGetRelativePath(songPath, out var relativePath);
long dirTimestamp;
try
{
var songPath = Path.GetDirectoryName(result)!;
if (Directory.GetParent(songPath)?.Name == "Backups")
{
continue;
}
dirTimestamp = Directory.GetLastWriteTimeUtc(songPath).ToFileTimeUtc();
}
catch
{
Plugin.Log.Warn($"Skipping missing or corrupt folder: '{folder}'");
LoadingProgress = (float) Interlocked.Increment(ref processedSongsCount) / songFoldersCount;
return;
}

if (!fullRefresh && (CustomLevels.ContainsKey(songPath) || CustomWIPLevels.ContainsKey(songPath)))
{
continue;
}
BinaryCache.CacheEntry cached = null;
bool hasCacheHit = BinaryCache.TryGetValid(relativePath, dirTimestamp, out cached);

var wip = songPath.Contains("CustomWIPLevels");
var customLevel = LoadCustomLevel(songPath);
if (!customLevel.HasValue)
if (hasCacheHit && cached != null
&& !string.IsNullOrEmpty(cached.SongHash) && !string.IsNullOrEmpty(cached.InfoDatJson))
{

Interlocked.Increment(ref fastPathCount);
var reconstructed = ReconstructFromCache(songPath, cached);
if (reconstructed.HasValue)
{
Plugin.Log.Error($"Failed to load custom level: {folder}");
continue;
var (_, level) = reconstructed.Value;
var wip = songPath.Contains("CustomWIPLevels");
if (!wip)
{
CustomLevelsById[level.levelID] = level;
CustomLevels[songPath] = level;
}
else
{
CustomWIPLevels[songPath] = level;
}
foundSongPaths.TryAdd(songPath, false);
LoadingProgress = (float) Interlocked.Increment(ref processedSongsCount) / songFoldersCount;
return;
}
}

if (!File.Exists(Path.Combine(songPath, CustomLevelPathHelper.kStandardLevelInfoFilename)))
{
Plugin.Log.Warn($"Folder: '{folder}' is missing {CustomLevelPathHelper.kStandardLevelInfoFilename} file!");
LoadingProgress = (float) Interlocked.Increment(ref processedSongsCount) / songFoldersCount;
return;
}

Interlocked.Increment(ref slowPathCount);
var customLevel = LoadCustomLevel(songPath);
if (!customLevel.HasValue)
{
Plugin.Log.Error($"Failed to load custom level: {folder}");
LoadingProgress = (float) Interlocked.Increment(ref processedSongsCount) / songFoldersCount;
return;
}

{
var (_, level) = customLevel.Value;
var wip = songPath.Contains("CustomWIPLevels");
if (!wip)
{
CustomLevelsById[level.levelID] = level;
Expand All @@ -532,14 +567,13 @@ void AddOfficialBeatmapLevelsRepository(BeatmapLevelsRepository levelsRepository
{
CustomWIPLevels[songPath] = level;
}

foundSongPaths.TryAdd(songPath, false);
}
catch (Exception e)
{
Plugin.Log.Error($"Failed to load song folder: {result}");
Plugin.Log.Error(e);
}
}
catch (Exception e)
{
Plugin.Log.Error($"Failed to load song folder: {folder}");
Plugin.Log.Error(e);
}

LoadingProgress = (float) Interlocked.Increment(ref processedSongsCount) / songFoldersCount;
Expand Down Expand Up @@ -677,7 +711,7 @@ void AddOfficialBeatmapLevelsRepository(BeatmapLevelsRepository levelsRepository
int folderCount = songCount - songCountWSF;
string songOrSongs = songCount == 1 ? "song" : "songs";
string folderOrFolders = folderCount == 1 ? "folder" : "folders";
Plugin.Log.Info($"Loaded {songCount} new {songOrSongs} ({songCountWSF}) in CustomLevels | {folderCount} in separate {folderOrFolders}) in {stopwatch.Elapsed.TotalSeconds} seconds");
Plugin.Log.Info($"Loaded {songCount} new {songOrSongs} ({songCountWSF}) in CustomLevels | {folderCount} in separate {folderOrFolders}) in {stopwatch.Elapsed.TotalSeconds} seconds (fast:{fastPathCount} slow:{slowPathCount})");
try
{
#region AddSeparateFolderBeatmapsToRespectivePacks
Expand Down Expand Up @@ -745,10 +779,14 @@ await UnityMainThreadTaskScheduler.Factory.StartNew(() =>
_loadingTask = null;
await UnityMainThreadTaskScheduler.Factory.StartNew(() => SongsLoadedEvent?.Invoke(this, CustomLevels));

// Write our cached hash info and
Hashing.UpdateCachedHashesInternal(foundSongPaths.Keys);
Hashing.UpdateCachedAudioDataInternal(foundSongPaths.Keys);
await Collections.SaveCustomLevelSongDataAsync();

if (fullRefresh)
{
BinaryCache.SaveAndPrune(foundSongPaths.Keys);
await Collections.SaveCustomLevelSongDataAsync();
}
};

try
Expand Down Expand Up @@ -910,6 +948,16 @@ private void DeleteSingleSong(string folderPath, bool deleteFolder)

Accessors.LevelIDAccessor(ref beatmapLevel) = levelID;
GetSongDuration(loadedSaveData, beatmapLevel);

Hashing.TryGetRelativePath(loadedSaveData.customLevelFolderInfo.folderPath, out var cacheRelPath);
var cacheEntry = BinaryCache.TryGet(cacheRelPath, out var existing) ? existing : new BinaryCache.CacheEntry();
cacheEntry.InfoDatJson = loadedSaveData.customLevelFolderInfo.levelInfoJsonString;
cacheEntry.LevelId = levelID;
if (Collections.CustomSongsData.TryGetValue(levelID, out var cachedSongData))
{
cacheEntry.SongDataJson = JsonConvert.SerializeObject(cachedSongData);
}
BinaryCache.Set(cacheRelPath, cacheEntry);
}
catch (Exception e)
{
Expand Down Expand Up @@ -1063,6 +1111,108 @@ private bool AssignBeatmapToSeparateFolder(
return false;
}


private (string hash, BeatmapLevel beatmapLevel)? ReconstructFromCache(string songPath, BinaryCache.CacheEntry cached)
{
try
{
var directoryInfo = new DirectoryInfo(songPath);
var json = cached.InfoDatJson;
var customLevelFolderInfo = new CustomLevelFolderInfo(directoryInfo.FullName, directoryInfo.Name, json);

CustomLevelLoader.LoadedSaveData loadedSaveData;
BeatmapLevel? beatmapLevel;

var version = BeatmapSaveDataHelpers.GetVersion(json);
if (version < BeatmapSaveDataHelpers.version4)
{
var standardLevelInfoSaveData = StandardLevelInfoSaveData.DeserializeFromJSONString(json);
if (standardLevelInfoSaveData == null) return null;

loadedSaveData = new CustomLevelLoader.LoadedSaveData { customLevelFolderInfo = customLevelFolderInfo, standardLevelInfoSaveData = standardLevelInfoSaveData };
beatmapLevel = _customLevelLoader.CreateBeatmapLevelFromV3(customLevelFolderInfo, standardLevelInfoSaveData);
}
else
{
var beatmapLevelSaveData = JsonConvert.DeserializeObject<BeatmapLevelSaveData>(json, JsonSettings.readableWithDefault);
if (beatmapLevelSaveData == null) return null;
BeatmapLevelSaveDataUtils.MigrateBeatmapLevelSaveData(beatmapLevelSaveData);
loadedSaveData = new CustomLevelLoader.LoadedSaveData { customLevelFolderInfo = customLevelFolderInfo, beatmapLevelSaveData = beatmapLevelSaveData };
beatmapLevel = _customLevelLoader.CreateBeatmapLevelFromV4(customLevelFolderInfo, beatmapLevelSaveData);
}

var hash = cached.SongHash;
var wip = songPath.Contains("CustomWIPLevels");

string levelID = CustomLevelLoader.kCustomLevelPrefixId + hash;
string folderName = directoryInfo.Name;
while (!Collections.LevelHashDictionary.TryAdd(levelID + (wip ? " WIP" : ""), hash))
{
levelID += $"_{folderName}";
}

if (wip)
{
levelID += " WIP";
}

Collections.HashLevelDictionary.AddOrUpdate(hash, new List<string> { levelID }, (_, levels) =>
{
lock (levels)
{
levels.Add(levelID);
}
return levels;
});

Accessors.LevelIDAccessor(ref beatmapLevel) = levelID;

if (cached.Duration > 0)
{
Accessors.SongDurationAccessor(ref beatmapLevel) = cached.Duration;
}
else
{
GetSongDuration(loadedSaveData, beatmapLevel);
}

LoadedBeatmapSaveData.TryAdd(levelID, loadedSaveData);

Hashing.TryGetRelativePath(songPath, out var cacheRelPath);
Hashing.cachedSongHashData[cacheRelPath] = new SongHashData(cached.DirTimestamp, hash);

if (cached.Duration > 0)
{
Hashing.cachedAudioData[cacheRelPath] = new AudioCacheData(levelID, cached.Duration);
}

if (!string.IsNullOrEmpty(cached.SongDataJson))
{
try
{
var songData = JsonConvert.DeserializeObject<SongData>(cached.SongDataJson);
if (songData != null)
{
Collections.CustomSongsData.TryAdd(levelID, songData);
}
}
catch { }
}

if (!Collections.CustomSongsData.ContainsKey(levelID))
{
Collections.CreateCustomLevelSongData(levelID, loadedSaveData);
}

return (hash, beatmapLevel);
}
catch (Exception ex)
{
Plugin.Log.Warn($"Cache reconstruction failed for '{songPath}': {ex.Message}");
return null;
}
}

public static (string hash, BeatmapLevel beatmapLevel)? LoadCustomLevel(string customLevelPath, SongFolderEntry? entry = null)
{
return Instance.LoadCustomLevelInternal(customLevelPath, entry);
Expand Down
Loading