diff --git a/samples/VersionDatabaseCreator/Program.cs b/samples/VersionDatabaseCreator/Program.cs index e747795..1803070 100644 --- a/samples/VersionDatabaseCreator/Program.cs +++ b/samples/VersionDatabaseCreator/Program.cs @@ -54,21 +54,21 @@ // It takes a couple of minutes to scan TWD:DE with a stock Ryzen 5 2600. var areSymbolsHashed = false; var msv = MetaStreamVersion.Mbin; -ArchiveVersion ttarchVersion = 0; +TTArchiveVersion ttarchVersion = 0; await Parallel.ForEachAsync(archivePaths, async (filePath, _) => { Console.WriteLine($"Start reading {filePath}."); try { - using ArchiveBase archive = Toolkit.Instance.LoadArchive(filePath, blowfishKey, false); + using Archive archive = Toolkit.Instance.LoadArchive(filePath, blowfishKey); ttarchVersion = archive.Info.Version; - if (archive.FileEntries.Length > 0) + if (archive.Entries.Values.Any()) { - MemoryStream firstFile = archive.ExtractFile(archive.FileEntries[0].Name); + Stream firstFile = archive.OpenResource(archive.Entries[0].Name); - if (Toolkit.IsMetaFile(archive.FileEntries[0].Name)) + if (Toolkit.IsMetaFile(archive.Entries[0].Name)) { MetaStreamConfiguration config = new MetaStreamReader(firstFile).Configuration; @@ -77,11 +77,11 @@ await Parallel.ForEachAsync(archivePaths, async (filePath, _) => } } - foreach (TelltaleFileEntry entry in archive.FileEntries) + foreach (ResourceEntry entry in archive.Entries.Values) { // Console.WriteLine($"Reading {entry.Name}"); - using MemoryStream file = archive.ExtractFile(entry.Name); + using Stream file = archive.OpenResource(entry.Name); if (!Toolkit.IsMetaFile(file)) continue; diff --git a/src/TelltaleToolKit/GamesDatabase/GameProfile.cs b/src/TelltaleToolKit/GamesDatabase/GameProfile.cs index 76adb83..fbffdf5 100644 --- a/src/TelltaleToolKit/GamesDatabase/GameProfile.cs +++ b/src/TelltaleToolKit/GamesDatabase/GameProfile.cs @@ -42,8 +42,8 @@ public sealed class GameProfile /// /// Gets the version of the TTARCH archive format used by the game. /// - /// - public ArchiveVersion TtarchVersion { get; set; } + /// + public TTArchiveVersion TtarchVersion { get; set; } /// /// Gets the Lua scripting engine version used by the game. diff --git a/src/TelltaleToolKit/Resource/ArchiveProvider.cs b/src/TelltaleToolKit/Resource/ArchiveProvider.cs index 1bb15b9..2fd208f 100644 --- a/src/TelltaleToolKit/Resource/ArchiveProvider.cs +++ b/src/TelltaleToolKit/Resource/ArchiveProvider.cs @@ -7,14 +7,14 @@ namespace TelltaleToolKit.Resource; /// Wraps a single Telltale archive (.ttarch or .ttarch2) as an . /// /// -/// The underlying is loaded through the workspace so that the +/// The underlying is loaded through the workspace so that the /// workspace's ArchiveLoaded event fires and blowfish decryption is applied /// consistently. Dispose this provider when the archive is no longer needed to release /// the file handle held by the archive. /// public sealed class ArchiveProvider : IFileProvider, IDisposable { - private readonly ArchiveBase _archive; + private readonly Archive _archive; /// /// Loads the archive at using the blowfish key embedded in @@ -45,18 +45,18 @@ public ArchiveProvider(string path, Workspace workspace) public string Path { get; } /// - /// Disposes the underlying , releasing any held file handles. + /// Disposes the underlying , releasing any held file handles. /// public void Dispose() => _archive.Dispose(); /// - public Stream? ExtractFile(ulong crc64) => _archive.ExtractFile(crc64); + public Stream? ExtractFile(ulong crc64) => _archive.OpenResource(crc64); /// - public bool ContainsFile(ulong crc64) => _archive.ContainsFile(crc64); + public bool ContainsFile(ulong crc64) => _archive.Entries.ContainsKey(crc64); /// - public TelltaleFileEntry? GetFileEntry(ulong crc64) => _archive.FindEntry(crc64); + public ResourceEntry? GetFileEntry(ulong crc64) => _archive.FindResource(crc64); /// public Stream? ExtractFile(string fileName) @@ -67,9 +67,9 @@ public bool ContainsFile(string fileName) => ContainsFile(Crc64.Compute(fileName)); /// - public TelltaleFileEntry? GetFileEntry(string fileName) + public ResourceEntry? GetFileEntry(string fileName) => GetFileEntry(Crc64.Compute(fileName)); /// - public IEnumerable GetAllEntries() => _archive.FileEntries; + public IEnumerable GetAllEntries() => _archive.Entries.Values; } diff --git a/src/TelltaleToolKit/Resource/FolderProvider.cs b/src/TelltaleToolKit/Resource/FolderProvider.cs index b2b95e5..d0e200f 100644 --- a/src/TelltaleToolKit/Resource/FolderProvider.cs +++ b/src/TelltaleToolKit/Resource/FolderProvider.cs @@ -104,7 +104,7 @@ public bool ContainsFile(ulong crc64) => _providers.Any(p => p.ContainsFile(crc64)); /// - public TelltaleFileEntry? GetFileEntry(ulong crc64) + public ResourceEntry? GetFileEntry(ulong crc64) { foreach (var provider in _providers) { @@ -123,11 +123,11 @@ public bool ContainsFile(string fileName) => ContainsFile(Crc64.Compute(fileName)); /// - public TelltaleFileEntry? GetFileEntry(string fileName) + public ResourceEntry? GetFileEntry(string fileName) => GetFileEntry(Crc64.Compute(fileName)); /// - public IEnumerable GetAllEntries() + public IEnumerable GetAllEntries() => _providers.SelectMany(p => p.GetAllEntries()); /// diff --git a/src/TelltaleToolKit/Resource/IFileProvider.cs b/src/TelltaleToolKit/Resource/IFileProvider.cs index a1fa793..ac82f3b 100644 --- a/src/TelltaleToolKit/Resource/IFileProvider.cs +++ b/src/TelltaleToolKit/Resource/IFileProvider.cs @@ -32,11 +32,11 @@ public interface IFileProvider bool ContainsFile(ulong crc64); /// - /// Returns the for the file whose CRC-64 matches + /// Returns the for the file whose CRC-64 matches /// , or if not found. /// /// Pre-computed CRC-64 of the filename. - TelltaleFileEntry? GetFileEntry(ulong crc64); + ResourceEntry? GetFileEntry(ulong crc64); /// /// Opens a read-only stream for the file with the given name. @@ -62,10 +62,10 @@ public interface IFileProvider /// /// The bare filename (e.g. obj_clementine.d3dtx), not a full path. /// - TelltaleFileEntry? GetFileEntry(string fileName); + ResourceEntry? GetFileEntry(string fileName); /// - /// Enumerates every accessible through this provider, + /// Enumerates every accessible through this provider, /// without duplicates. /// /// @@ -75,5 +75,5 @@ public interface IFileProvider /// child providers' entries. /// /// An enumerable of file entries; never . - public IEnumerable GetAllEntries(); + public IEnumerable GetAllEntries(); } diff --git a/src/TelltaleToolKit/Resource/LooseFileProvider.cs b/src/TelltaleToolKit/Resource/LooseFileProvider.cs index 540c97e..2a63031 100644 --- a/src/TelltaleToolKit/Resource/LooseFileProvider.cs +++ b/src/TelltaleToolKit/Resource/LooseFileProvider.cs @@ -50,13 +50,13 @@ public bool ContainsFile(ulong crc64) => crc64 == Crc64; /// - public TelltaleFileEntry? GetFileEntry(ulong crc64) + public ResourceEntry? GetFileEntry(ulong crc64) { if (crc64 != Crc64) return null; - return new TelltaleFileEntry + return new ResourceEntry { - Crc64 = crc64, Name = Name, FileOffset = 0, FileSize = (int)Size, + NameCrc = crc64, Name = Name, Offset = 0, Size = (uint)Size, }; } @@ -69,15 +69,15 @@ public bool ContainsFile(string fileName) => IsMatch(fileName); /// - public TelltaleFileEntry? GetFileEntry(string fileName) + public ResourceEntry? GetFileEntry(string fileName) => IsMatch(fileName) ? GetFileEntry(Crc64) : null; /// - public IEnumerable GetAllEntries() + public IEnumerable GetAllEntries() { - yield return new TelltaleFileEntry + yield return new ResourceEntry { - Crc64 = Crc64, Name = Name, FileOffset = 0, FileSize = (int)Size, + NameCrc = Crc64, Name = Name, Offset = 0, Size = (uint)Size, }; } diff --git a/src/TelltaleToolKit/Resource/ResourceContext.cs b/src/TelltaleToolKit/Resource/ResourceContext.cs index bfac6c2..e1f3102 100644 --- a/src/TelltaleToolKit/Resource/ResourceContext.cs +++ b/src/TelltaleToolKit/Resource/ResourceContext.cs @@ -122,13 +122,13 @@ public bool ContainsFile(ulong crc64) /// /// Returns immediately when the context is disabled. - public TelltaleFileEntry? GetFileEntry(ulong crc64) + public ResourceEntry? GetFileEntry(ulong crc64) { if (!IsEnabled) return null; foreach (IFileProvider provider in _providers) { - TelltaleFileEntry? entry = provider.GetFileEntry(crc64); + ResourceEntry? entry = provider.GetFileEntry(crc64); if (entry != null) return entry; } @@ -144,10 +144,10 @@ public bool ContainsFile(string fileName) => ContainsFile(Crc64.Compute(fileName)); /// - public TelltaleFileEntry? GetFileEntry(string fileName) + public ResourceEntry? GetFileEntry(string fileName) => GetFileEntry(Crc64.Compute(fileName)); /// - public IEnumerable GetAllEntries() + public IEnumerable GetAllEntries() => _providers.SelectMany(p => p.GetAllEntries()); } diff --git a/src/TelltaleToolKit/Serialization/Binary/MetaStreamReader.cs b/src/TelltaleToolKit/Serialization/Binary/MetaStreamReader.cs index 078a03e..5e5e520 100644 --- a/src/TelltaleToolKit/Serialization/Binary/MetaStreamReader.cs +++ b/src/TelltaleToolKit/Serialization/Binary/MetaStreamReader.cs @@ -53,6 +53,9 @@ public override void SerializeMetaHeader() // Read the number of serialized classes uint metaClassNum = this.ReadUInt32(); + if (metaClassNum >= 1000) + throw new InvalidDataException("Too many classes in the meta stream."); + for (var i = 0; i < metaClassNum; i++) { uint size = this.ReadUInt32(); @@ -94,10 +97,10 @@ public override void SerializeMetaHeader() long currentPosition = Reader.BaseStream.Position; if (Configuration.Version is MetaStreamVersion.Msv6 or MetaStreamVersion.Msv5) { - Sections[(int)SectionType.Main].Stream = new Substream(UnderlyingStream, currentPosition, mainSize); + Sections[(int)SectionType.Main].Stream = new SubStream(UnderlyingStream, currentPosition, mainSize); Sections[(int)SectionType.Debug].Stream = - new Substream(UnderlyingStream, currentPosition + mainSize, debugSize); - Sections[(int)SectionType.Async].Stream = new Substream( + new SubStream(UnderlyingStream, currentPosition + mainSize, debugSize); + Sections[(int)SectionType.Async].Stream = new SubStream( UnderlyingStream, currentPosition + mainSize + debugSize, asyncSize @@ -106,13 +109,13 @@ public override void SerializeMetaHeader() else { // For older versions, the entire stream is the main section - Sections[(int)SectionType.Main].Stream = new Substream( + Sections[(int)SectionType.Main].Stream = new SubStream( UnderlyingStream, currentPosition, UnderlyingStream.Length - currentPosition ); - Sections[(int)SectionType.Debug].Stream = new Substream(UnderlyingStream, 0, 0); - Sections[(int)SectionType.Async].Stream = new Substream(UnderlyingStream, 0, 0); + Sections[(int)SectionType.Debug].Stream = new SubStream(UnderlyingStream, 0, 0); + Sections[(int)SectionType.Async].Stream = new SubStream(UnderlyingStream, 0, 0); } InitSerializer(); diff --git a/src/TelltaleToolKit/Serialization/Binary/StreamExtensions.cs b/src/TelltaleToolKit/Serialization/Binary/StreamExtensions.cs index 2b2f936..40c59e5 100644 --- a/src/TelltaleToolKit/Serialization/Binary/StreamExtensions.cs +++ b/src/TelltaleToolKit/Serialization/Binary/StreamExtensions.cs @@ -12,6 +12,6 @@ public static class StreamExtensions /// The offset. /// The length. /// A substream. - public static Substream Substream(this Stream stream, long offset, long length) => - new Substream(stream, offset, length); + public static SubStream Substream(this Stream stream, long offset, long length) => + new SubStream(stream, offset, length); } \ No newline at end of file diff --git a/src/TelltaleToolKit/Serialization/Binary/Substream.cs b/src/TelltaleToolKit/Serialization/Binary/Substream.cs index 1e3d626..b96cbf3 100644 --- a/src/TelltaleToolKit/Serialization/Binary/Substream.cs +++ b/src/TelltaleToolKit/Serialization/Binary/Substream.cs @@ -3,7 +3,7 @@ /// /// Represents a substream of an underlying . /// -public class Substream : Stream +public class SubStream : Stream { /// /// Creates a new substream instance using the specified underlying stream at the specified offset with the specified length. @@ -11,7 +11,7 @@ public class Substream : Stream /// The underlying stream. /// The offset. /// The length. - public Substream(Stream stream, long offset, long length) + public SubStream(Stream stream, long offset, long length) { if (stream == null) { diff --git a/src/TelltaleToolKit/TelltaleArchives/Archive.cs b/src/TelltaleToolKit/TelltaleArchives/Archive.cs new file mode 100644 index 0000000..466bb8d --- /dev/null +++ b/src/TelltaleToolKit/TelltaleArchives/Archive.cs @@ -0,0 +1,408 @@ +using System.IO.Enumeration; +using TelltaleToolKit.T3Types; +using TelltaleToolKit.TelltaleArchives.Formats; +using TelltaleToolKit.Utility.Blowfish; +using TelltaleToolKit.Utility.Hashing; + +namespace TelltaleToolKit.TelltaleArchives; + +/// +/// Base class for all Telltale archive formats (.ttarch, .ttarch2, PK2). +/// +/// +/// +/// Subclasses implement to parse their specific header format and +/// populate the entry table via . Extraction is handled by +/// , which each subclass overrides to return an +/// appropriately positioned stream into the archive data. +/// +/// +/// Entries are keyed by CRC-64 hash of their filename. Use +/// for name-based lookups, or when the hash is already known. +/// +/// +/// Archives are opened via the static family of methods +/// and created via +/// +/// or the CreateFrom* convenience overloads. +/// +/// +/// Implements . Always dispose the archive (or use a using +/// block) to release the underlying stream. +/// +/// +public abstract class Archive : IDisposable +{ + private readonly Dictionary _entries = new(); + + // The raw underlying stream for the archive file. Subclasses read from this directly. + protected Stream? BaseStream { get; private set; } + + /// + /// Gets the original file path of the archive, if it was loaded from a file. + /// when the archive was opened from a . + /// + public string? FilePath { get; private set; } + + /// + /// Gets the length of the underlying stream in bytes, if the stream supports seeking. + /// when the stream is not seekable (e.g., a network stream). + /// + public long? Length => BaseStream?.CanSeek == true ? BaseStream.Length : null; + + /// + /// Gets the file name (without directory) of the archive, if a file path is available. + /// when is . + /// + public string? FileName => FilePath is not null ? Path.GetFileName(FilePath) : null; + + /// + /// Gets or sets archive metadata, including version, flags, Blowfish key, and chunk layout. + /// Populated during . + /// + public ArchiveInfo Info { get; set; } = new(); + + /// + /// Gets a read-only view of all entries loaded from the archive, keyed by CRC-64 filename hash. + /// + public IReadOnlyDictionary Entries => _entries; + + /// Number of entries in the archive. + public int Count => _entries.Count; + + // ------------------------------------------------------------------------- + // IDisposable + // ------------------------------------------------------------------------- + + public virtual void Dispose() + { + BaseStream?.Dispose(); + GC.SuppressFinalize(this); + } + + /// Enumerates all entries. + public IEnumerable GetAllEntries() => _entries.Values; + + // ------------------------------------------------------------------------- + // Loading + // ------------------------------------------------------------------------- + + /// + /// Opens and parses an archive from a file path using a raw Blowfish key string. + /// + /// The concrete archive type to instantiate (e.g. ). + /// Path to the archive file on disk. + /// Raw Blowfish key string. Pass for unencrypted archives. + /// A fully loaded archive instance. + /// Thrown if does not exist. + /// Thrown if the archive header is corrupt or unrecognised. + public static T Load(string filePath, string blowFishKey) where T : Archive, new() + { + FileStream stream = File.OpenRead(filePath); + T archive = Load(stream, blowFishKey); + archive.FilePath = filePath; + return archive; + } + + /// + /// Opens and parses an archive from a stream using a raw Blowfish key string. + /// + /// The concrete archive type to instantiate. + /// + /// A readable stream positioned at the start of the archive. Must remain open for the lifetime of the + /// returned archive. + /// + /// Raw Blowfish key string. Pass for unencrypted archives. + /// A fully loaded archive instance. + /// Thrown if is . + /// Thrown if the archive header is corrupt or unrecognized. + public static T Load(Stream stream, string blowFishKey) where T : Archive, new() + { + T archive = new(); + archive.BaseStream = stream ?? throw new ArgumentNullException(nameof(stream)); + archive.BaseStream.Position = 0; + archive.Info.BlowfishKey = blowFishKey; + archive.Activate(); + return archive; + } + + /// + /// Opens and parses an archive from a file path using a well-known game Blowfish key. + /// + /// The concrete archive type to instantiate. + /// Path to the archive file on disk. + /// The game key enum value whose Blowfish bytes will be used. + /// A fully loaded archive instance. + public static T Load(string filePath, T3BlowfishKey gameKey) where T : Archive, new() + => Load(filePath, gameKey.GetBlowfishKey()); + + /// + /// Opens and parses an archive from a stream using a well-known game Blowfish key. + /// + /// The concrete archive type to instantiate. + /// A readable stream positioned at the start of the archive. + /// The game key enum value whose Blowfish bytes will be used. + /// A fully loaded archive instance. + public static T Load(Stream stream, T3BlowfishKey gameKey) where T : Archive, new() + => Load(stream, gameKey.GetBlowfishKey()); + + /// + /// Parses the archive header from and populates the entry table. + /// Called exactly once, immediately after the stream and are initialised. + /// + protected abstract void Activate(); + + // ------------------------------------------------------------------------- + // Entry lookup + // ------------------------------------------------------------------------- + + /// Returns if an entry with the given CRC-64 hash exists. + public bool HasResource(ulong crc64) => _entries.ContainsKey(crc64); + + /// Returns if an entry with the given filename exists (CRC-64 computed internally). + public bool HasResource(string name) => HasResource(Crc64.Compute(name)); + + /// Returns if an entry matching the given exists. + public bool HasResource(Symbol symbol) => HasResource(symbol.Crc64); + + /// Finds an entry by CRC-64 hash, or returns if not found. + public ResourceEntry? FindResource(ulong crc64) => _entries.GetValueOrDefault(crc64); + + /// Finds an entry by filename (CRC-64 computed internally), or returns if not found. + public ResourceEntry? FindResource(string name) => FindResource(Crc64.Compute(name)); + + /// Finds an entry by , or returns if not found. + public ResourceEntry? FindResource(Symbol symbol) => FindResource(symbol.Crc64); + + /// + /// Returns all entries whose filenames match a simple glob pattern (e.g. "*.d3dtx"). + /// + /// + /// A glob expression passed to . + /// Supports * (any characters) and ? (single character) wildcards. + /// + public IEnumerable FindResourcesByMask(string pattern) + => GetAllEntries().Where(e => FileSystemName.MatchesSimpleExpression(pattern, e.Name)); + + /// + /// Returns the filenames of all entries matching a simple glob pattern. + /// + /// A glob expression (see ). + public IEnumerable FindNamesByMask(string pattern) + => FindResourcesByMask(pattern).Select(e => e.Name); + + // ------------------------------------------------------------------------- + // Entry table management (used by subclasses) + // ------------------------------------------------------------------------- + + /// + /// Replaces the entire entry map with the given collection. + /// Must be called by subclasses at the end of after all entries are parsed. + /// + /// The parsed entries to register. Duplicate CRC-64 keys overwrite earlier values. + protected void SetEntries(IEnumerable entries) + { + _entries.Clear(); + foreach (ResourceEntry e in entries) + { + _entries[e.NameCrc] = e; + } + } + + // ------------------------------------------------------------------------- + // Extraction + // ------------------------------------------------------------------------- + + /// + /// Opens a stream over a resource's data by filename. + /// Returns if no entry with that name exists. + /// + public Stream? OpenResource(string name) => OpenResource(FindResource(name)); + + /// + /// Opens a stream over a resource's data by CRC-64 hash. + /// Returns if no matching entry exists. + /// + public Stream? OpenResource(ulong crc64) => OpenResource(FindResource(crc64)); + + /// + /// Opens a stream over a resource's data by . + /// Returns if no matching entry exists. + /// + public Stream? OpenResource(Symbol symbol) => OpenResource(FindResource(symbol)); + + /// Extracts a file from the archive given its entry record. + protected abstract Stream? OpenResource(ResourceEntry? entry); + + /// + /// Extracts all entries to , preserving any directory + /// structure encoded in entry names. Entries are extracted in ascending offset order to + /// minimise seek distance on the underlying stream. + /// + /// + /// Root folder to write into. Created (including parents) if it does not already exist. + /// + public virtual void ExtractAll(string destinationPath) + { + Directory.CreateDirectory(destinationPath); + + foreach (ResourceEntry entry in _entries.Values.OrderBy(e => e.Offset)) + { + string fullPath = Path.Combine(destinationPath, entry.Name); + string? dir = Path.GetDirectoryName(fullPath); + if (dir is not null) + { + Directory.CreateDirectory(dir); + } + + using Stream? data = OpenResource(entry.NameCrc); + if (data is null) + { + Toolkit.Instance.Logger.LogWarning($"[Archive] Skipping '{entry.Name}' — extraction returned null."); + continue; + } + + using FileStream output = File.Create(fullPath); + data.Position = 0; + data.CopyTo(output); + } + } + + // ------------------------------------------------------------------------- + // Encryption / compression helpers + // ------------------------------------------------------------------------- + + /// + /// Returns if the archive uses standard or modified Blowfish encryption. + /// + protected bool IsEncrypted() => Info.Flags.IsEncrypted(); + + + /// + /// Returns if the archive uses any form of chunk compression + /// (zlib, raw deflate, or Oodle). + /// + protected bool IsCompressed() => Info.Flags.IsCompressed(); + + // ------------------------------------------------------------------------- + // Creation (static helpers) + // ------------------------------------------------------------------------- + + /// + /// Creates a new archive from a sequence of named streams and writes it to . + /// + /// + /// The archive type to write. Must be or . + /// + /// Writable stream to receive the archive bytes. Left open after the call. + /// + /// Sequence of (entry name, data stream) pairs. Data streams are read from their current + /// position. + /// + /// Controls compression, encryption, chunk size, and archive version. + /// Thrown if is not a supported type. + public static void Create( + Stream output, + IEnumerable<(string name, Stream dataStream)> entries, + ArchiveWriteOptions options) + where TArchive : Archive, new() + { + if (typeof(TArchive) == typeof(TTArchive)) + { + TTArchive.Create(output, entries, options); + } + else if (typeof(TArchive) == typeof(TTArchive2)) + { + TTArchive2.Create(output, entries, options); + } + else + { + throw new NotSupportedException($"Archive type {typeof(TArchive).Name} does not support creation."); + } + } + + /// + /// Creates a new archive from a sequence of named streams and writes it to a file at . + /// + /// The archive type to write. + /// Destination file path. Created or overwritten. + /// Sequence of (entry name, data stream) pairs. + /// Controls compression, encryption, chunk size, and archive version. + public static void Create( + string outputPath, + IEnumerable<(string name, Stream dataStream)> entries, + ArchiveWriteOptions options) + where TArchive : Archive, new() + { + using FileStream fs = new(outputPath, FileMode.Create, FileAccess.Write); + Create(fs, entries, options); + } + + /// + /// Creates a new archive from all files in the top level of , + /// using each file's name (without directory) as the entry name. + /// + /// The archive type to write. + /// Folder to scan. Only the top-level directory is searched. + /// Destination file path. Created or overwritten. + /// Controls compression, encryption, chunk size, and archive version. + /// Optional file glob (e.g. "*.d3dtx"). Defaults to "*". + public static void CreateFromFolder( + string folderPath, + string outputPath, + ArchiveWriteOptions options, + string searchPattern = "*") + where TArchive : Archive, new() + { + List<(string name, Stream stream)> entries = new(); + try + { + foreach (string filePath in Directory.GetFiles(folderPath, searchPattern, SearchOption.TopDirectoryOnly)) + { + entries.Add((Path.GetFileName(filePath), File.OpenRead(filePath))); + } + + Create(outputPath, entries, options); + } + finally + { + foreach ((string _, Stream stream) in entries) + { + stream.Dispose(); + } + } + } + + /// + /// Creates a new archive from an explicit list of file paths, using each file's name + /// (without directory) as the entry name. + /// + /// The archive type to write. + /// Destination file path. Created or overwritten. + /// Full paths of the files to include. + /// Controls compression, encryption, chunk size, and archive version. + public static void CreateFromFiles( + string outputPath, + IEnumerable filePaths, + ArchiveWriteOptions options) + where TArchive : Archive, new() + { + List<(string name, Stream stream)> entries = new(); + try + { + foreach (string filePath in filePaths) + { + entries.Add((Path.GetFileName(filePath), File.OpenRead(filePath))); + } + + Create(outputPath, entries, options); + } + finally + { + foreach ((string _, Stream stream) in entries) + { + stream.Dispose(); + } + } + } +} diff --git a/src/TelltaleToolKit/TelltaleArchives/ArchiveFlags.cs b/src/TelltaleToolKit/TelltaleArchives/ArchiveFlags.cs new file mode 100644 index 0000000..a677189 --- /dev/null +++ b/src/TelltaleToolKit/TelltaleArchives/ArchiveFlags.cs @@ -0,0 +1,53 @@ +namespace TelltaleToolKit.TelltaleArchives; + +/// +/// Flags that describe the encryption and compression state of a Telltale archive. +/// +[Flags] +public enum ArchiveFlags +{ + /// No special flags. + None = 0, + + /// Archive is encrypted with Blowfish (standard variant). + IsEncrypted = 1 << 1, + + /// Archive is compressed using zlib (wrapped DEFLATE). + IsZlibCompressed = 1 << 3, + + /// Archive is compressed using raw DEFLATE (no zlib header). + IsRawDeflateCompressed = 1 << 4, + + /// Archive is compressed using Oodle (not yet supported). + IsOodleCompressed = 1 << 5, + + /// Archive uses a modified Blowfish variant. That's all .ttarch2 and version >= 7 .ttarch files. + IsModifiedBlowfishEncrypted = 1 << 6, + + /// Archive uses "XMode" – an unknown flag seen in some version >= 5 archives. + IsXMode = 1 << 7 +} + +/// +/// Extension methods for to simplify common checks. +/// +public static class ArchiveFlagsExtensions +{ + /// + /// Indicates whether the archive uses any form of compression. + /// + /// The flags value. + /// True if compressed (raw DEFLATE, zlib, or Oodle). + public static bool IsCompressed(this ArchiveFlags value) + => value.HasFlag(ArchiveFlags.IsRawDeflateCompressed) || value.HasFlag(ArchiveFlags.IsZlibCompressed) || + value.HasFlag(ArchiveFlags.IsOodleCompressed); + + /// + /// Indicates whether the archive is encrypted (standard Blowfish). + /// + /// The flags value. + /// True if encrypted. + /// Does not detect . + public static bool IsEncrypted(this ArchiveFlags value) + => value.HasFlag(ArchiveFlags.IsEncrypted) || value.HasFlag(ArchiveFlags.IsModifiedBlowfishEncrypted); +} diff --git a/src/TelltaleToolKit/TelltaleArchives/ArchiveInfo.cs b/src/TelltaleToolKit/TelltaleArchives/ArchiveInfo.cs index 79031e8..cde8b59 100644 --- a/src/TelltaleToolKit/TelltaleArchives/ArchiveInfo.cs +++ b/src/TelltaleToolKit/TelltaleArchives/ArchiveInfo.cs @@ -1,20 +1,49 @@ namespace TelltaleToolKit.TelltaleArchives; +/// +/// Contains metadata about a Telltale archive, including its container type, +/// version, flags, chunk layout, encryption key, and file data offset. +/// +/// +/// This information is populated during archive loading and used by +/// subclasses to correctly read entries and data. +/// public class ArchiveInfo { - public ContainerVersion ContainerVersion { get; set; } - public ArchiveVersion Version { get; set; } - public ContainerFlags Flags { get; set; } - public int ChunkSize { get; set; } + /// + /// Gets or sets the inner archive version (Legacy, 1-9, TTA2, TTA3, TTA4). + /// + public TTArchiveVersion Version { get; set; } + + /// + /// Gets or sets flags indicating encryption, compression, and other properties. + /// + public ArchiveFlags Flags { get; set; } + + /// + /// Gets or sets the decompressed size of each chunk in bytes (0 if uncompressed). + /// + public uint ChunkSize { get; set; } + + /// + /// Gets or sets the number of compressed chunks (0 if uncompressed). + /// public uint ChunkCount { get; set; } - public long[] ChunkBlockSizes { get; set; } = []; /// - /// The Blowfish key used for encryption/decryption. + /// Gets or sets an array of compressed sizes for each chunk. + /// Length equals ; empty if uncompressed. + /// + public ulong[] ChunkBlockSizes { get; set; } = []; + + /// + /// Gets or sets the Blowfish key used for encryption/decryption. + /// Must match the key used by the original game. /// public string BlowfishKey { get; set; } = string.Empty; - public long FilesOffset { get; set; } - public long ArchiveOffset { get; set; } // This is for TTA2, TTA3, TTA4 - public uint FileCount { get; set; } -} \ No newline at end of file + /// + /// Gets or sets the byte offset (from the start of the archive stream) where the raw file data region begins. + /// + public ulong FilesOffset { get; set; } +} diff --git a/src/TelltaleToolKit/TelltaleArchives/ArchiveVersion.cs b/src/TelltaleToolKit/TelltaleArchives/ArchiveVersion.cs deleted file mode 100644 index 6a567be..0000000 --- a/src/TelltaleToolKit/TelltaleArchives/ArchiveVersion.cs +++ /dev/null @@ -1,19 +0,0 @@ -namespace TelltaleToolKit.TelltaleArchives; - -public enum ArchiveVersion -{ - Zero = 0, - One, - Two, - Three, - Four, - Five, - Six, - Seven, - Eight, - Nine, - - Tta2 = 0x54544132, // "TTA2" - Tta3 = 0x54544133, // "TTA3" - Tta4 = 0x54544134, // "TTA4" -} \ No newline at end of file diff --git a/src/TelltaleToolKit/TelltaleArchives/ArchiveWriteOptions.cs b/src/TelltaleToolKit/TelltaleArchives/ArchiveWriteOptions.cs new file mode 100644 index 0000000..01f9ff8 --- /dev/null +++ b/src/TelltaleToolKit/TelltaleArchives/ArchiveWriteOptions.cs @@ -0,0 +1,69 @@ +namespace TelltaleToolKit.TelltaleArchives; + +/// +/// Controls how an archive is written or repacked. +/// +public sealed class ArchiveWriteOptions +{ + /// + /// Gets or sets the inner archive version to embed in the payload header. + /// Defaults to . + /// + public TTArchiveVersion TTArchiveVersion { get; set; } = TTArchiveVersion.Tta4; + + /// + /// Gets or sets whether Blowfish encryption is applied to each chunk. + /// + /// When , must be non-empty. + /// Ignored for container versions that encode encryption in the version magic + /// (e.g. ); the magic takes precedence. + /// + /// Defaults to . + /// + public bool Encrypt { get; set; } + + /// + /// Gets or sets the compression algorithm to use. + /// Defaults to . + /// + public CompressionAlgorithm Algorithm { get; set; } = CompressionAlgorithm.Deflate; + + /// + /// Gets or sets the decompressed size of each chunk in bytes. + /// The value is rounded up to the nearest multiple of 1 024 when writing + /// .ttarch (version ≥ 7) archives. + /// Defaults to 65 536 (64 KiB). + /// + public uint ChunkSize { get; set; } = 65_536; + + /// + /// Gets or sets a value indicating whether to set the XMode. + /// Set this to if the archive crashes when loading in the game. + /// + public bool XMode { get; set; } = false; + + /// + /// Gets or sets the Blowfish key used when is + /// . + /// Ignored when the archive is not encrypted. + /// + public string BlowfishKey { get; set; } = string.Empty; +} + +/// Compression algorithm to use when writing archive chunks. +public enum CompressionAlgorithm +{ + /// No compression. + None, + + /// Raw DEFLATE (no zlib header). Used by most Telltale games. + Deflate, + + /// Zlib-wrapped DEFLATE. Used by some CSI and earlier titles. + Zlib, + + /// + /// Oodle compression. + /// + Oodle +} diff --git a/src/TelltaleToolKit/TelltaleArchives/Caching/ChunkCache.cs b/src/TelltaleToolKit/TelltaleArchives/Caching/ChunkCache.cs new file mode 100644 index 0000000..ffb1546 --- /dev/null +++ b/src/TelltaleToolKit/TelltaleArchives/Caching/ChunkCache.cs @@ -0,0 +1,28 @@ +namespace TelltaleToolKit.TelltaleArchives.Caching; + +/// +/// Defines the contract for a chunk-level cache. +/// +/// +/// Implementations differ in memory usage and access-pattern suitability: +/// +/// — one chunk hot at a time; ideal for linear scans. +/// — N most-recently-used chunks; best for clustered random access. +/// +/// +public interface IChunkCache +{ + /// + /// Returns the decoded bytes for chunk , decoding + /// if not already cached. The caller must not mutate the returned array. + /// + /// Zero-based index into the archive's chunk table. + /// + /// Factory that performs the actual decompress+decrypt for the chunk. + /// Called only on a cache miss. + /// + byte[] GetOrDecode(int chunkIndex, Func decode); + + /// Discards all cached state. Called between unrelated extraction operations. + void Clear(); +} diff --git a/src/TelltaleToolKit/TelltaleArchives/Caching/LruChunkCache.cs b/src/TelltaleToolKit/TelltaleArchives/Caching/LruChunkCache.cs new file mode 100644 index 0000000..5785a1d --- /dev/null +++ b/src/TelltaleToolKit/TelltaleArchives/Caching/LruChunkCache.cs @@ -0,0 +1,69 @@ +namespace TelltaleToolKit.TelltaleArchives.Caching; + +/// +/// Cache strategy that retains the N most-recently-used decoded chunks. +/// +/// +/// +/// Memory overhead: up to N × ChunkSize bytes (e.g., N=8, ChunkSize=64 KiB → 512 KiB). +/// +/// +/// Not thread-safe. Each concurrent extraction path should use its own instance. +/// +/// +public sealed class LruChunkCache : IChunkCache +{ + private readonly int _capacity; + + // Doubly linked list of (chunkIndex, data), most-recent at front. + private readonly LinkedList<(int index, byte[] data)> _list = new(); + private readonly Dictionary> _map = new(); + + /// + /// Initializes a new with the given capacity. + /// + /// + /// Maximum number of decoded chunks to retain simultaneously. + /// Typical values: 4 (conservative), 8 (balanced), 16 (aggressive). + /// Must be ≥ 1. + /// + public LruChunkCache(int capacity = 8) => + // ArgumentOutOfRangeException.ThrowIfLessThan(capacity, 1); + _capacity = capacity; + + /// + public byte[] GetOrDecode(int chunkIndex, Func decode) + { + // Cache hit — move to front (most recently used) and return. + if (_map.TryGetValue(chunkIndex, out LinkedListNode<(int, byte[] data)>? node)) + { + _list.Remove(node); + _list.AddFirst(node); + return node.Value.data; + } + + // Cache miss — decode the chunk. + byte[] data = decode(chunkIndex); + + // Evict the least-recently-used entry if at capacity. + if (_list.Count == _capacity) + { + LinkedListNode<(int index, byte[] data)> lru = _list.Last!; + _map.Remove(lru.Value.index); + _list.RemoveLast(); + } + + // Insert at front. + LinkedListNode<(int, byte[])> newNode = _list.AddFirst((chunkIndex, data)); + _map[chunkIndex] = newNode; + + return data; + } + + /// + public void Clear() + { + _list.Clear(); + _map.Clear(); + } +} diff --git a/src/TelltaleToolKit/TelltaleArchives/Caching/SinglePageCache.cs b/src/TelltaleToolKit/TelltaleArchives/Caching/SinglePageCache.cs new file mode 100644 index 0000000..9c6a0e8 --- /dev/null +++ b/src/TelltaleToolKit/TelltaleArchives/Caching/SinglePageCache.cs @@ -0,0 +1,46 @@ +namespace TelltaleToolKit.TelltaleArchives.Caching; + +/// +/// Cache strategy that keeps exactly one decoded chunk in memory at a time. +/// +/// +/// +/// Memory overhead: one chunk (~64 KiB constant). +/// +/// +/// Best for: sequential / linear access patterns such. +/// +/// +/// Worst for: random or backward access — any access to a different chunk evicts +/// the current one, potentially causing re-decodes for clustered small files that +/// straddle two chunks. +/// +/// +/// Not thread-safe. Each concurrent extraction path should use its own instance. +/// +/// +public sealed class SinglePageCache : IChunkCache +{ + private byte[] _cachedData = []; + private int _cachedIndex = -1; + + /// + public byte[] GetOrDecode(int chunkIndex, Func decode) + { + if (_cachedIndex == chunkIndex) + { + return _cachedData; + } + + _cachedData = decode(chunkIndex); + _cachedIndex = chunkIndex; + return _cachedData; + } + + /// + public void Clear() + { + _cachedIndex = -1; + _cachedData = []; + } +} diff --git a/src/TelltaleToolKit/TelltaleArchives/ContainerFlags.cs b/src/TelltaleToolKit/TelltaleArchives/ContainerFlags.cs deleted file mode 100644 index 851633e..0000000 --- a/src/TelltaleToolKit/TelltaleArchives/ContainerFlags.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace TelltaleToolKit.TelltaleArchives; - -[Flags] -public enum ContainerFlags -{ - None = 0, - IsEncrypted = 1 << 1, // If the archive is Blowfish encrypted - IsZlibCompressed = 1 << 3, // If the archive is compressed with zlib - IsRawDeflateCompressed = 1 << 4, // If the archive is compressed with raw deflate (zlib header stripped) - IsOodleCompressed = 1 << 5, // If the archive is compressed with Oodle - IsModifiedBlowfishEncrypted = 1 << 6, // If the archive is encrypted with a weird Blowfish key - IsXMode = 1 << 7, -} \ No newline at end of file diff --git a/src/TelltaleToolKit/TelltaleArchives/ContainerMagic.cs b/src/TelltaleToolKit/TelltaleArchives/ContainerMagic.cs new file mode 100644 index 0000000..651a467 --- /dev/null +++ b/src/TelltaleToolKit/TelltaleArchives/ContainerMagic.cs @@ -0,0 +1,36 @@ +namespace TelltaleToolKit.TelltaleArchives; + +/// +/// Four‑byte magic numbers identifying the container format of a file. +/// +/// +/// The magic determines whether the archive is compressed, encrypted, +/// and which compression algorithm (zlib or Oodle) is used. +/// +public enum ContainerMagic +{ + /// + /// "TTCN" – No compression, no encryption. Plain container. + /// + TTCN = 0x5454434E, + + /// + /// "TTCE" – Encrypted container with raw DEFLATE compression. + /// + TTCE = 0x54544345, + + /// + /// "TTCe" – Encrypted container supporting Oodle compression. + /// + TTCe = 0x54544365, + + /// + /// "TTCZ" – Zlib‑compressed container (raw DEFLATE), no encryption. + /// + TTCZ = 0x5454435A, + + /// + /// "TTCz" – Zlib‑compressed container (raw DEFLATE) supporting Oodle compression. + /// + TTCz = 0x5454437A +} diff --git a/src/TelltaleToolKit/TelltaleArchives/ContainerVersion.cs b/src/TelltaleToolKit/TelltaleArchives/ContainerVersion.cs deleted file mode 100644 index d50e38c..0000000 --- a/src/TelltaleToolKit/TelltaleArchives/ContainerVersion.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace TelltaleToolKit.TelltaleArchives; - -public enum ContainerVersion -{ - Ttcn = 0x5454434E, // TTCN (NCTT) Telltale Tool Container Normal? Non-compressed? - Ttce = 0x54544345, // TTCE (ECTT) Telltale Tool Container Encrypted - TtCe = 0x54544365, // TTCe (eCTT) Telltale Tool Container Encrypted (supports Oodle compression) - Ttcz = 0x5454435A, // Telltale Tool Container Zlib - TtCz = 0x5454437A, // Telltale Tool Container Zlib (supports Oodle compression) -} \ No newline at end of file diff --git a/src/TelltaleToolKit/TelltaleArchives/Formats/PK2.cs b/src/TelltaleToolKit/TelltaleArchives/Formats/PK2.cs new file mode 100644 index 0000000..21d0156 --- /dev/null +++ b/src/TelltaleToolKit/TelltaleArchives/Formats/PK2.cs @@ -0,0 +1,9 @@ +namespace TelltaleToolKit.TelltaleArchives.Formats; + +// TODO: PK2 for CSI. +public class Pk2TTArchive : Archive +{ + protected override void Activate() => throw new NotImplementedException(); + + protected override Stream? OpenResource(ResourceEntry? entry) => throw new NotImplementedException(); +} diff --git a/src/TelltaleToolKit/TelltaleArchives/Formats/TTArchive.cs b/src/TelltaleToolKit/TelltaleArchives/Formats/TTArchive.cs new file mode 100644 index 0000000..7c1e53e --- /dev/null +++ b/src/TelltaleToolKit/TelltaleArchives/Formats/TTArchive.cs @@ -0,0 +1,688 @@ +using System.IO.Compression; +using System.IO.Hashing; +using System.Text; +using ICSharpCode.SharpZipLib.Zip.Compression.Streams; +using TelltaleToolKit.Serialization.Binary; +using TelltaleToolKit.TelltaleArchives.IO; +using TelltaleToolKit.Utility.Blowfish; +using Crc64 = TelltaleToolKit.Utility.Hashing.Crc64; + +namespace TelltaleToolKit.TelltaleArchives.Formats; + +/// +/// Reader for the original .ttarch archive format (versions 0–9). +/// +/// +/// All .ttarch implementations are based on ttarchext and TTG Tools research. +/// +public class TTArchive : Archive +{ + private Blowfish _blowfish; + // ------------------------------------------------------------------------- + // Legacy stream (version 0 / pre-versioned encrypted archives) + // ------------------------------------------------------------------------- + + private Stream? _dataStream; // decompressed view of the entire file-data region + + protected override void Activate() + { + using BinaryReader reader = new(BaseStream!, Encoding.UTF8, true); + + Info.Version = (TTArchiveVersion)reader.ReadUInt32(); + int version = (int)Info.Version; + + // Anything outside 1–9 falls back to the legacy reader + // (version 0, or a number so large it wrapped - both indicate the old format). + // The only issue with this is the folder number - if the folders are between 1 and 9, this fails. + if (version is < 1 or > 9) + { + ReadLegacyStream(reader); + return; + } + + _blowfish = new Blowfish(Info.BlowfishKey, (int)Info.Version); + + // ---- Version 1+ header ---- + uint decryptionMode = reader.ReadUInt32(); + if (decryptionMode > 1) + { + Toolkit.Instance.Logger.LogWarning( + $"Decryption mode {decryptionMode} is not supported. Falling back to encrypted."); + } + + Info.Flags |= decryptionMode == 1 ? ArchiveFlags.IsEncrypted : ArchiveFlags.None; + + if (version >= 2) + { + _ = reader.ReadUInt32(); // unknown field + } + + // files_mode: 2 = compressed, 1 - normal + uint filesMode = 1; + + // ---- Compression (version 3+) ---- + if (version >= 3) + { + filesMode = reader.ReadUInt32(); + + if (filesMode > 2) + { + Toolkit.Instance.Logger.LogWarning( + $"Files mode {filesMode} is not supported. Falling back to uncompressed."); + } + + Info.ChunkCount = reader.ReadUInt32(); + + Info.ChunkBlockSizes = new ulong[Info.ChunkCount]; + + for (int i = 0; i < Info.ChunkCount; i++) + { + Info.ChunkBlockSizes[i] = reader.ReadUInt32(); + } + + if (Info.ChunkCount > 0) + { + // Both flags are set; DecompressBlock sorts out which algorithm is active. + Info.ChunkSize = 0x10000; + Info.Flags |= ArchiveFlags.IsRawDeflateCompressed | ArchiveFlags.IsZlibCompressed; + } + + _ = reader.ReadUInt32(); // total file-data region size AFTER the ttarch header is read (and the name) + } + + if (version >= 4) + { + _ = reader.ReadUInt32(); // unknown (possibly priority) + _ = reader.ReadUInt32(); // unknown + } + + if (version >= 5) + { + uint xMode1 = reader.ReadUInt32(); // unknown + uint xMode2 = reader.ReadUInt32(); // unknown + Info.Flags |= xMode1 == 1 || xMode2 == 1 ? ArchiveFlags.IsXMode : ArchiveFlags.None; + } + + if (version >= 7) + { + const uint blockSizeFactor = 1024; + Info.ChunkSize = reader.ReadUInt32() * blockSizeFactor; + } + + if (version >= 8) + { + // This is a boolean value that indicates whether a symbol table should be created or not. + byte createSymbolTable = reader.ReadByte(); + + if (createSymbolTable > 0) + { + _ = reader.ReadInt32(); // no idea if this is true + } + } + + if (version >= 9) + { + _ = reader.ReadUInt32(); // crc32 for checksum of the header (decompressed and decrypted) + } + + int infoHeaderSize = reader.ReadInt32(); + + // This is compressed + byte[] header; + if (version >= 6 && filesMode == 2) + { + int compressedHeaderSize = reader.ReadInt32(); + header = reader.ReadBytes(compressedHeaderSize); + ArchiveFlags archiveFlags = Info.Flags; + header = ChunkDecoder.DecompressBlock(header, infoHeaderSize, ref archiveFlags); + Info.Flags = archiveFlags; + } + else + { + header = reader.ReadBytes(infoHeaderSize); + } + + if (decryptionMode == 1) + _blowfish.Decipher(header, header.Length); + + Info.FilesOffset = (ulong)reader.BaseStream.Position; + ParseEntries(new MemoryStream(header)); + + _dataStream = BuildFileDataStream(version, filesMode, decryptionMode); + } + + /// + /// Reads the oldest encrypted format where the file begins with a raw header-size + /// field rather than a version number. + /// + /// + /// The 128-byte threshold distinguishes a genuine header-size field + /// (always larger) from what would otherwise look like a small version number. + /// Archives below the threshold are treated as plain unencrypted legacy files. + /// + private void ReadLegacyStream(BinaryReader reader) + { + reader.BaseStream.Seek(0, SeekOrigin.Begin); + Info.Version = TTArchiveVersion.Legacy; + _blowfish = new Blowfish(Info.BlowfishKey, (int)Info.Version); + + uint headerSize = reader.ReadUInt32(); + + // This is a lame work-around, because Telltale had stopped reading the original archives. + // There shouldn't be archives with more than 128 folders + if (headerSize > 128) + { + // Encrypted legacy archive — header block is Blowfish-decrypted in place. + Info.Flags |= ArchiveFlags.IsEncrypted; + + byte[] header = reader.ReadBytes((int)headerSize); + _blowfish.Decipher(header, header.Length); + + ParseEntries(new MemoryStream(header)); + Info.FilesOffset = (ulong)reader.BaseStream.Position; + _dataStream = BuildFileDataStream(0, 0, 1); + } + else + { + // Plain legacy archive — entries are written directly into the stream. + reader.BaseStream.Seek(0, SeekOrigin.Begin); + ParseEntries(reader.BaseStream); + Info.FilesOffset = reader.ReadUInt32(); + _ = reader.ReadUInt32(); // file size + _dataStream = BuildFileDataStream(0, 0, 0); + } + } + + private void ParseEntries(Stream stream) + { + using BinaryReader reader = new(stream, Encoding.ASCII, true); + + int directoriesCount = reader.ReadInt32(); + for (int i = 0; i < directoriesCount; i++) + { + int nameLength = reader.ReadInt32(); + + if (nameLength is < 1 or > 1023) + { + throw new ArgumentOutOfRangeException( + $"Error: directory name length {nameLength} is too long.", nameof(nameLength)); + } + + _ = Encoding.ASCII.GetString(reader.ReadBytes(nameLength)); + } + + uint fileCount = reader.ReadUInt32(); + + List entries = new((int)fileCount); + + for (int i = 0; i < fileCount; i++) + { + int nameLength = reader.ReadInt32(); + if (nameLength is < 1 or > 255) + { + throw new ArgumentOutOfRangeException( + $"Error: File name length {nameLength} is too long.", nameof(nameLength)); + } + + string name = Encoding.ASCII.GetString(reader.ReadBytes(nameLength)); + _ = reader.ReadInt32(); // folder-index field — always 0, purpose unclear + uint offset = reader.ReadUInt32(); + uint size = reader.ReadUInt32(); + + entries.Add(new ResourceEntry { NameCrc = Crc64.Compute(name), Name = name, Offset = offset, Size = size }); + } + + SetEntries(entries); + } + + private Stream BuildFileDataStream(int version, uint filesMode, uint decryptionMode) + { + // No compression → raw substream (may still need per-file decryption later) + if (Info.ChunkCount == 0 || filesMode != 2) + { + return new SubStream(BaseStream!, (long)Info.FilesOffset, long.MaxValue); + } + + // Compressed chunked region → on-demand decompression + return new TtarchiveChunkedDataStream( + BaseStream!, + (long)Info.FilesOffset, + Info.ChunkSize, + Info.ChunkBlockSizes, + Info.Flags, + Info.BlowfishKey, + version, + decryptionMode == 1); + } + + protected override Stream? OpenResource(ResourceEntry? entry) + { + if (_dataStream == null) + { + throw new InvalidOperationException("Archive not loaded."); + } + + // TODO: Implement decrypting in MetaStream + return entry is null ? null : new SubStream(_dataStream, (long)entry.Offset, entry.Size); + } + + /// + /// Creates a new .ttarch archive (versions 1–9) directly to a stream. + /// + /// Stream to write the archive to (left open). + /// Sequence of (filename, data stream) pairs. Filenames are ASCII, paths are flattened. + /// Archive options (version, encryption, compression, chunk size, Blowfish key). + public static void Create(Stream output, IEnumerable<(string name, Stream dataStream)> entries, + ArchiveWriteOptions options) + { + if (!output.CanSeek) + { + throw new ArgumentException("Output stream must be seekable for patching header fields.", nameof(output)); + } + + int version = options.TTArchiveVersion.ToJsonNumber(); + + switch (version) + { + case < 0 or > 9: + throw new NotSupportedException($"TTArchive.Create supports versions 1-9, got {version}."); + case 0: + CreateLegacy(output, entries, options); + return; + } + + bool encrypt = options.Encrypt; + bool compressFileData = options.Algorithm != CompressionAlgorithm.None && version >= 3; + bool compressHeader = compressFileData && version >= 6; // compressed entry table block (filesMode=2) + + Blowfish bf = new(options.BlowfishKey, version); + + // ------------------------------------------------------------------------ + // 1. Collect metadata & compute file offsets (within file-data region) + // ------------------------------------------------------------------------ + List<(string name, Stream stream, long length, ulong crc64, uint offset)> entryList = new(); + uint currentOffset = 0; + foreach ((string name, Stream stream) in entries) + { + long len = stream.Length; + if (len > uint.MaxValue) + { + throw new InvalidOperationException($"File {name} exceeds 4 GiB – not supported in TTArchive."); + } + + entryList.Add((name, stream, len, Crc64.Compute(name), currentOffset)); + currentOffset += (uint)len; + } + + uint totalFileDataSize = currentOffset; + + // ------------------------------------------------------------------------ + // 2. Build entry table header (directories + files) + // ------------------------------------------------------------------------ + using MemoryStream headerMs = new(); + using BinaryWriter headerWriter = new(headerMs, Encoding.ASCII, true); + + // directories (none) + headerWriter.Write(0); // directoriesCount + // files + headerWriter.Write(entryList.Count); + foreach ((string name, _, long length, ulong _, uint offset) in entryList) + { + byte[] nameBytes = Encoding.ASCII.GetBytes(name); + if (nameBytes.Length > 255) + { + throw new ArgumentException($"Filename too long: {name}"); + } + + headerWriter.Write(nameBytes.Length); + headerWriter.Write(nameBytes); + headerWriter.Write(0); // folder-index (always 0) + headerWriter.Write(offset); + headerWriter.Write((uint)length); + } + + byte[] rawHeader = headerMs.ToArray(); + int rawHeaderSize = rawHeader.Length; + + // ------------------------------------------------------------------------ + // 3. Helper to compress a block (raw deflate or zlib) + // ------------------------------------------------------------------------ + byte[] CompressBlock(byte[] data) + { + using MemoryStream ms = new(); + if (options.Algorithm == CompressionAlgorithm.Zlib) + { + using DeflaterOutputStream zlib = new(ms); + zlib.Write(data, 0, data.Length); + zlib.Finish(); + } + else + { + using DeflateStream deflate = new(ms, CompressionLevel.Optimal, true); + deflate.Write(data, 0, data.Length); + } + + return ms.ToArray(); + } + + // ------------------------------------------------------------------------ + // 4. Write outer header (with placeholders) and entry table + // ------------------------------------------------------------------------ + MemoryStream headerBuffer = new(); // buffer for header part (we'll write it after patching) + using BinaryWriter headerBufWriter = new(headerBuffer, Encoding.UTF8, true); + + headerBufWriter.Write((uint)version); + headerBufWriter.Write(encrypt ? 1u : 0u); // decryptionMode + if (version >= 2) + { + headerBufWriter.Write(0u); // unknown. Platform? + } + + long chunkCountPos = -1, chunkTableStartPos = -1; + if (version >= 3) + { + uint filesMode = compressFileData ? 2u : 1u; + headerBufWriter.Write(filesMode); + + // chunkCount placeholder + chunkCountPos = headerBufWriter.BaseStream.Position; + headerBufWriter.Write(0u); // will be overwritten + + if (compressFileData) + { + // Reserve space for chunk block sizes list (max possible chunks) + int maxChunks = (int)((totalFileDataSize + options.ChunkSize - 1) / options.ChunkSize); + chunkTableStartPos = headerBufWriter.BaseStream.Position; + for (int i = 0; i < maxChunks; i++) + { + headerBufWriter.Write(0u); // placeholder for each compressed size + } + } + else + { + // No compression: chunkCount = 0, then totalFileDataSize + headerBufWriter.Write(totalFileDataSize); + } + } + + // Version 4 unknowns + if (version >= 4) + { + headerBufWriter.Write(0u); + headerBufWriter.Write(0u); + } + + // Version 5 xMode + if (version >= 5) + { + int xModeValue = options.XMode ? 1 : 0; + headerBufWriter.Write(xModeValue); // xMode1 + headerBufWriter.Write(xModeValue); // xMode2 + } + + // Version 7 chunk size divisor + if (version >= 7 && compressFileData) + { + headerBufWriter.Write(0x40); + } + + // Version 8 symbol table flag + if (version >= 8) + { + headerBufWriter.Write((byte)0); // createSymbolTable = false + } + + // Version 9 CRC32 placeholder + long crc32Pos = -1; + if (version >= 9) + { + crc32Pos = headerBufWriter.BaseStream.Position; + headerBufWriter.Write(0u); + } + + // ---- Write entry table header (maybe compressed/encrypted) ---- + if (compressHeader) + { + byte[] compressedHeader = CompressBlock(rawHeader); + if (encrypt) + { + bf.Encipher(compressedHeader, compressedHeader.Length); + } + + headerBufWriter.Write(rawHeaderSize); + headerBufWriter.Write(compressedHeader.Length); + headerBufWriter.Write(compressedHeader); + } + else + { + if (encrypt) + { + bf.Encipher(rawHeader, rawHeader.Length); + } + + headerBufWriter.Write(rawHeader.Length); + headerBufWriter.Write(rawHeader); + } + + // ------------------------------------------------------------------------ + // 5. Write file-data region (raw or compressed chunks) + // ------------------------------------------------------------------------ + List compressedSizes = new(); + if (compressFileData) + { + int chunkSize = (int)options.ChunkSize; + byte[] chunkBuffer = new byte[chunkSize]; + int bufferPos = 0; + + // We'll write the compressed chunks directly to output after the header buffer. + // First, write the header buffer to output (to reserve space for header). + headerBuffer.Position = 0; + headerBuffer.CopyTo(output); + + // Process each file stream sequentially, filling chunkBuffer. + foreach ((string _, Stream stream, long length, ulong _, uint _) in entryList) + { + stream.Position = 0; + long bytesLeft = length; + while (bytesLeft > 0) + { + int want = chunkSize - bufferPos; + int readSize = (int)Math.Min(want, bytesLeft); + byte[] temp = new byte[readSize]; + if (stream.Read(temp, 0, readSize) != readSize) + { + throw new EndOfStreamException(); + } + + Buffer.BlockCopy(temp, 0, chunkBuffer, bufferPos, readSize); + bufferPos += readSize; + bytesLeft -= readSize; + + if (bufferPos != chunkSize) + { + continue; + } + + // Full chunk: compress and write + byte[] compressed = CompressBlock(chunkBuffer); + if (encrypt) + { + bf.Encipher(compressed, compressed.Length); + } + + output.Write(compressed, 0, compressed.Length); + compressedSizes.Add((uint)compressed.Length); + bufferPos = 0; + } + } + + // Final partial chunk + if (bufferPos > 0) + { + byte[] lastChunk = new byte[bufferPos]; + Buffer.BlockCopy(chunkBuffer, 0, lastChunk, 0, bufferPos); + byte[] compressed = CompressBlock(lastChunk); + if (encrypt) + { + bf.Encipher(compressed, compressed.Length); + } + + output.Write(compressed, 0, compressed.Length); + compressedSizes.Add((uint)compressed.Length); + } + + // Now patch the header with actual chunk count and block sizes + output.Seek(chunkCountPos, SeekOrigin.Begin); + BinaryWriter bw = new(output, Encoding.UTF8, true); + bw.Write((uint)compressedSizes.Count); + // Write block sizes + output.Seek(chunkTableStartPos, SeekOrigin.Begin); + foreach (uint size in compressedSizes) + { + bw.Write(size); + } + + // Seek back to end (optional) + output.Seek(0, SeekOrigin.End); + } + else + { + // Uncompressed: write header buffer, then raw file data + headerBuffer.Position = 0; + headerBuffer.CopyTo(output); + foreach ((string _, Stream stream, long _, ulong _, uint _) in entryList) + { + stream.Position = 0; + stream.CopyTo(output); + } + } + + // ------------------------------------------------------------------------ + // 6. Patch CRC32 for version 9 (if applicable) + // ------------------------------------------------------------------------ + if (version >= 9) + { + // CRC32 of decompressed+decrypted header (i.e., rawHeader) + uint crc = ComputeCrc32(rawHeader); + output.Seek(crc32Pos, SeekOrigin.Begin); + BinaryWriter bw = new(output, Encoding.UTF8, true); + bw.Write(crc); + // Seek back to end + output.Seek(0, SeekOrigin.End); + } + } + + private static uint ComputeCrc32(byte[] data) + { + Crc32 crc32 = new(); + crc32.Append(data); + return crc32.GetCurrentHashAsUInt32(); + } + + /// + /// Creates a legacy (version 0) .ttarch archive, optionally encrypted. + /// + /// Stream to write the archive to (must be seekable for patching header size). + /// Sequence of (filename, data stream) pairs. Filenames are ASCII. + /// + /// Archive options. must be + /// . + /// determines whether the header block is encrypted. + /// is required when Encrypt is true. + /// + public static void CreateLegacy(Stream output, IEnumerable<(string name, Stream dataStream)> entries, + ArchiveWriteOptions options) + { + if (options.TTArchiveVersion != TTArchiveVersion.Legacy) + { + throw new ArgumentException("Legacy creation requires TTArchiveVersion.Legacy", nameof(options)); + } + + if (options.Encrypt && string.IsNullOrEmpty(options.BlowfishKey)) + { + throw new ArgumentException("Encryption requires a Blowfish key", nameof(options)); + } + + // ------------------------------------------------------------------------ + // 1. Collect metadata & compute file offsets + // ------------------------------------------------------------------------ + List<(string name, Stream stream, long length, ulong crc64, uint offset)> entryList = new(); + uint currentOffset = 0; + foreach ((string name, Stream stream) in entries) + { + long len = stream.Length; + if (len > uint.MaxValue) + { + throw new InvalidOperationException($"File {name} exceeds 4 GiB – not supported in legacy TTArchive."); + } + + entryList.Add((name, stream, len, Crc64.Compute(name), currentOffset)); + currentOffset += (uint)len; + } + + uint totalFileDataSize = currentOffset; + + // ------------------------------------------------------------------------ + // 2. Build entry table (directories + files) + // ------------------------------------------------------------------------ + using MemoryStream headerMs = new(); + using BinaryWriter headerWriter = new(headerMs, Encoding.ASCII, true); + // directories (none) + headerWriter.Write(0); + // files + headerWriter.Write(entryList.Count); + foreach ((string name, _, long length, ulong _, uint offset) in entryList) + { + byte[] nameBytes = Encoding.ASCII.GetBytes(name); + if (nameBytes.Length > 255) + { + throw new ArgumentException($"Filename too long: {name}"); + } + + headerWriter.Write(nameBytes.Length); + headerWriter.Write(nameBytes); + headerWriter.Write(0); // folder-index (always 0) + headerWriter.Write(offset); + headerWriter.Write((uint)length); + } + + byte[] entryTable = headerMs.ToArray(); + + // ------------------------------------------------------------------------ + // 3. Write the archive + // ------------------------------------------------------------------------ + if (!options.Encrypt) + { + // Plain legacy: write entry table directly + output.Write(entryTable, 0, entryTable.Length); + // Then two trailing uints: filesOffset (where file data starts) and filesSize (total size of file data) + long filesOffset = output.Position; + using BinaryWriter bw = new(output, Encoding.UTF8, true); + bw.Write((uint)filesOffset); + bw.Write(totalFileDataSize); + } + else + { + // Encrypted legacy: write headerSize (encrypted block length) then encrypted entry table + Blowfish blowfish = new(options.BlowfishKey, 0); + byte[] encrypted = (byte[])entryTable.Clone(); + blowfish.Encipher(encrypted, encrypted.Length); + uint headerSize = (uint)encrypted.Length; + using BinaryWriter bw = new(output, Encoding.UTF8, true); + bw.Write(headerSize); + bw.Write(encrypted); + // Files offset is right after the encrypted header + // No trailing uints in encrypted legacy format according to reader (the reader sets FilesOffset to current position) + // So we don't write anything else. + } + + // ------------------------------------------------------------------------ + // 4. Write file data (raw, unencrypted) + // ------------------------------------------------------------------------ + foreach ((string _, Stream stream, long _, ulong _, uint _) in entryList) + { + stream.Position = 0; + stream.CopyTo(output); + } + } +} diff --git a/src/TelltaleToolKit/TelltaleArchives/Formats/TTArchive2.cs b/src/TelltaleToolKit/TelltaleArchives/Formats/TTArchive2.cs new file mode 100644 index 0000000..e183c1d --- /dev/null +++ b/src/TelltaleToolKit/TelltaleArchives/Formats/TTArchive2.cs @@ -0,0 +1,330 @@ +using System.Text; +using TelltaleToolKit.Serialization.Binary; +using TelltaleToolKit.TelltaleArchives.IO; +using TelltaleToolKit.Utility.Hashing; + +namespace TelltaleToolKit.TelltaleArchives.Formats; + +/// +/// Reader for the .ttarch2 archive format (TTA2 / TTA3 / TTA4). +/// +/// +/// +/// A .ttarch2 file is double-layered: an outer container header +/// (one of the TTC* four-byte magic values) wraps the actual TTA2/3/4 archive. +/// The container controls encryption and compression at the chunk level; the inner header +/// describes the file-entry table. This split is now modelled explicitly by +/// , which the read path opens before parsing the TTA header. +/// +/// +/// Entry lookup is O(1) via the base-class dictionary. +/// +/// +public class TTArchive2 : Archive +{ + private const int NamePageSize = 0x10000; + private ContainerStream? _containerStream; + + protected override void Activate() + { + // Open a ContainerStream over the raw file stream. + // ContainerStream reads the TTCN/TTCE/TTCZ/etc. header, sets up chunk tables, + // and then exposes a seekable view of the decrypted+decompressed payload. + _containerStream = new ContainerStream(BaseStream!, Info.BlowfishKey); + // Copy container state back to Info so the rest of the code (and callers + // that inspect Info) see the right values. + Info.Flags = _containerStream.Flags; + Info.ChunkSize = _containerStream.ChunkSize; + Info.ChunkCount = _containerStream.ChunkCount; + Info.ChunkBlockSizes = _containerStream.ChunkBlockSizes; + + ReadTtaHeader(_containerStream); + } + + private void ReadTtaHeader(ContainerStream container) + { + using BinaryReader reader = new(container, Encoding.UTF8, true); + + Info.Version = (TTArchiveVersion)reader.ReadUInt32(); + + if (Info.Version is TTArchiveVersion.Tta3 or TTArchiveVersion.Tta2) + { + _ = reader.ReadUInt32(); // unknown / reserved + } + + // mNamePageCount — number of 64 KiB pages in the name stream (mpNameStream). + uint fileNameBufferSize = reader.ReadUInt32(); + if (fileNameBufferSize > 0x10000000) + { + throw new InvalidDataException($"[TTArchive2] Name buffer size {fileNameBufferSize} is implausibly large."); + } + + int fileCount = reader.ReadInt32(); + if (fileCount > 0xFFFFF) + { + throw new InvalidDataException($"[TTArchive2] File count {fileCount} is implausibly large."); + } + + // FilesOffset: position inside the decoded payload where resource data begins — + // immediately after the entry table and the name stream. + + List entries = new(fileCount); + + for (int i = 0; i < fileCount; i++) + { + ulong crc64 = reader.ReadUInt64(); // mNameCRC + ulong fileOffset = reader.ReadUInt64(); // mOffset + + if (Info.Version is TTArchiveVersion.Tta2) + { + _ = reader.ReadUInt32(); // extra field present only in TTA2. It's an alternative to namepages. + } + + uint size = reader.ReadUInt32(); // mSize + uint preloadSize = reader.ReadUInt32(); // mPreloadSize + + ushort namePageIndex = 0; + ushort namePageOff = 0; + + if (Info.Version is not TTArchiveVersion.Tta2) + { + namePageIndex = reader.ReadUInt16(); // mNamePageIndex + namePageOff = reader.ReadUInt16(); // mNamePageOffset + } + + entries.Add(new ResourceEntry + { + NameCrc = crc64, + Offset = fileOffset, + Size = size, + PreloadSize = preloadSize, + NamePageIndex = namePageIndex, + NamePageOffset = namePageOff + }); + } + + Info.FilesOffset = (ulong)(reader.BaseStream.Position + fileNameBufferSize); + + // ---- Name stream ---- + // Consists of namePageCount contiguous 64 KiB pages. + // Each entry addresses its name as: absoluteByte = pageIndex * 0x10000 + pageOffset. + ResolveNames(reader, entries, fileNameBufferSize); + + SetEntries(entries); + } + + private static void ResolveNames( + BinaryReader reader, + List entries, + long nameStreamLen) + { + if (nameStreamLen == 0 || entries.Count == 0) + { + return; + } + + long namesStart = reader.BaseStream.Position; + + List<(ResourceEntry Entry, long Position)> sorted = entries + .Select(e => (Entry: e, Position: (long)e.NamePageIndex * NamePageSize + e.NamePageOffset)) + .OrderBy(x => x.Position) + .ToList(); + + reader.BaseStream.Seek(namesStart + sorted[0].Position, SeekOrigin.Begin); + + const int maxNameLength = 256; // sane limit to prevent infinite loops + + foreach ((ResourceEntry entry, long targetPos) in sorted) + { + long absolutePos = namesStart + targetPos; + if (reader.BaseStream.Position != absolutePos) + { + reader.BaseStream.Seek(absolutePos, SeekOrigin.Begin); + } + + int namePos = entry.NamePageIndex * NamePageSize + entry.NamePageOffset; + + if ((uint)namePos >= (uint)nameStreamLen) + { + Toolkit.Instance.Logger.LogWarning( + $"[TTArchive2] 0x{entry.NameCrc:X16}: name position 0x{namePos:X} " + + $"exceeds name stream length 0x{nameStreamLen:X}. Name left null."); + continue; + } + + StringBuilder nameBuilder = new(); + for (int i = 0; i < maxNameLength; i++) + { + byte b = reader.ReadByte(); + if (b == 0) + { + break; + } + + nameBuilder.Append((char)b); + } + + entry.Name = nameBuilder.ToString(); + } + } + + protected override Stream? OpenResource(ResourceEntry? entry) + { + if (_containerStream == null) + { + throw new InvalidOperationException("Archive not properly loaded."); + } + + if (entry is null) + { + return null; + } + + return new SubStream(_containerStream, (long)(Info.FilesOffset + entry.Offset), entry.Size); + } + + public override void Dispose() + { + _containerStream?.Dispose(); + base.Dispose(); + GC.SuppressFinalize(this); + } + + /// + /// Creates a new .ttarch2 archive from a collection of named streams. + /// + /// Stream to write the final archive to (will be left open). + /// Sequence of (entry name, data stream) pairs. + /// Container options (compression, encryption, chunk size, Blowfish key). + public static void Create(Stream output, IEnumerable entries, ArchiveWriteOptions options) + { + List<(string path, Stream dataStream)> finalEntries = []; + + foreach (string path in entries) + { + try + { + FileStream fs = File.Open(path, FileMode.Open); + finalEntries.Add((path, fs)); + } + catch (Exception e) + { + Toolkit.Instance.Logger.LogError($"[TTArchive2] Failed to open file {path}: {e.Message}"); + } + } + + Create(output, finalEntries, options); + } + + /// + /// Creates a new .ttarch2 archive from a collection of named streams. + /// + /// Stream to write the final archive to (will be left open). + /// Sequence of (entry name, data stream) pairs. + /// Container options (compression, encryption, chunk size, Blowfish key). + public static void Create(Stream output, IEnumerable<(string name, Stream dataStream)> entries, + ArchiveWriteOptions options) + { + // Collect metadata (no data buffering) + List<(string name, Stream stream, long length, ulong crc64)> entryList = new(); + foreach ((string name, Stream stream) in entries) + { + long len = stream.Length; + entryList.Add((name, stream, len, Crc64.Compute(name))); + } + + // Build name-page stream and positions + using MemoryStream nameMs = new(); + (ushort pageIndex, ushort pageOffset)[] namePositions = + new (ushort pageIndex, ushort pageOffset)[entryList.Count]; + for (int i = 0; i < entryList.Count; i++) + { + string name = entryList[i].name; + long pos = nameMs.Position; + namePositions[i] = ((ushort)(pos / 0x10000), (ushort)(pos % 0x10000)); + byte[] nameBytes = Encoding.ASCII.GetBytes(name); + nameMs.Write(nameBytes); + nameMs.WriteByte(0); + } + + uint namePageCount = (uint)((nameMs.Length + 0x10000 - 1) / 0x10000); + long paddedLen = namePageCount * 0x10000; + nameMs.SetLength(paddedLen); + byte[] namePageBytes = nameMs.ToArray(); + + // Calculate file offsets (cumulative sum of sizes) + ulong[] fileOffsets = new ulong[entryList.Count]; + ulong currentOffset = 0; + for (int i = 0; i < entryList.Count; i++) + { + fileOffsets[i] = currentOffset; + currentOffset += (ulong)entryList[i].length; + } + + // Write inner payload to a temporary file + string tempFile = Path.GetTempFileName(); + try + { + using (FileStream inner = new(tempFile, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, + FileOptions.DeleteOnClose | FileOptions.SequentialScan)) + using (BinaryWriter bw = new(inner, Encoding.UTF8, true)) + { + // Header + bw.Write((uint)options.TTArchiveVersion); + if (options.TTArchiveVersion is TTArchiveVersion.Tta2 or TTArchiveVersion.Tta3) + { + bw.Write(0u); + } + + bw.Write(namePageCount); + bw.Write((uint)entryList.Count); + + // Entry table + for (int i = 0; i < entryList.Count; i++) + { + bw.Write(entryList[i].crc64); + bw.Write(fileOffsets[i]); + if (options.TTArchiveVersion is TTArchiveVersion.Tta2) + { + bw.Write(0u); + } + + bw.Write((uint)entryList[i].length); + bw.Write(0u); + bw.Write(namePositions[i].pageIndex); + bw.Write(namePositions[i].pageOffset); + } + + // Name pages + bw.Write(namePageBytes); + + // File data + for (int i = 0; i < entryList.Count; i++) + { + Stream stream = entryList[i].stream; + stream.Position = 0; + stream.CopyTo(inner); + } + } + + // Now wrap with container + using (FileStream innerReader = new(tempFile, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, + FileOptions.DeleteOnClose | FileOptions.SequentialScan)) + { + ContainerStream.Create(output, innerReader, options); + } + } + finally + { + // Ensure temp file is deleted (FileOptions.DeleteOnClose should handle, but just in case) + try + { + File.Delete(tempFile); + } + catch + { + // ignored + } + } + } +} diff --git a/src/TelltaleToolKit/TelltaleArchives/IO/ChunkDecoder.cs b/src/TelltaleToolKit/TelltaleArchives/IO/ChunkDecoder.cs new file mode 100644 index 0000000..c2fce78 --- /dev/null +++ b/src/TelltaleToolKit/TelltaleArchives/IO/ChunkDecoder.cs @@ -0,0 +1,138 @@ +using System.IO.Compression; +using ICSharpCode.SharpZipLib.Zip.Compression.Streams; +using TelltaleToolKit.TelltaleArchives.Caching; +using TelltaleToolKit.TelltaleArchives.Formats; +using TelltaleToolKit.Utility.Blowfish; + +namespace TelltaleToolKit.TelltaleArchives.IO; + +/// +/// Provides chunk-level decrypt + decompress logic shared by both +/// and extraction paths. +/// +/// +/// +/// All methods are thread-safe with respect to the archive stream as long as each +/// concurrent caller holds its own instance and does not +/// share a reference. +/// +/// +internal static class ChunkDecoder +{ + /// + /// Decrypts (if encrypted) and decompresses a raw compressed chunk read from disk. + /// Returns the decoded byte array. + /// + public static byte[] DecryptAndDecompress(byte[] compressed, uint expectedSize, ArchiveInfo info) + { + // Decrypt in place when the archive uses chunk-level encryption. + if (info.Flags.IsEncrypted()) + { + Blowfish bf = new(info.BlowfishKey, 7); + bf.Decipher(compressed.AsSpan(), compressed.Length); + } + + ArchiveFlags flags = info.Flags; + byte[] arr = DecompressBlock(compressed, (int)expectedSize, ref flags); + info.Flags = flags; + return arr; + } + + // ------------------------------------------------------------------------- + // Build a block-offset table for the range [blockStart, blockEnd] + // ------------------------------------------------------------------------- + + // ------------------------------------------------------------------------- + // Helpers + // ------------------------------------------------------------------------- + + /// Reads exactly buffer.Length bytes from . + internal static void ReadExact(Stream stream, byte[] buffer) + { + int total = 0; + while (total < buffer.Length) + { + int read = stream.Read(buffer, total, buffer.Length - total); + if (read == 0) + { + throw new EndOfStreamException( + $"Unexpected end of stream after {total} / {buffer.Length} bytes."); + } + + total += read; + } + } + + public static byte[] DecompressBlock(byte[] compressedData, int expectedSize, ref ArchiveFlags flags) + { + // This is a hack? In TWD:DE, 401_txmesh, there's a page which is the same size as the expected size (65535) + // C#'s raw deflate fails. + // ttarchext returns the same data: + // https://github.com/infernokun/TelltaleGamesExtractionGUI/blob/fec9fc1a70b545bcc67062fedc3fe3f7cd0d3e1b/bin/Debug/net6.0-windows/ttarchext/ttarchext.c#L1512 + if (compressedData.Length == expectedSize) + { + return compressedData; + } + + // Try both Deflate and Zlib if both flags are set + // TODO: Investigate ttarch + // CSI games use zlib compression instead of the regular raw deflate. + // According to TTG Tools, version 8 archives use zlib (presumably for the files themselves) + if (flags.HasFlag(ArchiveFlags.IsRawDeflateCompressed) && flags.HasFlag(ArchiveFlags.IsZlibCompressed)) + { + // CompressionAlgorithm compressionAlgorithm = DetectCompressionAlgorithm(compressedData); + + try + { + byte[] result = Decompress(ms => new DeflateStream(ms, CompressionMode.Decompress)); + flags &= ~ArchiveFlags.IsZlibCompressed; + return result; + } + catch (Exception) + { + // flags &= ~ContainerFlags.IsRawDeflateCompressed; + } + + try + { + byte[] result = Decompress(ms => new InflaterInputStream(ms)); + flags &= ~ArchiveFlags.IsRawDeflateCompressed; + return result; + } + catch (Exception) + { + // flags &= ~ContainerFlags.IsZlibCompressed; + } + + throw new InvalidDataException("Compression not supported"); + } + + if (flags.HasFlag(ArchiveFlags.IsRawDeflateCompressed)) + { + return Decompress(ms => new DeflateStream(ms, CompressionMode.Decompress)); + } + + if (flags.HasFlag(ArchiveFlags.IsZlibCompressed)) + { + return Decompress(ms => new InflaterInputStream(ms)); + } + + if (flags.HasFlag(ArchiveFlags.IsOodleCompressed)) + { + // TODO: Add Oodle compression. + throw new NotSupportedException("Oodle compression is not supported yet."); + } + + // No compression, return the original data + return compressedData; + + // Local function to handle decompression + byte[] Decompress(Func streamFactory) + { + using MemoryStream outputStream = new(expectedSize); + using Stream decompressStream = streamFactory(new MemoryStream(compressedData)); + decompressStream.CopyTo(outputStream); + return outputStream.ToArray(); + } + } +} diff --git a/src/TelltaleToolKit/TelltaleArchives/IO/ContainerStream.cs b/src/TelltaleToolKit/TelltaleArchives/IO/ContainerStream.cs new file mode 100644 index 0000000..2b0e9ee --- /dev/null +++ b/src/TelltaleToolKit/TelltaleArchives/IO/ContainerStream.cs @@ -0,0 +1,456 @@ +using System.Text; +using TelltaleToolKit.TelltaleArchives.Caching; +using TelltaleToolKit.TelltaleArchives.Formats; +using TelltaleToolKit.Utility.Blowfish; + +namespace TelltaleToolKit.TelltaleArchives.IO; + +/// +/// A forward-readable, seekable that decodes the outer +/// Telltale Tool Container layer of a .ttarch2 file. +/// +/// +/// +/// Single responsibility: parse the container header and expose the +/// inner TTA2/3/4 payload as a readable stream so can walk +/// the entry table and name-page stream. It is disposed immediately after that forward +/// pass — extraction never goes through it. +/// +/// +/// No caching. Header parsing is a single sequential forward pass over +/// a tiny fraction of the archive. Decoding a chunk once and discarding it is always +/// enough. This avoids any heap accumulation for 2 GB+ archives. +/// +/// +/// O(1) chunk offset lookup. A prefix-sum table () +/// is built once from the chunk-size array during construction, so seeking to any chunk +/// does not require walking the size array. +/// +/// +public sealed class ContainerStream : Stream +{ + private readonly string _blowfishKey; + private readonly IChunkCache _cache; + + // Prefix-sum table: _chunkPrefixSums[i] = sum of compressed sizes of chunks 0..i-1. + // Length = ChunkCount + 1 so _chunkPrefixSums[ChunkCount] = total compressed payload size. + // Built once during construction for O(1) chunk-offset lookup. + private readonly ulong[] _chunkPrefixSums; + // ------------------------------------------------------------------------- + // Fields + // ------------------------------------------------------------------------- + + private readonly Stream _source; + + // Logical read position within the decoded (uncompressed) payload. + private long _position; + + // ------------------------------------------------------------------------- + // Construction + // ------------------------------------------------------------------------- + + /// + /// Reads the container header from and prepares the + /// prefix-sum table for O(1) chunk-offset lookup. + /// + /// Raw file stream, positioned at byte 0. Not disposed by this class. + /// Blowfish key for this game's archives. + public ContainerStream(Stream source, string blowfishKey, IChunkCache? cache = null) + { + _source = source ?? throw new ArgumentNullException(nameof(source)); + _blowfishKey = blowfishKey ?? throw new ArgumentNullException(nameof(blowfishKey)); + _cache = cache ?? new SinglePageCache(); + + using BinaryReader reader = new(source, Encoding.UTF8, true); + + // ---- Container magic ---- + ContainerMagic = (ContainerMagic)reader.ReadUInt32(); + ArchiveFlags flags = ArchiveFlags.None; + + switch (ContainerMagic) + { + case ContainerMagic.TTCN: + // No encryption, no compression — nothing to add. + break; + + case ContainerMagic.TTCE: + flags |= ArchiveFlags.IsEncrypted | ArchiveFlags.IsRawDeflateCompressed; + break; + + case ContainerMagic.TTCZ: + flags |= ArchiveFlags.IsRawDeflateCompressed; + break; + + case ContainerMagic.TTCe: + case ContainerMagic.TTCz: + { + if (ContainerMagic is ContainerMagic.TTCe) + { + flags |= ArchiveFlags.IsEncrypted; + } + + uint compressionType = reader.ReadUInt32(); + flags |= compressionType switch + { + 0 => ArchiveFlags.IsRawDeflateCompressed, + 1 => ArchiveFlags.IsOodleCompressed, + _ => throw new NotSupportedException( + $"[ContainerStream] Unknown compression type {compressionType}.") + }; + break; + } + + default: + throw new NotSupportedException( + $"[ContainerStream] Unknown container version 0x{(uint)ContainerMagic:X8}."); + } + + Flags = flags; + + if (flags.IsCompressed()) + { + ChunkSize = reader.ReadUInt32(); + + ChunkCount = reader.ReadUInt32(); + ChunkBlockSizes = new ulong[ChunkCount]; + + // The on-disk table stores ChunkCount + 1 cumulative absolute offsets. + // Element 0 is the start of the first chunk (always 0 relative to payload start). + // We derive per-chunk sizes from successive differences. + ulong prev = reader.ReadUInt64(); // always 0 + for (int i = 0; i < ChunkCount; i++) + { + ulong next = reader.ReadUInt64(); + ChunkBlockSizes[i] = next - prev; + prev = next; + } + + // Build prefix-sum table for O(1) chunk-offset lookup. + _chunkPrefixSums = BuildPrefixSums(ChunkBlockSizes, ChunkCount); + + Length = ChunkCount * ChunkSize; + } + else + { + Length = reader.ReadInt64(); // uncompressed container payload size — unused + ChunkBlockSizes = []; + _chunkPrefixSums = [0L]; + } + + PayloadStart = source.Position; // first byte of TTA2/3/4 payload + } + + // ------------------------------------------------------------------------- + // Properties exposed to TTArchive2 + // ------------------------------------------------------------------------- + + /// Four-byte container magic read from the file header. + public ContainerMagic ContainerMagic { get; } + + /// Encryption and compression flags derived from the container magic. + public ArchiveFlags Flags { get; } + + /// Decompressed size of each chunk in bytes. Zero for uncompressed containers. + public uint ChunkSize { get; } + + /// Number of compressed chunks. Zero for uncompressed containers. + public uint ChunkCount { get; } + + /// Compressed byte-length of each chunk. Empty for uncompressed containers. + public ulong[] ChunkBlockSizes { get; } + + /// + /// Byte offset in the raw source stream where the TTA2/3/4 payload begins + /// (immediately after the container header). + /// + public long PayloadStart { get; } + + // ------------------------------------------------------------------------- + // Stream overrides + // ------------------------------------------------------------------------- + + /// + public override bool CanRead => true; + + /// + public override bool CanSeek => true; + + /// + public override bool CanWrite => false; + + /// + public override long Length { get; } + + /// + public override long Position + { + get => _position; + set + { + if (value < 0) + { + throw new ArgumentOutOfRangeException(nameof(value)); + } + + _position = value; + } + } + + /// + public override long Seek(long offset, SeekOrigin origin) + { + _position = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => _position + offset, + SeekOrigin.End => Length + offset, + _ => throw new ArgumentOutOfRangeException(nameof(origin)) + }; + return _position; + } + + /// + public override int Read(byte[] buffer, int offset, int count) + => Read(buffer.AsSpan(offset, count)); + + /// + public override int Read(Span buffer) + { + if (buffer.IsEmpty) + { + return 0; + } + + // ---- Uncompressed — backed by MemoryStream ---- + if (!Flags.IsCompressed()) + { + _source.Position = PayloadStart + _position; + int bytesToRead = (int)Math.Min(buffer.Length, Length - _position); + if (bytesToRead <= 0) + { + return 0; + } + + Span temp = buffer.Slice(0, bytesToRead); + int read = _source.Read(temp); + if (read == 0) + { + return 0; + } + + if (Flags.IsEncrypted()) + { + Blowfish bf = new(_blowfishKey, 7); + bf.Decipher(temp, read); + } + + _position += read; + return read; + } + + // ---- Compressed — decode on demand, no caching ---- + // ContainerStream is used only for a single sequential forward pass over the + // TTA header + name pages. Re-decoding the same chunk twice should never happen + // in practice, so caching would only waste memory. + int totalRead = 0; + + while (totalRead < buffer.Length) + { + int chunkIndex = (int)(_position / ChunkSize); + if ((uint)chunkIndex >= ChunkCount) + { + break; // past end of payload + } + + // Use cache to get decoded chunk + byte[] decoded = _cache.GetOrDecode(chunkIndex, DecodeChunk); + + int inChunk = (int)(_position % ChunkSize); + int avail = decoded.Length - inChunk; + if (avail <= 0) + { + break; + } + + int toCopy = Math.Min(avail, buffer.Length - totalRead); + decoded.AsSpan(inChunk, toCopy).CopyTo(buffer[totalRead..]); + + totalRead += toCopy; + _position += toCopy; + } + + return totalRead; + } + + private byte[] DecodeChunk(int chunkIndex) + { + long chunkSrcOffset = PayloadStart + (long)_chunkPrefixSums[chunkIndex]; + _source.Seek(chunkSrcOffset, SeekOrigin.Begin); + + byte[] compressed = new byte[ChunkBlockSizes[chunkIndex]]; + ChunkDecoder.ReadExact(_source, compressed); + + ArchiveInfo tempInfo = new() { Flags = Flags, BlowfishKey = _blowfishKey }; + return ChunkDecoder.DecryptAndDecompress(compressed, ChunkSize, tempInfo); + } + + // ------------------------------------------------------------------------- + // Write / mutate — not supported + // ------------------------------------------------------------------------- + + /// + public override void Write(byte[] buffer, int offset, int count) + => throw new NotSupportedException("ContainerStream is read-only. Use ContainerWriter to produce output."); + + /// + public override void SetLength(long value) + => throw new NotSupportedException("ContainerStream is read-only."); + + /// + public override void Flush() + { + /* read-only — no-op */ + } + + // ------------------------------------------------------------------------- + // Helpers + // ------------------------------------------------------------------------- + + /// + /// Builds a prefix-sum array of length + 1 from + /// , enabling O(1) compressed-chunk offset lookup. + /// result[i] = sum of sizes[0..i-1]. + /// + private static ulong[] BuildPrefixSums(ulong[] sizes, uint count) + { + ulong[] sums = new ulong[count + 1]; + for (int i = 0; i < (int)count; i++) + { + sums[i + 1] = sums[i] + sizes[i]; + } + + return sums; + } + + /// + /// Writes a Telltale Tool Container (TTC*) around the given payload. + /// Automatically selects the magic number based on encryption and compression settings. + /// + /// Stream to write the container to. Will be left open. + /// Uncompressed, unencrypted inner data (TTA header + files). + /// Compression, encryption, chunk size, Blowfish key, etc. + public static void Create(Stream destination, ReadOnlySpan payload, ArchiveWriteOptions options) + { + using MemoryStream ms = new(payload.ToArray()); + Create(destination, ms, options); + } + + /// + /// Streaming version – writes a Telltale Tool Container without buffering the entire payload. + /// + /// Output stream (e.g. FileStream). + /// + /// Stream containing the uncompressed inner payload. Must support seeking and provide a correct Length. + /// + /// Archive options (compression, encryption, chunk size, key). + public static void Create(Stream destination, Stream payloadStream, ArchiveWriteOptions options) + { + if (options.Algorithm == CompressionAlgorithm.Oodle) + { + throw new NotSupportedException("Oodle compression not yet implemented."); + } + + // Choose magic automatically + ContainerMagic magic = (options.Encrypt, options.Algorithm) switch + { + (false, CompressionAlgorithm.None) => ContainerMagic.TTCN, + (true, CompressionAlgorithm.Deflate) => ContainerMagic.TTCE, + (false, CompressionAlgorithm.Deflate) => ContainerMagic.TTCZ, + (true, CompressionAlgorithm.Oodle) => ContainerMagic + .TTCe, // encryption without compression? Rare, but fallback + (false, CompressionAlgorithm.Oodle) => ContainerMagic + .TTCz, // encryption without compression? Rare, but fallback + _ => throw new NotSupportedException( + $"Unsupported combination: Encrypt={options.Encrypt}, Algorithm={options.Algorithm}") + }; + + using BinaryWriter writer = new(destination, Encoding.UTF8, true); + writer.Write((uint)magic); + + if (magic is ContainerMagic.TTCe or ContainerMagic.TTCz) + { + uint comprType = options.Algorithm == CompressionAlgorithm.Deflate ? 0u : 1u; // 1 = Oodle + writer.Write(comprType); + } + + bool compress = options.Algorithm != CompressionAlgorithm.None; + Blowfish blowfish = new(options.BlowfishKey, 7); + + if (!compress) + { + // Uncompressed container: just length + raw data (optional encryption) + writer.Write((ulong)payloadStream.Length); + payloadStream.CopyTo(destination); + if (options.Encrypt) + { + throw new NotSupportedException( + "Encryption without compression is not supported by this container type."); + } + + // Note: plain encryption of whole payload would be possible but no known game uses it. + return; + } + + int chunkSize = (int)options.ChunkSize; + long totalLen = payloadStream.Length; + int chunkCount = (int)((totalLen + chunkSize - 1) / chunkSize); + + writer.Write(chunkSize); + writer.Write((uint)chunkCount); + + // Reserve space for the offset table (chunkCount+1 entries, each 8 bytes) + long offsetTablePos = writer.BaseStream.Position; + for (int i = 0; i <= chunkCount; i++) + { + writer.Write(0L); // placeholder + } + + // We'll store compressed sizes as we write the chunks, then seek back and write cumulative offsets. + ulong[] compressedSizes = new ulong[chunkCount]; + byte[] compressedBuffer = new byte[chunkSize + chunkSize / 8 + 512]; // worst-case overhead for compression + + payloadStream.Seek(0, SeekOrigin.Begin); + for (int i = 0; i < chunkCount; i++) + { + int bytesToRead = (int)Math.Min(chunkSize, totalLen - payloadStream.Position); + Span chunk = new byte[bytesToRead]; + if (payloadStream.Read(chunk) != bytesToRead) + { + throw new EndOfStreamException("Unexpected end of payload stream."); + } + + int compressedLen = TelltaleArchiveUtilities.CompressBlock(chunk, options.Algorithm, compressedBuffer); + byte[] finalData = compressedBuffer.AsSpan(0, compressedLen).ToArray(); + if (options.Encrypt) + { + blowfish.Encipher(finalData, finalData.Length); + } + + compressedSizes[i] = (ulong)finalData.Length; + writer.Write(finalData); + } + + // Go back and write the cumulative offset table + long endPos = writer.BaseStream.Position; + writer.BaseStream.Seek(offsetTablePos, SeekOrigin.Begin); + ulong cumulative = 0; + writer.Write(cumulative); + for (int i = 0; i < chunkCount; i++) + { + cumulative += compressedSizes[i]; + writer.Write(cumulative); + } + + writer.BaseStream.Seek(endPos, SeekOrigin.Begin); + } +} diff --git a/src/TelltaleToolKit/TelltaleArchives/IO/LegacyEncrypted.cs b/src/TelltaleToolKit/TelltaleArchives/IO/LegacyEncrypted.cs new file mode 100644 index 0000000..01d93ca --- /dev/null +++ b/src/TelltaleToolKit/TelltaleArchives/IO/LegacyEncrypted.cs @@ -0,0 +1,70 @@ +using TelltaleToolKit.Utility.Blowfish; + +namespace TelltaleToolKit.TelltaleArchives.IO; + +/// +/// Stream wrapper that decrypts TTArchive per‑file data on the fly. +/// For simplicity, it reads the entire file into memory – these files are small +/// (config files, scripts) so the overhead is acceptable. +/// +internal sealed class LegacyEncrypted : Stream +{ + private readonly Blowfish _blowfish; + private readonly Stream _inner; + + public LegacyEncrypted(Stream inner, Blowfish blowfish) + { + _inner = inner; + _blowfish = blowfish; + // Read the whole file, decrypt it, and keep the result in memory. + using MemoryStream ms = new(); + _inner.CopyTo(ms); + // _decryptedData = ms.ToArray(); + // TelltaleArchiveUtilities.DecryptFile(_decryptedData, _blowfish); + } + + public override bool CanRead => true; + public override bool CanSeek => true; + public override bool CanWrite => false; + + public override long Length { get; } + // public override long Length => _decryptedData.Length; + + public override long Position { get; set; } + + public override int Read(byte[] buffer, int offset, int count) + => Read(buffer.AsSpan(offset, count)); + + public override int Read(Span buffer) + { + long remaining = Length - Position; + if (remaining <= 0) + { + return 0; + } + + int toRead = (int)Math.Min(buffer.Length, remaining); + // _decryptedData.AsSpan((int)_position, toRead).CopyTo(buffer); + Position += toRead; + return toRead; + } + + public override long Seek(long offset, SeekOrigin origin) + { + Position = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => Position + offset, + SeekOrigin.End => Length + offset, + _ => throw new ArgumentOutOfRangeException(nameof(origin)) + }; + return Position; + } + + public override void Flush() + { + } + + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); +} diff --git a/src/TelltaleToolKit/TelltaleArchives/IO/TtarchiveChunkedDataStream.cs b/src/TelltaleToolKit/TelltaleArchives/IO/TtarchiveChunkedDataStream.cs new file mode 100644 index 0000000..c464ba7 --- /dev/null +++ b/src/TelltaleToolKit/TelltaleArchives/IO/TtarchiveChunkedDataStream.cs @@ -0,0 +1,152 @@ +using TelltaleToolKit.TelltaleArchives.Caching; +using TelltaleToolKit.Utility.Blowfish; + +namespace TelltaleToolKit.TelltaleArchives.IO; + +/// +/// Presents a continuous, seekable stream of decompressed file data from a +/// TTArchive compressed chunk table (version>=6, filesMode=2). +/// +internal sealed class TtarchiveChunkedDataStream : Stream +{ + private readonly Blowfish _blowfish; + private readonly IChunkCache _cache; + private readonly ulong[] _chunkPrefixSums; + private readonly uint _chunkSize; + private readonly ulong[] _chunkSizes; + private readonly long _dataStart; + private ArchiveFlags _flags; + private readonly bool _isEncrypted; + private readonly Stream _source; + private long _position; + + public TtarchiveChunkedDataStream( + Stream source, + long dataStart, + uint chunkSize, + ulong[] chunkSizes, + ArchiveFlags flags, + string blowfishKey, + int archiveVersion, + bool isEncrypted, IChunkCache? cache = null) + { + _source = source; + _dataStart = dataStart; + _chunkSize = chunkSize; + _chunkSizes = chunkSizes; + _flags = flags; + _isEncrypted = isEncrypted; + _blowfish = new Blowfish(blowfishKey, archiveVersion); + _cache = cache ?? new SinglePageCache(); + + // Build prefix sums for O(1) chunk offset lookup + _chunkPrefixSums = new ulong[chunkSizes.Length + 1]; + for (int i = 0; i < chunkSizes.Length; i++) + { + _chunkPrefixSums[i + 1] = _chunkPrefixSums[i] + chunkSizes[i]; + } + } + + public override bool CanRead => true; + public override bool CanSeek => true; + public override bool CanWrite => false; + + // Total decompressed length of the file-data region + public override long Length => _chunkSizes.Length * _chunkSize; + + public override long Position + { + get => _position; + set + { + if (value < 0 || value > Length) + { + throw new ArgumentOutOfRangeException(nameof(value)); + } + + _position = value; + } + } + + public override int Read(byte[] buffer, int offset, int count) + => Read(buffer.AsSpan(offset, count)); + + public override int Read(Span buffer) + { + if (buffer.IsEmpty) + { + return 0; + } + + long remaining = Length - _position; + if (remaining <= 0) + { + return 0; + } + + int toRead = (int)Math.Min(buffer.Length, remaining); + + int totalRead = 0; + while (totalRead < toRead) + { + int chunkIdx = (int)(_position / _chunkSize); + if (chunkIdx >= _chunkSizes.Length) + { + break; + } + + byte[] decompressed = _cache.GetOrDecode(chunkIdx, DecodeChunk); + + int inChunk = (int)(_position % _chunkSize); + int avail = decompressed.Length - inChunk; + int copyNow = Math.Min(avail, toRead - totalRead); + + decompressed.AsSpan(inChunk, copyNow).CopyTo(buffer[totalRead..]); + totalRead += copyNow; + _position += copyNow; + } + + return totalRead; + } + + private byte[] DecodeChunk(int chunkIdx) + { + // Seek to the compressed chunk + long offset = _dataStart + (long)_chunkPrefixSums[chunkIdx]; + _source.Seek(offset, SeekOrigin.Begin); + + byte[] compressed = new byte[_chunkSizes[chunkIdx]]; + ChunkDecoder.ReadExact(_source, compressed); + + // Decrypt chunk if needed (per‑chunk, not per‑file) + if (_isEncrypted) + { + _blowfish.Decipher(compressed.AsSpan(), compressed.Length); + } + + // Decompress the chunk + ArchiveFlags flags = _flags; + var res = ChunkDecoder.DecompressBlock(compressed, (int)_chunkSize, ref flags); + _flags = flags; + return res; + } + + public override long Seek(long offset, SeekOrigin origin) + { + _position = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => _position + offset, + SeekOrigin.End => Length + offset, + _ => throw new ArgumentOutOfRangeException(nameof(origin)) + }; + return _position; + } + + public override void Flush() + { + } + + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); +} diff --git a/src/TelltaleToolKit/TelltaleArchives/PK2.cs b/src/TelltaleToolKit/TelltaleArchives/PK2.cs deleted file mode 100644 index 846c7d2..0000000 --- a/src/TelltaleToolKit/TelltaleArchives/PK2.cs +++ /dev/null @@ -1,25 +0,0 @@ -namespace TelltaleToolKit.TelltaleArchives; - -// TODO: PK2 for CSI. -public class PK2Archive : ArchiveBase -{ - protected override void ReadMetadata() - { - throw new NotImplementedException(); - } - - public override MemoryStream? ExtractFile(string fileName) - { - throw new NotImplementedException(); - } - - public override MemoryStream? ExtractFile(ulong hash) - { - throw new NotImplementedException(); - } - - public override void ExtractAll(string destinationPath) - { - throw new NotImplementedException(); - } -} \ No newline at end of file diff --git a/src/TelltaleToolKit/TelltaleArchives/ResourceEntry.cs b/src/TelltaleToolKit/TelltaleArchives/ResourceEntry.cs new file mode 100644 index 0000000..411286c --- /dev/null +++ b/src/TelltaleToolKit/TelltaleArchives/ResourceEntry.cs @@ -0,0 +1,79 @@ +using TelltaleToolKit.Reflection; +using TelltaleToolKit.Serialization; +using TelltaleToolKit.Serialization.Serializers; + +namespace TelltaleToolKit.TelltaleArchives; + +/// +/// Represents a single file record inside a Telltale archive. +/// +/// +/// +/// is the canonical lookup key used by 's +/// internal dictionary. It is computed from when the archive is loaded +/// and must not change after insertion. +/// +/// +[MetaClassSerializerGlobal(typeof(DefaultClassSerializer))] +public sealed class ResourceEntry +{ + /// + /// Gets the CRC-64 hash of . This is the dictionary key in . + /// + [MetaMember("mNameCrc")] + public ulong NameCrc { get; init; } + + /// + /// Gets or sets the original filename stored in the archive. + /// + public string Name { get; set; } = string.Empty; + + /// + /// Gets or sets the byte offset of this entry's data relative to the start of the + /// file-data region (ArchiveInfo.FilesOffset). + /// + /// + /// Internal - mutated by archive writers when packing or repacking. + /// + [MetaMember("mOffset")] + public ulong Offset { get; internal init; } + + /// + /// Gets or sets the raw (compressed) byte size of this entry's data in the archive. + /// + /// + /// Internal - mutated by archive writers when packing or repacking. + /// + [MetaMember("mSize")] + public uint Size { get; internal init; } + + /// + /// No idea. Needs investigation. + /// + /// + /// Internal - mutated by archive writers when packing or repacking. + /// + [MetaMember("mPreloadSize")] + public uint PreloadSize { get; internal set; } + + /// + /// Gets or sets the index of the name page containing this entry's filename. + /// + /// + /// Internal - mutated by archive writers when packing or repacking. + /// + [MetaMember("mNamePageIndex")] + public ushort NamePageIndex { get; internal init; } + + /// + /// Gets or sets the offset of this entry's filename within the name page. + /// + /// + /// Internal - mutated by archive writers when packing or repacking. + /// + [MetaMember("mNamePageOffset")] + public ushort NamePageOffset { get; internal init; } + + /// + public override string ToString() => $"{Name} (CRC: {NameCrc:X16}, Offset: {Offset}, Size: {Size})"; +} diff --git a/src/TelltaleToolKit/TelltaleArchives/TTArchiveVersion.cs b/src/TelltaleToolKit/TelltaleArchives/TTArchiveVersion.cs new file mode 100644 index 0000000..9a2ce55 --- /dev/null +++ b/src/TelltaleToolKit/TelltaleArchives/TTArchiveVersion.cs @@ -0,0 +1,121 @@ +namespace TelltaleToolKit.TelltaleArchives; + +/// +/// Version identifier for the inner Telltale archive format. +/// +/// +/// +/// Versions 0–9 are used by the original .ttarch format. +/// Versions , , +/// are used by the newer .ttarch2 format and are stored as four‑character codes. +/// +/// +public enum TTArchiveVersion +{ + /// Legacy version 0 (unversioned, used by very early games). + Legacy = 0, + + /// Version 1 – basic header with encryption flag. + One, + + /// Version 2 – adds an unknown uint field. + Two, + + /// Version 3 – introduces compression and chunk tables. + Three, + + /// Version 4 – adds two unknown uint fields (priority?). + Four, + + /// Version 5 – adds XMode flags. + Five, + + /// Version 6 – adds compressed entry table support. + Six, + + /// Version 7 – chunk size stored as multiple of 1024. + Seven, + + /// Version 8 – adds optional symbol table(?). + Eight, + + /// Version 9 – adds header CRC32. + Nine, + + /// + /// "TTA2" – .ttarch2 format version 2 (unused). + /// + Tta2 = 0x54544132, + + /// + /// "TTA3" – .ttarch2 format version 3. + /// + Tta3 = 0x54544133, + + /// + /// "TTA4" – .ttarch2 format version 4. + /// + Tta4 = 0x54544134 +} + +/// +/// Use and +/// +/// to convert between the enum and integer representations. +/// +public static class TTArchiveVersionExtensions +{ + /// + /// Converts an int to ArchiveVersion. + /// + public static TTArchiveVersion TtarchVersionFromNumber(this int value, bool isArchive2) + { + if (isArchive2) + { + return value switch + { + 2 => TTArchiveVersion.Tta2, + 3 => TTArchiveVersion.Tta3, + 4 => TTArchiveVersion.Tta4, + _ => throw new ArgumentOutOfRangeException(nameof(value), value, null) + }; + } + + return value switch + { + 0 => TTArchiveVersion.Legacy, + 1 => TTArchiveVersion.One, + 2 => TTArchiveVersion.Two, + 3 => TTArchiveVersion.Three, + 4 => TTArchiveVersion.Four, + 5 => TTArchiveVersion.Five, + 6 => TTArchiveVersion.Six, + 7 => TTArchiveVersion.Seven, + 8 => TTArchiveVersion.Eight, + 9 => TTArchiveVersion.Nine, + _ => throw new ArgumentOutOfRangeException(nameof(value), value, null) + }; + } + + /// + /// Converts ArchiveVersion to int. + /// + public static int ToJsonNumber(this TTArchiveVersion version) => + version switch + { + TTArchiveVersion.Legacy => 0, + TTArchiveVersion.One => 1, + TTArchiveVersion.Two => 2, + TTArchiveVersion.Three => 3, + TTArchiveVersion.Four => 4, + TTArchiveVersion.Five => 5, + TTArchiveVersion.Six => 6, + TTArchiveVersion.Seven => 7, + TTArchiveVersion.Eight => 8, + TTArchiveVersion.Nine => 9, + TTArchiveVersion.Tta2 => 2, + TTArchiveVersion.Tta3 => 3, + TTArchiveVersion.Tta4 => 4, + _ => throw new ArgumentOutOfRangeException(nameof(version), version, null) + }; +} diff --git a/src/TelltaleToolKit/TelltaleArchives/TelltaleArchive.cs b/src/TelltaleToolKit/TelltaleArchives/TelltaleArchive.cs deleted file mode 100644 index 3aba984..0000000 --- a/src/TelltaleToolKit/TelltaleArchives/TelltaleArchive.cs +++ /dev/null @@ -1,160 +0,0 @@ -using TelltaleToolKit.Utility.Blowfish; - -namespace TelltaleToolKit.TelltaleArchives; - -// TODO: Rework the archive system for v0.X.0. Currently it is very messy due to debugging reasons with no documentation. -// All ttarch implementations are based on ttarchext and TTG Tools. -public abstract class ArchiveBase : IDisposable -{ - protected Stream? ArchiveStream { get; set; } - public ArchiveInfo Info { get; set; } = new(); - public TelltaleFileEntry[] FileEntries { get; set; } = []; - - public void Dispose() - { - ArchiveStream?.Dispose(); - GC.SuppressFinalize(this); - } - - public static T Load(string filePath, string blowFishKey, bool sortEntries = true, bool debugPrint = false) - where T : ArchiveBase, new() - { - FileStream stream = File.OpenRead(filePath); - - if (debugPrint) - { - Toolkit.Instance.Logger.LogInfo($"[TelltaleArchive] Loading {Path.GetFileName(filePath)} from {filePath}"); - } - - return Load(stream, blowFishKey, debugPrint); - } - - public static T Load(Stream stream, string blowFishKey, bool sortEntries = true, bool debugPrint = false) - where T : ArchiveBase, new() - { - var resource = new T(); - resource.InitializeWithStream(stream); - resource.Info.BlowfishKey = blowFishKey ?? throw new ArgumentNullException(nameof(blowFishKey)); - - resource.ReadMetadata(); - - if (sortEntries) - { - resource.SortEntries(); - } - - if (debugPrint) - { - resource.PrintDebug(); - } - - return resource; - } - - public static T Load(string filePath, T3BlowfishKey gameKey, bool sortEntries = true, bool debugPrint = false) - where T : ArchiveBase, new() - { - FileStream stream = File.OpenRead(filePath); - string blowFishKey = gameKey.GetBlowfishKey(); - return Load(stream, blowFishKey, sortEntries, debugPrint); - } - - public static T Load(Stream stream, T3BlowfishKey gameKey, bool sortEntries = true, bool debugPrint = false) - where T : ArchiveBase, new() - { - var archive = new T(); - archive.InitializeWithStream(stream); - archive.Info.BlowfishKey = gameKey.GetBlowfishKey(); - archive.ReadMetadata(); - - if (sortEntries) - { - archive.SortEntries(); - } - - if (debugPrint) - { - archive.PrintDebug(); - } - - return archive; - } - - private void InitializeWithStream(Stream stream) - { - ArchiveStream = stream ?? throw new ArgumentNullException(nameof(stream)); - ArchiveStream.Position = 0; - } - - protected abstract void ReadMetadata(); - public abstract MemoryStream? ExtractFile(string fileName); - public abstract MemoryStream? ExtractFile(ulong crc64); - - public bool ContainsFile(string fileName) - { - return FileEntries.Any(file => file.Name.Equals(fileName, StringComparison.OrdinalIgnoreCase)); - } - - public bool ContainsFile(ulong crc64) - { - return FileEntries.Any(file => file.Crc64 == crc64); - } - - public abstract void ExtractAll(string destinationPath); - - protected bool IsEncrypted() => - Info.Flags.HasFlag(ContainerFlags.IsEncrypted) || - Info.Flags.HasFlag(ContainerFlags.IsModifiedBlowfishEncrypted); - - protected bool IsCompressed() => - Info.Flags.HasFlag(ContainerFlags.IsZlibCompressed) - || Info.Flags.HasFlag(ContainerFlags.IsOodleCompressed) - || Info.Flags.HasFlag(ContainerFlags.IsRawDeflateCompressed); - - protected int FileCount() => FileEntries.Length; - - public TelltaleFileEntry? FindEntry(string name) => - FileEntries.FirstOrDefault(file => file.Name.Equals(name, StringComparison.OrdinalIgnoreCase)); - - public TelltaleFileEntry? FindEntry(ulong crc64) => - FileEntries.FirstOrDefault(file => file.Crc64 == crc64); - - protected void DecryptBlock(Span data, int version, string key, ContainerFlags flags) - { - if (!IsEncrypted()) - { - return; // No encryption, return the original data - } - - Blowfish dec = new(key, version); - dec.Decipher(data, data.Length); - } - - public void SortEntries() - => Array.Sort(FileEntries, (x, y) => string.Compare(x.Name, y.Name, StringComparison.OrdinalIgnoreCase)); - - public void PrintEntries() => FileEntries.ToList().ForEach(Console.WriteLine); - - public void PrintInfo() => - Console.WriteLine( - $""" - Container Version: {Info.ContainerVersion} - Archive Version: {Info.Version} - Flags: {Info.Flags} - Chunk Size: {Info.ChunkSize} - Chunk Count: {Info.ChunkCount} - File Count: {FileCount()} - Blowfish Key: {Info.BlowfishKey} - File Count: {Info.FileCount} - Is ZlibCompressed: {Info.Flags.HasFlag(ContainerFlags.IsZlibCompressed)} - Is RawInflate: {Info.Flags.HasFlag(ContainerFlags.IsRawDeflateCompressed)} - Is Encrypted: {IsEncrypted()} - """ - ); - - private void PrintDebug() - { - PrintInfo(); - PrintEntries(); - } -} diff --git a/src/TelltaleToolKit/TelltaleArchives/TelltaleArchive1.cs b/src/TelltaleToolKit/TelltaleArchives/TelltaleArchive1.cs deleted file mode 100644 index 45ce560..0000000 --- a/src/TelltaleToolKit/TelltaleArchives/TelltaleArchive1.cs +++ /dev/null @@ -1,316 +0,0 @@ -using System.Text; - -namespace TelltaleToolKit.TelltaleArchives; - -// The real name is ttarch which stands for "Telltale Archive". -public class T3Archive : ArchiveBase -{ - /// - /// Reads the oldest available encrypted versions of the archives. - /// - /// - public void ReadLegacyStream(BinaryReader reader) - { - reader.BaseStream.Seek(0, SeekOrigin.Begin); - Info.Version = ArchiveVersion.Zero; - - uint headerSize = reader.ReadUInt32(); - - if (headerSize > 128) - { - Info.Flags |= ContainerFlags.IsEncrypted; - - byte[] header = reader.ReadBytes((int)headerSize); - - DecryptBlock(header, 0, Info.BlowfishKey, Info.Flags); - - ReadEntries(new MemoryStream(header)); - Info.FilesOffset = reader.BaseStream.Position; - } - else - { - reader.BaseStream.Seek(0, SeekOrigin.Begin); - ReadEntries(reader); - Info.FilesOffset = reader.ReadUInt32(); - uint filesSize = reader.ReadUInt32(); - } - } - - public void ReadEntries(Stream stream) - { - using var reader = new BinaryReader(stream, Encoding.UTF8, true); - ReadEntries(reader); - } - - private void ReadEntries(BinaryReader reader) - { - int directoriesCount = reader.ReadInt32(); - for (var i = 0; i < directoriesCount; i++) - { - int directoryNameLength = reader.ReadInt32(); - - if (directoryNameLength > 255) - { - throw new ArgumentOutOfRangeException( - $"Error: directory name length {directoryNameLength} is too long.", nameof(directoryNameLength) - ); - } - - string name = Encoding.ASCII.GetString(reader.ReadBytes(directoryNameLength)); - } - - Info.FileCount = reader.ReadUInt32(); - - FileEntries = new TelltaleFileEntry[Info.FileCount]; - - for (var i = 0; i < Info.FileCount; i++) - { - FileEntries[i] = new TelltaleFileEntry(); - - int filenameLength = reader.ReadInt32(); - if (filenameLength > 255) - { - throw new ArgumentOutOfRangeException( - $"Error: File name length {filenameLength} is too long.", nameof(filenameLength)); - } - - byte[] fileName = reader.ReadBytes(filenameLength); - - FileEntries[i].Name = Encoding.ASCII.GetString(fileName); - _ = reader.ReadInt32(); // always shows 0 value. Probably a way to assign to a folder, in the order of the folder names. - FileEntries[i].FileOffset = reader.ReadUInt32(); - FileEntries[i].FileSize = reader.ReadInt32(); - - FileEntries[i].Crc64 = Utility.Hashing.Crc64.Compute(FileEntries[i].Name); - } - } - - protected override void ReadMetadata() - { - using BinaryReader reader = new(ArchiveStream, Encoding.UTF8, true); - - Info.Version = (ArchiveVersion)reader.ReadUInt32(); - var version = (int)Info.Version; - - if (version is > 9 or < 1) - { - ReadLegacyStream(reader); - return; - } - - uint decryptionMode = reader.ReadUInt32(); - if (decryptionMode > 1) - { - throw new ArgumentOutOfRangeException(nameof(decryptionMode), - "Error: decryptionMode not supported yet"); - } - - Info.Flags |= decryptionMode == 1 ? ContainerFlags.IsEncrypted : ContainerFlags.None; - - // Unknown - uint _ = reader.ReadUInt32(); - - // Files mode 2 is the most common value. No idea what it means. - uint filesMode = version >= 3 ? reader.ReadUInt32() : 0; - if (filesMode > 2) - { - throw new ArgumentOutOfRangeException(nameof(filesMode), - "Error: files_mode {filesMode} is not supported yet"); - } - - // Compression is supported from version 3. - if (version >= 3) - { - Info.ChunkCount = reader.ReadUInt32(); - if (Info.ChunkCount > 100000) - { - throw new ArgumentOutOfRangeException(nameof(Info.ChunkCount), - "Error: Chunk counts are more than expected!"); - } - - if (Info.ChunkCount > 0) - { - // Set both flags on. The decompression function will recognize which one is used. - Info.Flags |= ContainerFlags.IsRawDeflateCompressed | ContainerFlags.IsZlibCompressed; - Info.ChunkBlockSizes = new long[Info.ChunkCount]; - for (var i = 0; i < Info.ChunkCount; i++) - { - Info.ChunkBlockSizes[i] = reader.ReadUInt32(); - } - } - - uint FileDataSize = - reader.ReadUInt32(); // the size of the field where are stored all the files contents - - if (version >= 4) - { - reader.ReadUInt32(); // unknown ? priority ? - reader.ReadUInt32(); - - if (version >= 7) - { - uint xModeValue1 = reader.ReadUInt32(); // unknown - uint xModeValue2 = reader.ReadUInt32(); // unknown - - Info.Flags |= xModeValue1 == 1 || xModeValue2 == 1 - ? ContainerFlags.IsXMode - : ContainerFlags.None; - - const uint blockSize = 1024; - Info.ChunkSize = (int)(reader.ReadUInt32() * blockSize); - if (version >= 8) - { - var t = reader.ReadByte(); // unknown boolean - } - - // FilesMode 2 = compressed - // FilesMode 1 = normal? - // Or maybe filesmode 2 means NO folders, while 1 means there are? - if (version >= 9 && filesMode >= 1) - { - // Another 0? - reader.ReadUInt32(); - } - } - } - } - - int infoHeaderSize = reader.ReadInt32(); // size of the header - - // This is a workaround for some version 8 archives. I...I don't know why. - infoHeaderSize = infoHeaderSize == 0 ? reader.ReadInt32() : infoHeaderSize; - - int compressedInfoHeaderSize = - version >= 7 && filesMode >= 2 ? reader.ReadInt32() : 0; // size of the compressed data - - byte[] header = - version >= 7 && filesMode == 2 - ? reader.ReadBytes(compressedInfoHeaderSize) - : reader.ReadBytes(infoHeaderSize); - - Info.FilesOffset = reader.BaseStream.Position; - - if (IsCompressed() && version >= 7 && filesMode == 2) - { - ContainerFlags newFlags = Info.Flags; - // Headers are not compressed for some reason in earlier versions. - header = TelltaleArchiveUtilities.DecompressBlock(header, infoHeaderSize, ref newFlags); - // Info.Flags = newFlags; - } - - DecryptBlock(header, version, Info.BlowfishKey, Info.Flags); - - // Set the archive offset (info header) - var streamHeader = new MemoryStream(header); - ReadEntries(streamHeader); - } - - - public override void ExtractAll(string destinationPath) - { - throw new NotImplementedException(); - } - - /// - /// Finds the memory stream referenced by the hash inside of this archive - /// - /// Hash - /// A MemoryStream for the file at the hash or null if the file is not found - public override MemoryStream? ExtractFile(ulong crc64) - { - TelltaleFileEntry? entry = FindEntry(crc64); - - return entry == null ? null : ExtractFile(entry); - } - - /// - /// Finds the memory stream referenced by the file name inside of this archive - /// - /// File name - /// A MemoryStream for the file at the file name or null if the file is not found - public override MemoryStream? ExtractFile(string name) - { - TelltaleFileEntry? entry = FindEntry(name); - - return entry == null ? null : ExtractFile(entry); - } - - public MemoryStream ExtractFile(TelltaleFileEntry entry) - { - byte[] result; // Initialize the result array - - int fileSize = entry.FileSize; // Get the file size - - if (!IsCompressed()) - { - ArchiveStream.Seek(Info.FilesOffset + entry.FileOffset, SeekOrigin.Begin); - result = new byte[fileSize]; - var totalRead = 0; - while (totalRead < result.Length) - { - int bytesRead = ArchiveStream.Read(result, totalRead, result.Length - totalRead); - if (bytesRead == 0) - throw new EndOfStreamException($"Unable to read {result.Length} bytes from stream."); - totalRead += bytesRead; - } // Read the file data directly - - TelltaleArchiveUtilities.DecryptFile(result, Info.BlowfishKey, (int)Info.Version); - } - else - { - var blockStartIndex = (int)(entry.FileOffset / Info.ChunkSize); - var blockEndIndex = (int)((entry.FileOffset + fileSize) / Info.ChunkSize); - - if (blockStartIndex < 0) - throw new ArgumentOutOfRangeException(nameof(blockStartIndex), "Block start index is less than 0."); - if (blockStartIndex > (int)Info.ChunkCount) - throw new ArgumentOutOfRangeException(nameof(blockStartIndex), - "Block start index is greater than chunk count."); - if (blockEndIndex > (int)Info.ChunkCount) - throw new ArgumentOutOfRangeException(nameof(blockEndIndex), - "Block end index is greater than chunk count."); - - long blockStartOffset = 0; - - for (var i = 0; i < blockStartIndex; i++) - { - blockStartOffset += Info.ChunkBlockSizes[i]; - } - - ArchiveStream.Seek(Info.FilesOffset + blockStartOffset, - SeekOrigin.Begin); // Seek to the block start offset - - using MemoryStream ms = new(); - for (int i = blockStartIndex; i <= blockEndIndex; i++) - { - var tmp = new byte[Info.ChunkBlockSizes[i]]; - - // Replace ReadExactly with manual reading - var totalRead = 0; - while (totalRead < tmp.Length) - { - int bytesRead = ArchiveStream.Read(tmp, totalRead, tmp.Length - totalRead); - if (bytesRead == 0) - throw new EndOfStreamException($"Unable to read {tmp.Length} bytes from stream."); - totalRead += bytesRead; - } - - DecryptBlock(tmp, (int)Info.Version, Info.BlowfishKey, Info.Flags); - - ContainerFlags containerFlags = Info.Flags; - tmp = TelltaleArchiveUtilities.DecompressBlock(tmp, Info.ChunkSize, ref containerFlags); - - ms.Write(tmp); - } - - byte[] block = ms.ToArray(); - result = new byte[fileSize]; - - // This is relative to the blocks themselves - long startIndex = entry.FileOffset - Info.ChunkSize * blockStartIndex; - Array.Copy(block, startIndex, result, 0, result.Length); - } - - return new MemoryStream(result); - } -} diff --git a/src/TelltaleToolKit/TelltaleArchives/TelltaleArchive2.cs b/src/TelltaleToolKit/TelltaleArchives/TelltaleArchive2.cs deleted file mode 100644 index 4e5866d..0000000 --- a/src/TelltaleToolKit/TelltaleArchives/TelltaleArchive2.cs +++ /dev/null @@ -1,326 +0,0 @@ -using System.Text; - -namespace TelltaleToolKit.TelltaleArchives; - -public class T3Archive2 : ArchiveBase -{ - public void ReadContainer(BinaryReader reader) - { - Info.ContainerVersion = (ContainerVersion)reader.ReadUInt32(); - - // Setup encryption and compression flags - if (Info.ContainerVersion is ContainerVersion.Ttce or ContainerVersion.TtCe) - { - Info.Flags |= ContainerFlags.IsEncrypted; - } - - switch (Info.ContainerVersion) - { - case ContainerVersion.Ttce or ContainerVersion.Ttcz: - Info.Flags |= ContainerFlags.IsRawDeflateCompressed; - break; - case ContainerVersion.TtCe or ContainerVersion.TtCz: - { - uint compressionType = reader.ReadUInt32(); - - Info.Flags = compressionType switch - { - 0 => ContainerFlags.IsRawDeflateCompressed, - 1 => ContainerFlags.IsOodleCompressed, - _ => throw new NotSupportedException($"Unsupported compression type: {compressionType}"), - }; - break; - } - } - - if (IsCompressed()) - { - Info.ChunkSize = reader.ReadInt32(); // Also known as window size. This is the size per page. - if (Info.ChunkSize < 0) - { - throw new InvalidDataException("Invalid chunk size"); - } - - Info.ChunkCount = reader.ReadUInt32(); // Number of chunks in the archive - - Info.ChunkBlockSizes = new long[Info.ChunkCount]; - - long previousChunkOffset = reader.ReadInt64(); // Read the first chunk block - for (var i = 0; i < Info.ChunkCount; i++) - { - long nextChunkOffset = reader.ReadInt64(); - - Info.ChunkBlockSizes[i] = - nextChunkOffset - previousChunkOffset; // Calculate the offset of the chunk block - - previousChunkOffset = nextChunkOffset; // Update the first chunk block offset - } - } - else - { - ulong containerSize = reader.ReadUInt64(); // Read the size of the archive - } - } - - protected override void ReadMetadata() - { - // TODO I am aware that the initial headers are not exclusive to TTArch2, but rather for telltale containers (used for save files) - // I will assume they are specific to TTArch2 for convenience. - using var mainReader = new BinaryReader(ArchiveStream, Encoding.UTF8, true); - - ReadContainer(mainReader); // Read the container header - Info.ArchiveOffset = - mainReader.BaseStream - .Position; // Set the file offset to the current position in the stream (Where TTA4/TTA3 starts) - - byte[] finalFullHeader = []; - uint fileNameBufferSize; - if (IsCompressed()) - { - // Try to read the TTArch2 header - byte[] header = DecodeBlock( - mainReader.ReadBytes((int)Info.ChunkBlockSizes[0]), - Info.ChunkSize, - Info.BlowfishKey, - Info.Flags - ); // Decrypt and decompress the header - - using BinaryReader headerReader = new(new MemoryStream(header)); - Info.Version = (ArchiveVersion)headerReader.ReadUInt32(); // Read the version of the archive - - if (Info.Version is ArchiveVersion.Tta3) - { - headerReader.ReadUInt32(); // Read the size of the header - } - - fileNameBufferSize = headerReader.ReadUInt32(); // Read the size of the file name buffer - if (fileNameBufferSize > 0x10000000) - { - throw new ArgumentOutOfRangeException(nameof(fileNameBufferSize)); - } - - Info.FileCount = headerReader.ReadUInt32(); // Read the number of files in the archive - if (Info.FileCount > 0xFFFFF) - { - throw new ArgumentOutOfRangeException(nameof(Info.FileCount)); - } - - FileEntries = new TelltaleFileEntry[Info.FileCount]; // Initialize the file entries dictionary - - Info.FilesOffset = headerReader.BaseStream.Position + (28 * Info.FileCount) + fileNameBufferSize; - - // Read all needed chunk blocks for the header - if (Info.FilesOffset > headerReader.BaseStream.Length) - { - mainReader.BaseStream.Seek(Info.ArchiveOffset, - SeekOrigin.Begin); // Reset the stream position to the beginning - long chunksToIterate = Info.FilesOffset / Info.ChunkSize + 1; - - using MemoryStream ms = new(); - - for (var i = 0; i < chunksToIterate; i++) - { - byte[] chunk = DecodeBlock( - mainReader.ReadBytes((int)Info.ChunkBlockSizes[i]), - Info.ChunkSize, - Info.BlowfishKey, - Info.Flags - ); - - ms.Write(chunk, 0, chunk.Length); - } - - finalFullHeader = ms.ToArray(); - } - } - - using BinaryReader fullHeader = IsCompressed() ? new(new MemoryStream(finalFullHeader)) : mainReader; - - Info.Version = (ArchiveVersion)fullHeader.ReadUInt32(); // Read the version of the archive - - if (Info.Version is ArchiveVersion.Tta3 or ArchiveVersion.Tta2) - { - uint unknown = fullHeader.ReadUInt32(); // Read the size of the header - } - - fileNameBufferSize = fullHeader.ReadUInt32(); // Read the size of the file name buffer - if (fileNameBufferSize > 0x10000000) - { - throw new ArgumentOutOfRangeException(nameof(fileNameBufferSize)); - } - - Info.FileCount = fullHeader.ReadUInt32(); // Read the number of files in the archive - if (Info.FileCount > 0xFFFFF) - { - throw new ArgumentOutOfRangeException(nameof(Info.FileCount)); - } - - Info.FilesOffset = IsCompressed() - ? Info.FilesOffset - : fullHeader.BaseStream.Position + (28 * Info.FileCount) + fileNameBufferSize; - - FileEntries = new TelltaleFileEntry[Info.FileCount]; // Initialize the file entries dictionary - - var filesAndNamesTotalSize = - (int)((Info.FileCount * 28) + fileNameBufferSize); // Calculate the total size of files and names - // 28 is not a good number - - var nameOffsets = new uint[FileCount()]; // Array to store name offsets - - for (var i = 0; i < FileCount(); i++) - { - FileEntries[i] = new TelltaleFileEntry - { - Crc64 = fullHeader.ReadUInt64(), - FileOffset = fullHeader.ReadInt64(), - }; - - if (Info.Version is ArchiveVersion.Tta2) - { - uint unused = fullHeader.ReadUInt32(); // Read an unused value (possibly a version CRC) - } - - FileEntries[i].FileSize = fullHeader.ReadInt32(); - int unknown = fullHeader.ReadInt32(); // Read an unknown value (possibly a folder index) - ushort nameBlock = fullHeader.ReadUInt16(); // Read the name block offset - ushort nameOffset = fullHeader.ReadUInt16(); // Read the name offset - - nameOffsets[i] = (uint)(nameBlock * 0x10000 + nameOffset); // Calculate the name offset - - // ArgumentOutOfRangeException.ThrowIfGreaterThan(FileEntries[i].NameOffset, fileNameBufferSize, ); // Ensure the name offset is valid - } - - long fileNamesOffset = fullHeader.BaseStream.Position; // position for filename table - - for (var i = 0; i < FileCount(); i++) - { - fullHeader.BaseStream.Seek(fileNamesOffset + nameOffsets[i], SeekOrigin.Begin); // Seek to the name offset - - MemoryStream nameEntry = new(); - byte character; - while ((character = fullHeader.ReadByte()) != 0) - { - nameEntry.WriteByte(character); - } - - FileEntries[i].Name = Encoding.ASCII.GetString(nameEntry.ToArray()); - } - } - - public byte[] DecodeBlock(byte[] data, int expectedBufferSize, string key, ContainerFlags flags) - { - if (!IsEncrypted() && !IsCompressed()) - { - return data; // No compression or encryption, return the original data - } - - DecryptBlock(data, 7, key, flags); - byte[] decompressedData = TelltaleArchiveUtilities.DecompressBlock(data, expectedBufferSize, ref flags); - return decompressedData; - } - - /// - /// Finds the memory stream referenced by the hash inside of this archive - /// - /// Hash - /// A MemoryStream for the file at the hash or null if the file is not found - public override MemoryStream? ExtractFile(ulong crc64) - { - TelltaleFileEntry? entry = FindEntry(crc64); - - return entry == null ? null : ExtractFile(entry); - } - - /// - /// Finds the memory stream referenced by the file name inside of this archive - /// - /// File name - /// A MemoryStream for the file at the file name or null if the file is not found - public override MemoryStream? ExtractFile(string name) - { - TelltaleFileEntry? entry = FindEntry(name); - - return entry == null ? null : ExtractFile(entry); - } - - public override void ExtractAll(string destinationPath) - { - throw new NotImplementedException(); - } - - public MemoryStream ExtractFile(TelltaleFileEntry entry) - { - using BinaryReader reader = new(ArchiveStream, Encoding.UTF8, true); - - long fileOffset = Info.FilesOffset + entry.FileOffset; // Calculate the file offset - int fileSize = entry.FileSize; // Get the file size - - MemoryStream result = new(fileSize); - BinaryWriter resultWriter = new(result); - - if (IsCompressed()) - { - var blockStartIndex = (int)(fileOffset / Info.ChunkSize); - var blockEndIndex = (int)((fileOffset + fileSize) / Info.ChunkSize); - - if (blockStartIndex < 0) - throw new ArgumentOutOfRangeException(nameof(blockStartIndex), "Block start index is less than 0."); - if (blockStartIndex > (int)Info.ChunkCount) - throw new ArgumentOutOfRangeException(nameof(blockStartIndex), - "Block start index is greater than chunk count."); - if (blockEndIndex > (int)Info.ChunkCount) - throw new ArgumentOutOfRangeException(nameof(blockEndIndex), - "Block end index is greater than chunk count."); - - long blockStartOffset = 0; - - for (var i = 0; i < blockStartIndex; i++) - { - blockStartOffset += Info.ChunkBlockSizes[i]; - } - - // Seek to the block start offset - reader.BaseStream.Seek(Info.ArchiveOffset + blockStartOffset, SeekOrigin.Begin); - - using MemoryStream ms = new(); - using BinaryWriter blockWriter = new(ms); - for (int blockIndex = blockStartIndex; blockIndex <= blockEndIndex; blockIndex++) - { - byte[] chunk = DecodeBlock( - reader.ReadBytes((int)Info.ChunkBlockSizes[blockIndex]), - Info.ChunkSize, - Info.BlowfishKey, - Info.Flags - ); // Read the chunk block - - blockWriter.Write(chunk, 0, chunk.Length); - } - - // This is relative to the blocks themselves - long startIndex = fileOffset - (Info.ChunkSize * blockStartIndex); - resultWriter.Write(ms.ToArray(), (int)startIndex, fileSize); // Write the data to the result stream - } - else - { - reader.BaseStream.Seek(fileOffset, SeekOrigin.Begin); - - // Read in chunks to avoid LOH allocations - const int bufferSize = 81920; // 80KB - under LOH threshold - int bytesRemaining = fileSize; - var buffer = new byte[Math.Min(bufferSize, bytesRemaining)]; - - while (bytesRemaining > 0) - { - int bytesToRead = Math.Min(buffer.Length, bytesRemaining); - int bytesRead = reader.Read(buffer, 0, bytesToRead); - - if (bytesRead == 0) break; - - result.Write(buffer, 0, bytesRead); - bytesRemaining -= bytesRead; - } - } - - return result; - } -} \ No newline at end of file diff --git a/src/TelltaleToolKit/TelltaleArchives/TelltaleArchiveUtilities.cs b/src/TelltaleToolKit/TelltaleArchives/TelltaleArchiveUtilities.cs index 79c69fc..6f0adc7 100644 --- a/src/TelltaleToolKit/TelltaleArchives/TelltaleArchiveUtilities.cs +++ b/src/TelltaleToolKit/TelltaleArchives/TelltaleArchiveUtilities.cs @@ -11,14 +11,26 @@ public static class TelltaleArchiveUtilities public static void DecryptFile(byte[] file, string blowfishKey, int archiveVersion) { if (file.Length < 4) + { return; + } + + DecryptFile(file, new Blowfish(blowfishKey, archiveVersion)); + } - var magicFourCc = (MetaStreamVersion)BitConverter.ToUInt32(file, 0); + public static void DecryptFile(byte[] file, Blowfish blowfish) + { + if (file.Length < 4) + { + return; + } + + MetaStreamVersion magicFourCc = (MetaStreamVersion)BitConverter.ToUInt32(file, 0); BlockConfiguration config = GetBlockConfiguration(magicFourCc); if (config.blockSize > 0) { - DecryptFileHelper(file, blowfishKey, archiveVersion, config); + DecryptFileHelper(file, blowfish, config); } } @@ -29,13 +41,11 @@ private static BlockConfiguration GetBlockConfiguration(MetaStreamVersion type) MetaStreamVersion.Unknown1 or MetaStreamVersion.Unknown2 or MetaStreamVersion.Unknown3 => (128, 32, 80), MetaStreamVersion.Unknown4 => (256, 8, 24), MetaStreamVersion.Mbes => (64, 64, 100), - _ => (0, 0, 0), + _ => (0, 0, 0) }; - private static void DecryptFileHelper(byte[] fileData, string key, int archiveVersion, BlockConfiguration config) + private static void DecryptFileHelper(byte[] fileData, Blowfish blowFish, BlockConfiguration config) { - var blowFish = new Blowfish(key, archiveVersion); - // If the blocksize is 0 (Stream version is not existent), use regular blowfish if (config.blockSize == 0) { @@ -48,9 +58,9 @@ private static void DecryptFileHelper(byte[] fileData, string key, int archiveVe long blocks = (fileData.Length - 4) / config.blockSize; - for (var i = 0; i < blocks; i++) + for (int i = 0; i < blocks; i++) { - var block = new Span(fileData, 4 + i * config.blockSize, config.blockSize); + Span block = new(fileData, 4 + i * config.blockSize, config.blockSize); if (i % config.cryptInterval == 0) { @@ -69,77 +79,42 @@ private static void DecryptFileHelper(byte[] fileData, string key, int archiveVe public static void XorBlock(Span block, byte xor) { - for (var i = 0; i < block.Length; i++) + for (int i = 0; i < block.Length; i++) { block[i] ^= xor; } } - public static byte[] DecompressBlock(byte[] compressedData, int expectedSize, ref ContainerFlags flags) + private static CompressionAlgorithm DetectCompressionAlgorithm(byte[] chunk) { - // This is a hack? In TWD:DE, 401_txmesh, there's a page which is the same size as the expected size (65535) - // C#'s raw deflate fails. - // ttarchext returns the same data: - // https://github.com/infernokun/TelltaleGamesExtractionGUI/blob/fec9fc1a70b545bcc67062fedc3fe3f7cd0d3e1b/bin/Debug/net6.0-windows/ttarchext/ttarchext.c#L1512 - if (compressedData.Length == expectedSize) - return compressedData; - - // Try both Deflate and Zlib if both flags are set - // TODO: Investigate ttarch - // CSI games use zlib compression instead of the regular raw deflate. - // According to TTG Tools, version 8 archives use zlib (presumably for the files themselves) - if (flags.HasFlag(ContainerFlags.IsRawDeflateCompressed) && flags.HasFlag(ContainerFlags.IsZlibCompressed)) + // Zlib header: first byte 0x78, second byte in {0x9C, 0xDA, 0x01, 0x5E, 0x9E, ...} + if (chunk[0] == 0x78 && (chunk[1] & 0xF0) == 0xC0) // 0x78 0x9C, 0x78 0xDA, etc. { - try - { - byte[] result = Decompress(ms => new DeflateStream(ms, CompressionMode.Decompress)); - flags &= ~ContainerFlags.IsZlibCompressed; - return result; - } - catch (Exception) - { - // flags &= ~ContainerFlags.IsRawDeflateCompressed; - } - - try - { - byte[] result = Decompress(ms => new InflaterInputStream(ms)); - return result; - } - catch (Exception) - { - // flags &= ~ContainerFlags.IsZlibCompressed; - } - - throw new InvalidDataException("Compresion not supported"); + return CompressionAlgorithm.Zlib; } - if (flags.HasFlag(ContainerFlags.IsRawDeflateCompressed)) - { - return Decompress(ms => new DeflateStream(ms, CompressionMode.Decompress)); - } + return CompressionAlgorithm.Deflate; // raw Deflate has no header + } - if (flags.HasFlag(ContainerFlags.IsZlibCompressed)) + /// + /// Compresses a single chunk using the specified algorithm. + /// + /// Number of compressed bytes written into . + public static int CompressBlock(ReadOnlySpan input, CompressionAlgorithm algorithm, byte[] outputBuffer) + { + using MemoryStream ms = new(outputBuffer, true); + if (algorithm == CompressionAlgorithm.Zlib) { - return Decompress(ms => new InflaterInputStream(ms)); + using DeflaterOutputStream zlib = new(ms); + zlib.Write(input); + zlib.Finish(); } - - if (flags.HasFlag(ContainerFlags.IsOodleCompressed)) + else // Deflate (raw) { - // TODO: Add Oodle compression. - throw new NotSupportedException("Oodle compression is not supported yet."); + using DeflateStream deflate = new(ms, CompressionLevel.Optimal, true); + deflate.Write(input); } - // No compression, return the original data - return compressedData; - - // Local function to handle decompression - byte[] Decompress(Func streamFactory) - { - using var outputStream = new MemoryStream(expectedSize); - using Stream decompressStream = streamFactory(new MemoryStream(compressedData)); - decompressStream.CopyTo(outputStream); - return outputStream.ToArray(); - } + return (int)ms.Position; } -} \ No newline at end of file +} diff --git a/src/TelltaleToolKit/TelltaleArchives/TelltaleArchiveVersionExtensions.cs b/src/TelltaleToolKit/TelltaleArchives/TelltaleArchiveVersionExtensions.cs deleted file mode 100644 index 9d369cd..0000000 --- a/src/TelltaleToolKit/TelltaleArchives/TelltaleArchiveVersionExtensions.cs +++ /dev/null @@ -1,60 +0,0 @@ -namespace TelltaleToolKit.TelltaleArchives; - -public static class TelltaleArchiveVersionExtensions -{ - /// - /// Converts an int to ArchiveVersion. - /// - public static ArchiveVersion TtarchVersionFromNumber(this int value, bool isArchive2) - { - if (isArchive2) - { - return value switch - { - 2 => ArchiveVersion.Tta2, - 3 => ArchiveVersion.Tta3, - 4 => ArchiveVersion.Tta4, - _ => throw new ArgumentOutOfRangeException(nameof(value), value, null) - }; - } - - return value switch - { - 0 => ArchiveVersion.Zero, - 1 => ArchiveVersion.One, - 2 => ArchiveVersion.Two, - 3 => ArchiveVersion.Three, - 4 => ArchiveVersion.Four, - 5 => ArchiveVersion.Five, - 6 => ArchiveVersion.Six, - 7 => ArchiveVersion.Seven, - 8 => ArchiveVersion.Eight, - 9 => ArchiveVersion.Nine, - _ => throw new ArgumentOutOfRangeException(nameof(value), value, null) - }; - } - - /// - /// Converts ArchiveVersion to int. - /// - public static int ToJsonNumber(this ArchiveVersion version) - { - return version switch - { - ArchiveVersion.Zero => 0, - ArchiveVersion.One => 1, - ArchiveVersion.Two => 2, - ArchiveVersion.Three => 3, - ArchiveVersion.Four => 4, - ArchiveVersion.Five => 5, - ArchiveVersion.Six => 6, - ArchiveVersion.Seven => 7, - ArchiveVersion.Eight => 8, - ArchiveVersion.Nine => 9, - ArchiveVersion.Tta2 => 2, - ArchiveVersion.Tta3 => 3, - ArchiveVersion.Tta4 => 4, - _ => throw new ArgumentOutOfRangeException(nameof(version), version, null) - }; - } -} \ No newline at end of file diff --git a/src/TelltaleToolKit/TelltaleArchives/TelltaleContainers/StreamContainer.cs b/src/TelltaleToolKit/TelltaleArchives/TelltaleContainers/StreamContainer.cs deleted file mode 100644 index 1273c9d..0000000 --- a/src/TelltaleToolKit/TelltaleArchives/TelltaleContainers/StreamContainer.cs +++ /dev/null @@ -1,66 +0,0 @@ -namespace TelltaleToolKit.TelltaleArchives.TelltaleContainers; - -public class DataStreamContainer : Stream -{ - // Public properties for settings - public bool IsCompressed { get; } - public bool IsEncrypted { get; } - public CompressionLibrary CompressionLibrary { get; } - - // Internal state - private Stream _sourceStream; - private long _containerSize; - private long _position; - - // Constructors: just take what you need - public DataStreamContainer(Stream source, bool isCompressed = false, bool isEncrypted = false, CompressionLibrary clib = CompressionLibrary.None) - { - _sourceStream = source; - IsCompressed = isCompressed; - IsEncrypted = isEncrypted; - CompressionLibrary = clib; - _containerSize = source.Length; - _position = 0; - } - - // Implement Stream abstract members - public override bool CanRead => true; - public override bool CanSeek => true; - public override bool CanWrite => false; - public override long Length => _containerSize; - public override long Position - { - get => _position; - set => _position = value; - } - - public override int Read(byte[] buffer, int offset, int count) - { - // Add decompression/decryption as needed here - _sourceStream.Position = _position; - int bytesRead = _sourceStream.Read(buffer, offset, count); - _position += bytesRead; - return bytesRead; - } - - public override long Seek(long offset, SeekOrigin origin) - { - switch (origin) - { - case SeekOrigin.Begin: _position = offset; break; - case SeekOrigin.Current: _position += offset; break; - case SeekOrigin.End: _position = _containerSize + offset; break; - } - return _position; - } - - public override void SetLength(long value) => throw new NotSupportedException(); - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - public override void Flush() { } -} - -public enum CompressionLibrary -{ - None, - // Add more as needed -} \ No newline at end of file diff --git a/src/TelltaleToolKit/TelltaleArchives/TelltaleFileEntry.cs b/src/TelltaleToolKit/TelltaleArchives/TelltaleFileEntry.cs deleted file mode 100644 index 3fc1cdc..0000000 --- a/src/TelltaleToolKit/TelltaleArchives/TelltaleFileEntry.cs +++ /dev/null @@ -1,16 +0,0 @@ -namespace TelltaleToolKit.TelltaleArchives; - -public class TelltaleFileEntry -{ - public string Name { get; set; } = string.Empty; - public ulong Crc64 { get; set; } - public long FileOffset { get; set; } // The offset starts when the actual files are written. - public int FileSize { get; set; } - - public override string ToString() - { - return Crc64 == 0 - ? $"{Name} (Offset: {FileOffset}, Size: {FileSize})" - : $"{Name} (CRC: {Crc64:X16}, Offset: {FileOffset}, Size: {FileSize})"; - } -} \ No newline at end of file diff --git a/src/TelltaleToolKit/TelltaleToolKit.csproj b/src/TelltaleToolKit/TelltaleToolKit.csproj index a6b2d1d..7a3d1e0 100644 --- a/src/TelltaleToolKit/TelltaleToolKit.csproj +++ b/src/TelltaleToolKit/TelltaleToolKit.csproj @@ -42,4 +42,20 @@ + + + + + + + + + + + + + + + + diff --git a/src/TelltaleToolKit/Toolkit.cs b/src/TelltaleToolKit/Toolkit.cs index c46433b..5761f46 100644 --- a/src/TelltaleToolKit/Toolkit.cs +++ b/src/TelltaleToolKit/Toolkit.cs @@ -7,6 +7,7 @@ using TelltaleToolKit.Serialization.Binary; using TelltaleToolKit.T3Types; using TelltaleToolKit.TelltaleArchives; +using TelltaleToolKit.TelltaleArchives.Formats; using TelltaleToolKit.Utility.Blowfish; namespace TelltaleToolKit; @@ -385,29 +386,25 @@ public void ResolveSymbols(IEnumerable symbols) /// /// Path to the archive file. /// Enum value identifying the game's blowfish key. - /// When , sorts entries for faster binary search. - /// When , prints diagnostic output to stdout. - /// The loaded . + /// The loaded . /// Thrown when the file extension is not recognized. - public ArchiveBase LoadArchive(string archivePath, T3BlowfishKey game, bool sort = true, bool debugPrint = false) - => LoadArchive(archivePath, game.GetBlowfishKey(), sort, debugPrint); + public Archive LoadArchive(string archivePath, T3BlowfishKey game) + => LoadArchive(archivePath, game.GetBlowfishKey()); /// /// Loads a Telltale archive file (.ttarch or .ttarch2) using a raw blowfish key string. /// /// Path to the archive file. /// The decryption key for this game's archives. - /// When , sorts entries for faster binary search. - /// When , prints diagnostic output to stdout. - /// The loaded . + /// The loaded . /// Thrown when the file extension is not .ttarch or .ttarch2. - public ArchiveBase LoadArchive(string archivePath, string blowfishKey, bool sort = true, bool debugPrint = false) + public Archive LoadArchive(string archivePath, string blowfishKey) { if (archivePath.EndsWith(".ttarch2", StringComparison.OrdinalIgnoreCase)) - return ArchiveBase.Load(archivePath, blowfishKey, sort, debugPrint); + return Archive.Load(archivePath, blowfishKey); if (archivePath.EndsWith(".ttarch", StringComparison.OrdinalIgnoreCase)) - return ArchiveBase.Load(archivePath, blowfishKey, sort, debugPrint); + return Archive.Load(archivePath, blowfishKey); throw new NotSupportedException( $"Unsupported archive format '{Path.GetExtension(archivePath)}'. Expected .ttarch or .ttarch2."); diff --git a/src/TelltaleToolKit/Workspace.cs b/src/TelltaleToolKit/Workspace.cs index 7074654..bf196e0 100644 --- a/src/TelltaleToolKit/Workspace.cs +++ b/src/TelltaleToolKit/Workspace.cs @@ -138,12 +138,12 @@ public LuaState ResdecLuaState /// Raised after an archive has been loaded and is ready for use. /// Subscribe to track which archives are open, e.g., for progress reporting. /// - public event Action? ArchiveLoaded; + public event Action? ArchiveLoaded; /// /// Raised just before an archive is unloaded and its resources released. /// - public event Action? ArchiveUnloaded; + public event Action? ArchiveUnloaded; #region Version 1: Game Folder Mounting @@ -343,7 +343,7 @@ private async Task ParseResourceDescriptionAsync(string descPath) /// /// Loads a Telltale archive from disk, fires the event, - /// and returns the raw . + /// and returns the raw . /// /// /// Prefer the overload that returns a when you want the @@ -351,12 +351,10 @@ private async Task ParseResourceDescriptionAsync(string descPath) /// need to pass the raw archive to a lower-level API. /// /// Absolute or relative path to the .ttarch / .ttarch2 file. - /// Whether to sort the archive's entry table for faster binary search. - /// When , prints loading diagnostics to stdout. - /// The loaded . - public ArchiveBase LoadArchive(string archivePath, bool sort = true, bool debugPrint = false) + /// The loaded . + public Archive LoadArchive(string archivePath) { - ArchiveBase archive = _toolkit.LoadArchive(archivePath, BlowfishKey, sort, debugPrint); + Archive archive = _toolkit.LoadArchive(archivePath, BlowfishKey); ArchiveLoaded?.Invoke(archive); return archive; } @@ -698,13 +696,13 @@ public bool ContainsFile(string name) /// searching enabled contexts from the highest priority to lowest. /// /// The entry, or if not found. - public TelltaleFileEntry? FindFileEntry(ulong crc64) + public ResourceEntry? FindFileEntry(ulong crc64) { for (int i = _contexts.Count - 1; i >= 0; i--) { if (!_contexts[i].IsEnabled) continue; - TelltaleFileEntry? entry = _contexts[i].GetFileEntry(crc64); + ResourceEntry? entry = _contexts[i].GetFileEntry(crc64); if (entry != null) return entry; } @@ -745,7 +743,7 @@ public bool ContainsFile(string name) /// /// A flat sequence of objects from all enabled contexts. /// - public IEnumerable GetAllEntries() + public IEnumerable GetAllEntries() => _contexts.Where(c => c.IsEnabled).SelectMany(c => c.GetAllEntries()); #endregion @@ -797,7 +795,7 @@ public bool ResolveSymbol(Symbol? symbol) if (_toolkit.ResolveSymbol(symbol)) return true; if (LocalHashDatabase.ResolveSymbol(symbol)) return true; - TelltaleFileEntry? fileEntry = FindFileEntry(symbol.Crc64); + ResourceEntry? fileEntry = FindFileEntry(symbol.Crc64); if (fileEntry?.Name != null) { symbol.Resolve(fileEntry.Name); diff --git a/tools/TTKDatabaseGenerator/Program.cs b/tools/TTKDatabaseGenerator/Program.cs index ad13457..fdfea80 100644 --- a/tools/TTKDatabaseGenerator/Program.cs +++ b/tools/TTKDatabaseGenerator/Program.cs @@ -136,7 +136,7 @@ static async Task Main(string[] args) var areSymbolsHashed = true; var msv = MetaStreamVersion.Mtre; - ArchiveVersion ttarchVersion = 0; + TTArchiveVersion ttarchVersion = 0; var processed = 0; int total = archivePaths.Length; @@ -150,17 +150,17 @@ await Parallel.ForEachAsync(archivePaths, parallelOptions, async (filePath, _) = // PrintInfo($"Processing... {Path.GetFileName(filePath)}"); Log($"Reading {filePath}"); - using ArchiveBase archive = customKey != null - ? Toolkit.Instance.LoadArchive(filePath, customKey, false) - : Toolkit.Instance.LoadArchive(filePath, key!.Value, false); + using Archive archive = customKey != null + ? Toolkit.Instance.LoadArchive(filePath, customKey) + : Toolkit.Instance.LoadArchive(filePath, key!.Value); ttarchVersion = archive.Info.Version; - if (archive.FileEntries.Length > 0) + if (archive.Entries.Count > 0) { - using MemoryStream? firstFile = archive.ExtractFile(archive.FileEntries[0].Name); + await using Stream? firstFile = archive.OpenResource(archive.Entries[0].Name); - if (Toolkit.IsMetaFile(archive.FileEntries[0].Name)) + if (Toolkit.IsMetaFile(archive.Entries[0].Name)) { MetaStreamConfiguration config = new MetaStreamReader(firstFile).Configuration; areSymbolsHashed = config.AreSymbolsHashed; @@ -168,9 +168,9 @@ await Parallel.ForEachAsync(archivePaths, parallelOptions, async (filePath, _) = } } - foreach (TelltaleFileEntry entry in archive.FileEntries) + foreach (ResourceEntry entry in archive.Entries.Values) { - using MemoryStream? file = archive.ExtractFile(entry.Name); + await using Stream? file = archive.OpenResource(entry.Name); if (!Toolkit.IsMetaFile(file)) continue;