From b000f5ebf3e068536e163c827e7f144536b88976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oscar=20Hellstr=C3=B6m?= Date: Fri, 29 Aug 2025 21:31:54 +0200 Subject: [PATCH] Fix intermittent XZ errors Replacing the custom NativeMethods P/Invoke code with Joveler.Compression.XZ. This seem to have gotten rid of my intermittent failures with large files. The Joveler.Compression.XZ doesn't expose lzma_stream_footer_decode (and the other functions used to read the uncompressed size). Instead this code is implemented in C#. It would of course be possible to add this to Joveler.Compression.XZ... This approach also makes this library have less code to maintain. --- .../DebTaskIntegrationTests.cs | 295 +++++++++++++ .../IO/XZOutputStreamTests.cs | 8 - Packaging.Targets/IO/LzmaAction.cs | 136 ------ Packaging.Targets/IO/LzmaCheck.cs | 34 -- Packaging.Targets/IO/LzmaDecodeFlags.cs | 47 -- Packaging.Targets/IO/LzmaMT.cs | 154 ------- Packaging.Targets/IO/LzmaResult.cs | 230 ---------- Packaging.Targets/IO/LzmaStream.cs | 121 ------ Packaging.Targets/IO/LzmaStreamFlags.cs | 98 ----- Packaging.Targets/IO/NativeMethods.cs | 407 ------------------ Packaging.Targets/IO/XZHelper.cs | 24 ++ Packaging.Targets/IO/XZInputStream.cs | 325 +++++--------- Packaging.Targets/IO/XZOutputStream.cs | 288 +++---------- Packaging.Targets/Packaging.Targets.csproj | 3 +- 14 files changed, 478 insertions(+), 1692 deletions(-) create mode 100644 Packaging.Targets.Tests/DebTaskIntegrationTests.cs delete mode 100644 Packaging.Targets/IO/LzmaAction.cs delete mode 100644 Packaging.Targets/IO/LzmaCheck.cs delete mode 100644 Packaging.Targets/IO/LzmaDecodeFlags.cs delete mode 100644 Packaging.Targets/IO/LzmaMT.cs delete mode 100644 Packaging.Targets/IO/LzmaResult.cs delete mode 100644 Packaging.Targets/IO/LzmaStream.cs delete mode 100644 Packaging.Targets/IO/LzmaStreamFlags.cs delete mode 100644 Packaging.Targets/IO/NativeMethods.cs create mode 100644 Packaging.Targets/IO/XZHelper.cs diff --git a/Packaging.Targets.Tests/DebTaskIntegrationTests.cs b/Packaging.Targets.Tests/DebTaskIntegrationTests.cs new file mode 100644 index 00000000..c85b9305 --- /dev/null +++ b/Packaging.Targets.Tests/DebTaskIntegrationTests.cs @@ -0,0 +1,295 @@ +using System; +using System.IO; +using System.Text; +using System.Collections; +using System.Collections.Generic; +using Xunit; +using Packaging.Targets; +using Packaging.Targets.Deb; +using Packaging.Targets.IO; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System.Security.Cryptography; + +namespace Packaging.Targets.Tests.Deb +{ + public class DebTaskIntegrationTests : IDisposable + { + private readonly string tempDirectory; + private readonly string publishDir; + private readonly string outputDir; + + public DebTaskIntegrationTests() + { + // Create a temporary directory that will be cleaned up automatically + tempDirectory = Path.Combine(Path.GetTempPath(), "DebTaskTest_" + Guid.NewGuid().ToString("N")[..8]); + Directory.CreateDirectory(tempDirectory); + + publishDir = Path.Combine(tempDirectory, "publish"); + outputDir = Path.Combine(tempDirectory, "output"); + + Directory.CreateDirectory(publishDir); + Directory.CreateDirectory(outputDir); + } + + public void Dispose() + { + // Cleanup temporary directory + if (Directory.Exists(tempDirectory)) + { + try + { + Directory.Delete(tempDirectory, recursive: true); + } + catch + { + // Best effort cleanup - don't fail test if cleanup fails + } + } + } + + [Fact] + public void CreateAndVerifyRealDebPackage() + { + // Fake .deb contents to validate against + var testFiles = new Dictionary + { + ["usr/bin/myapp"] = "#!/bin/bash\necho 'Hello from myapp'\nexit 0\n", + ["usr/share/myapp/config.json"] = "{\"name\": \"myapp\", \"version\": \"1.0.0\", \"enabled\": true}", + ["usr/share/myapp/data.txt"] = "This is some test data\nLine 2\nLine 3\nEnd of file\n", + ["etc/myapp/settings.conf"] = "# Configuration file\nlog_level=INFO\nport=8080\n", + ["usr/share/doc/myapp/README.md"] = "# MyApp\n\nThis is a test application for .deb packaging.\n\n## Usage\n\nRun with `myapp`.\n", + ["usr/share/myapp/large_content_file1.bin"] = getCompressableString(1024 * 1024 * 100), + ["usr/share/myapp/large_content_file2.bin"] = getCompressableString(1024 * 1024 * 100), + }; + + CreateTestFiles(testFiles); + + // Create Content items for DebTask (this is what MSBuild would normally provide) + var contentItems = CreateContentItems(testFiles); + + var debPath = Path.Combine(outputDir, "myapp.deb"); + var debTarPath = Path.Combine(outputDir, "myapp.tar"); + var debTarXzPath = Path.Combine(outputDir, "myapp.tar.xz"); + + var debTask = new DebTask + { + PublishDir = publishDir, + DebPath = debPath, + DebTarPath = debTarPath, + DebTarXzPath = debTarXzPath, + Prefix = "/", + Version = "1.0.0-test", + PackageName = "myapp", + Content = contentItems, + Maintainer = "Test Suite ", + Description = "Test package for DebTask integration test", + RuntimeIdentifier = "linux-x64", + DebPackageArchitecture = "amd64", + Section = "utils", + Homepage = "https://example.com/myapp", + Priority = "optional", + BuildEngine = new MockBuildEngine() + }; + + Assert.True(debTask.Execute(), "DebTask.Execute() should succeed"); + Assert.True(File.Exists(debPath), $"DEB file should exist at {debPath}"); + Assert.True(File.Exists(debTarPath), $"TAR file should exist at {debTarPath}"); + Assert.True(File.Exists(debTarXzPath), $"TAR.XZ file should exist at {debTarXzPath}"); + + var debInfo = new FileInfo(debPath); + Assert.True(debInfo.Length > 1000, "DEB file should have substantial content"); + + // Verify XZ compression worked (compressed file should be smaller than TAR) + var tarInfo = new FileInfo(debTarPath); + var tarXzInfo = new FileInfo(debTarXzPath); + Assert.True(tarXzInfo.Length < tarInfo.Length, "XZ compressed file should be smaller than TAR"); + Assert.True(tarXzInfo.Length > 0, "XZ compressed file should have content"); + + // Read the .deb package back and verify we can extract the payload + using var debStream = File.OpenRead(debPath); + var package = DebPackageReader.Read(debStream); + Assert.NotNull(package); + + // Extract and verify the compressed payload can be read + var extractedFiles = ExtractDebPayload(debPath); + foreach (var kvp in testFiles) + { + // TAR paths can have ./path or .//path format - find the file by checking both variants + var expectedPath1 = "./" + kvp.Key; + var expectedPath2 = ".//" + kvp.Key; + + string actualPath = null; + if (extractedFiles.ContainsKey(expectedPath1)) + { + actualPath = expectedPath1; + } + else if (extractedFiles.ContainsKey(expectedPath2)) + { + actualPath = expectedPath2; + } + + Assert.True(actualPath != null, $"File {expectedPath1} or {expectedPath2} should be in the DEB payload. Found files: {string.Join(", ", extractedFiles.Keys)}"); + Assert.Equal(kvp.Value, extractedFiles[actualPath]); + } + + Assert.True(extractedFiles.Count >= testFiles.Count, + $"Should have at least {testFiles.Count} files, but found {extractedFiles.Count}"); + } + + private void CreateTestFiles(Dictionary testFiles) + { + foreach (var kvp in testFiles) + { + var fullPath = Path.Combine(publishDir, kvp.Key); + var directory = Path.GetDirectoryName(fullPath); + + if (!Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + + File.WriteAllText(fullPath, kvp.Value, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); + } + } + + private string getCompressableString(int length) { + var bytes = new byte[length / 4]; + using (RandomNumberGenerator rng = RandomNumberGenerator.Create()) + { + rng.GetBytes(bytes); + } + var data = Convert.ToBase64String(bytes); + data += new string('a', length/4); + data += new string('b', length/4); + data += new string('c', length/4); + return data; + } + + private void CreateTarFile(string sourceDir, string tarPath) + { + var entries = new List(); + CreateArchiveEntries(sourceDir, "", entries); + + using var tarStream = File.Create(tarPath); + TarFileCreator.FromArchiveEntries(entries, tarStream); + } + + private void CreateArchiveEntries(string baseDir, string relativePath, List entries) + { + var currentDir = Path.Combine(baseDir, relativePath); + + foreach (var file in Directory.GetFiles(currentDir)) + { + var fileName = Path.GetFileName(file); + var tarPath = string.IsNullOrEmpty(relativePath) ? $"/{fileName}" : $"/{relativePath}/{fileName}"; + + var fileInfo = new FileInfo(file); + var entry = new ArchiveEntry + { + TargetPath = tarPath, + SourceFilename = file, + FileSize = (uint)fileInfo.Length, + Mode = LinuxFileMode.S_IRUSR | LinuxFileMode.S_IWUSR | LinuxFileMode.S_IRGRP | LinuxFileMode.S_IROTH, // 644 + Modified = fileInfo.LastWriteTimeUtc, + Owner = "root", + Group = "root" + }; + + entries.Add(entry); + } + + foreach (var dir in Directory.GetDirectories(currentDir)) + { + var dirName = Path.GetFileName(dir); + var newRelativePath = string.IsNullOrEmpty(relativePath) ? dirName : $"{relativePath}/{dirName}"; + + CreateArchiveEntries(baseDir, newRelativePath, entries); + } + } + + private Dictionary ExtractTarPayload(Stream tarStream) + { + var files = new Dictionary(); + + using var tarFile = new TarFile(tarStream, leaveOpen: true); + + while (tarFile.Read()) + { + // Skip directories and other non-regular files - check if we can read the file + if (!string.IsNullOrEmpty(tarFile.FileName) && ((TarHeader)tarFile.FileHeader).FileSize > 0) + { + try + { + using var entryStream = tarFile.Open(); + using var memoryStream = new MemoryStream(); + entryStream.CopyTo(memoryStream); + + var content = Encoding.UTF8.GetString(memoryStream.ToArray()); + files[tarFile.FileName] = content; + } + catch + { + // Skip files that can't be read (might be directories or special files) + } + } + } + + return files; + } + + private ITaskItem[] CreateContentItems(Dictionary testFiles) + { + var items = new List(); + + foreach (var kvp in testFiles) + { + var item = new TaskItem(Path.Combine(publishDir, kvp.Key)); + item.SetMetadata("TargetPath", kvp.Key); + items.Add(item); + } + + return items.ToArray(); + } + + private Dictionary ExtractDebPayload(string debPath) + { + using var debStream = File.OpenRead(debPath); + + // Get the decompressed payload stream (data.tar) + using var payloadStream = DebPackageReader.GetPayloadStream(debStream); + + // Extract TAR contents + return ExtractTarPayload(payloadStream); + } + } + + internal class MockBuildEngine : IBuildEngine + { + public bool ContinueOnError => false; + public int LineNumberOfTaskNode => 0; + public int ColumnNumberOfTaskNode => 0; + public string ProjectFileOfTaskNode => ""; + + public bool BuildProjectFile(string projectFileName, string[] targetNames, IDictionary globalProperties, IDictionary targetOutputs) + { + return true; + } + + public void LogCustomEvent(CustomBuildEventArgs e) + { + } + + public void LogErrorEvent(BuildErrorEventArgs e) + { + } + + public void LogMessageEvent(BuildMessageEventArgs e) + { + } + + public void LogWarningEvent(BuildWarningEventArgs e) + { + } + } +} diff --git a/Packaging.Targets.Tests/IO/XZOutputStreamTests.cs b/Packaging.Targets.Tests/IO/XZOutputStreamTests.cs index 15a90527..e4bb92c2 100644 --- a/Packaging.Targets.Tests/IO/XZOutputStreamTests.cs +++ b/Packaging.Targets.Tests/IO/XZOutputStreamTests.cs @@ -7,14 +7,6 @@ namespace Packaging.Targets.Tests.IO { public class XZOutputStreamTests { - // These tests help ensure that the version of liblzma we use on the different operating systems support - // multithreading. Single-threaded compression can seriously impact performance. - [Fact] - public void SupportsMultiThreadingTest() - { - Assert.True(XZOutputStream.SupportsMultiThreading); - } - [Fact] public void DefaultThreadsTest() { diff --git a/Packaging.Targets/IO/LzmaAction.cs b/Packaging.Targets/IO/LzmaAction.cs deleted file mode 100644 index 51d7a68c..00000000 --- a/Packaging.Targets/IO/LzmaAction.cs +++ /dev/null @@ -1,136 +0,0 @@ -namespace Packaging.Targets.IO -{ - /// - /// The `action' argument for lzma_code() - /// - /// - /// After the first use of , , , - /// or , the same `action' must is used until returns - /// . Also, the amount of input (that is, strm->avail_in) must - /// not be modified by the application until returns - /// . Changing the `action' or modifying the amount of input - /// will make return . - /// - internal enum LzmaAction - { - /// - /// Continue coding - /// - /// - /// - /// Encoder: Encode as much input as possible. Some internal - /// buffering will probably be done(depends on the filter - /// chain in use), which causes latency: the input used won't - /// usually be decodeable from the output of the same - /// call. - /// - /// - /// Decoder: Decode as much input as possible and produce as - /// much output as possible. - /// - /// - Run = 0, - - /// - /// Make all the input available at output - /// - /// - /// - /// Normally the encoder introduces some latency. - /// forces all the buffered data to be - /// available at output without resetting the internal - /// - /// - /// state of the encoder.This way it is possible to use - /// compressed stream for example for communication over - /// network. - /// - /// - /// Only some filters support . Trying to use - /// with filters that don't support it will - /// make return . For example, - /// LZMA1 doesn't support but LZMA2 does. - /// - /// - /// Using very often can dramatically reduce - /// the compression ratio. With some filters (for example, - /// LZMA2), fine-tuning the compression options may help - /// mitigate this problem significantly (for example, - /// match finder with LZMA2). - /// - /// - /// Decoders don't support . - /// - /// - SyncFlush = 1, - - /// - /// Finish encoding of the current Block - /// - /// - /// - /// All the input data going to the current Block must have - /// been given to the encoder (the last bytes can still be - /// pending next_in). Call with - /// until it returns . Then continue normally - /// with or finish the Stream with . - /// - /// - /// This action is currently supported only by Stream encoder - /// and easy encoder (which uses Stream encoder). If there is - /// no unfinished Block, no empty Block is created. - /// - /// - FullFlush = 2, - - /// - /// Finish the coding operation - /// - /// - /// - /// All the input data must have been given to the encoder - /// (the last bytes can still be pending in next_in). - /// Call with until it returns - /// . Once has been used, - /// the amount of input must no longer be changed by - /// the application. - /// - /// - /// When decoding, using is optional unless the - /// flag was used when the decoder was - /// initialized. When was not used, the only - /// effect of is that the amount of input must not - /// be changed just like in the encoder. - /// - /// - Finish = 3, - - /// - /// Finish encoding of the current Block - /// - /// - /// - /// This is like except that this doesn't - /// necessarily wait until all the input has been made - /// available via the output buffer. That is, - /// might return as soon as all the input - /// has been consumed (avail_in == 0). - /// - /// - /// is useful with a threaded encoder if - /// one wants to split the .xz Stream into Blocks at specific - /// offsets but doesn't care if the output isn't flushed - /// immediately. Using allows keeping - /// the threads busy while would make - /// wait until all the threads have finished - /// until more data could be passed to the encoder. - /// - /// - /// With a initialized with the single-threaded - /// lzma_stream_encoder() or lzma_easy_encoder(), - /// is an alias for . - /// - /// - FullBarrier = 4 - } -} diff --git a/Packaging.Targets/IO/LzmaCheck.cs b/Packaging.Targets/IO/LzmaCheck.cs deleted file mode 100644 index d8603083..00000000 --- a/Packaging.Targets/IO/LzmaCheck.cs +++ /dev/null @@ -1,34 +0,0 @@ -namespace Packaging.Targets.IO -{ - /// - /// Type of the integrity check (Check ID) - /// - /// - /// The.xz format supports multiple types of checks that are calculated - /// from the uncompressed data. They vary in both speed and ability to - /// detect errors. - /// - /// - internal enum LzmaCheck - { - /// - /// No Check is calculated. - /// - None = 0, - - /// - /// CRC32 using the polynomial from the IEEE 802.3 standard - /// - Crc32 = 1, - - /// - /// CRC64 using the polynomial from the ECMA-182 standard - /// - Crc64 = 4, - - /// - /// SHA-256 - /// - Sha256 = 10 - } -} diff --git a/Packaging.Targets/IO/LzmaDecodeFlags.cs b/Packaging.Targets/IO/LzmaDecodeFlags.cs deleted file mode 100644 index 418de7f6..00000000 --- a/Packaging.Targets/IO/LzmaDecodeFlags.cs +++ /dev/null @@ -1,47 +0,0 @@ -namespace Packaging.Targets.IO -{ - /// - /// Flags that control the decoding behavior of liblzma. - /// - internal enum LzmaDecodeFlags : uint - { - /// - /// This flag makes return - /// if the input stream - /// being decoded has no integrity check. - /// - TellNoCheck = 0x01, - - /// - /// This flag makes return - /// if the input - /// stream has an integrity check, but the type of the integrity check is not - /// supported by this liblzma version or build. Such files can still be - /// decoded, but the integrity check cannot be verified. - /// - TellUnsupportedCheck = 0x02, - - /// - /// This flag makes return - /// as soon as the type - /// of the integrity check is known. - /// - TellAnyCheck = 0x04, - - /// - /// - /// This flag enables decoding of concatenated files with file formats that - /// allow concatenating compressed files as is. From the formats currently - /// supported by liblzma, only the .xz format allows concatenated files. - /// Concatenated files are not allowed with the legacy .lzma format. - /// - /// - /// This flag also affects the usage of the `action' argument for - /// When is used, won't return - /// unless is used as `action'. Thus, the application has to set - /// in the same way as it does when encoding. - /// - /// - Concatenated = 0x08 - } -} diff --git a/Packaging.Targets/IO/LzmaMT.cs b/Packaging.Targets/IO/LzmaMT.cs deleted file mode 100644 index 38b55e1e..00000000 --- a/Packaging.Targets/IO/LzmaMT.cs +++ /dev/null @@ -1,154 +0,0 @@ -using System; - -namespace Packaging.Targets.IO -{ -#pragma warning disable SA1307 // Accessible fields must begin with upper-case letter -#pragma warning disable SA1310 // Field names must not contain underscore -#pragma warning disable CS0169 // The field '' is never used -#pragma warning disable IDE0051 // Remove unused private members - /// - /// Multithreading options - /// - internal struct LzmaMT - { - /// - /// Set this to zero if no flags are wanted. - /// - /// - /// No flags are currently supported. - /// - public uint flags; - - /// - /// Number of worker threads to use - /// - public uint threads; - - /// - /// Maximum uncompressed size of a Block - /// - /// - /// - /// The encoder will start a new .xz Block every block_size bytes. - /// Using LZMA_FULL_FLUSH or LZMA_FULL_BARRIER with lzma_code() - /// the caller may tell liblzma to start a new Block earlier. - /// - /// - /// With LZMA2, a recommended block size is 2-4 times the LZMA2 - /// dictionary size. With very small dictionaries, it is recommended - /// to use at least 1 MiB block size for good compression ratio, even - /// if this is more than four times the dictionary size. Note that - /// these are only recommendations for typical use cases; feel free - /// to use other values. Just keep in mind that using a block size - /// less than the LZMA2 dictionary size is waste of RAM. - /// - /// - /// Set this to 0 to let liblzma choose the block size depending - /// on the compression options. For LZMA2 it will be 3*dict_size - /// or 1 MiB, whichever is more. - /// - /// - /// For each thread, about 3 * block_size bytes of memory will be - /// allocated. This may change in later liblzma versions. If so, - /// the memory usage will probably be reduced, not increased. - /// - /// - public ulong block_size; - - /// - /// Timeout to allow lzma_code() to return early - /// - /// - /// - /// Multithreading can make liblzma to consume input and produce - /// output in a very bursty way: it may first read a lot of input - /// to fill internal buffers, then no input or output occurs for - /// a while. - /// - /// - /// In single-threaded mode, lzma_code() won't return until it has - /// either consumed all the input or filled the output buffer. If - /// this is done in multithreaded mode, it may cause a call - /// lzma_code() to take even tens of seconds, which isn't acceptable - /// in all applications. - /// - /// - /// To avoid very long blocking times in lzma_code(), a timeout - /// (in milliseconds) may be set here. If lzma_code() would block - /// longer than this number of milliseconds, it will return with - /// LZMA_OK. Reasonable values are 100 ms or more. The xz command - /// line tool uses 300 ms. - /// - /// - /// If long blocking times are fine for you, set timeout to a special - /// value of 0, which will disable the timeout mechanism and will make - /// lzma_code() block until all the input is consumed or the output - /// buffer has been filled. - /// - /// - /// Even with a timeout, lzma_code() might sometimes take - /// somewhat long time to return. No timing guarantees - /// are made. - /// - /// - public uint timeout; - - /// - /// Compression preset (level and possible flags) - /// - /// - /// - /// The preset is set just like with lzma_easy_encoder(). - /// - /// - /// The preset is ignored if filters below is non-NULL. - /// - /// - public uint preset; - - /// - /// Filter chain (alternative to a preset). - /// - /// - /// If this is NULL, the preset above is used. Otherwise the preset - /// is ignored and the filter chain specified here is used. - /// - public IntPtr filters; - - /// - /// Integrity check type. - /// - /// - /// The xz command line tool defaults to LZMA_CHECK_CRC64, which is - /// a good choice if you are unsure. - /// - public LzmaCheck check; - - /// - /// Reserved space to allow possible future extensions without - /// breaking the ABI. You should not touch these, because the names - /// of these variables may change. These are and will never be used - /// with the currently supported options, so it is safe to leave these - /// uninitialized. - /// - private readonly int reserved_enum1; - private readonly int reserved_enum2; - private readonly int reserved_enum3; - private readonly int reserved_int1; - private readonly int reserved_int2; - private readonly int reserved_int3; - private readonly int reserved_int4; - private readonly ulong reserved_int5; - private readonly ulong reserved_int6; - private readonly ulong reserved_int7; - private readonly ulong reserved_int8; - private readonly IntPtr reserved_ptr1; - private readonly IntPtr reserved_ptr2; - private readonly IntPtr reserved_ptr3; - private readonly IntPtr reserved_ptr4; - } -#pragma warning restore SA1307 // Accessible fields must begin with upper-case letter -#pragma warning restore SA1310 // Field names must not contain underscore -#pragma warning restore CS0169 // The field '' is never used -#pragma warning restore IDE0051 // Remove unused private members -} diff --git a/Packaging.Targets/IO/LzmaResult.cs b/Packaging.Targets/IO/LzmaResult.cs deleted file mode 100644 index 03f4cc07..00000000 --- a/Packaging.Targets/IO/LzmaResult.cs +++ /dev/null @@ -1,230 +0,0 @@ -using System; - -namespace Packaging.Targets.IO -{ - /// - /// Return values used by several functions in liblzma. - /// - internal enum LzmaResult : uint - { - /// - /// The operation completed successfully. - /// - OK = 0, - - /// - /// End of stream was reached - /// - /// - /// - /// In encoder, , , or - /// was finished. In decoder, this indicates - /// that all the data was successfully decoded. - /// - /// - /// In all cases, when is returned, the last - /// output bytes should be picked from . - /// - /// - StreamEnd = 1, - - /// - /// Input stream has no integrity check - /// - /// - /// - /// This return value can be returned only if the - /// flag was used when initializing - /// the decoder. is just a warning, and - /// the decoding can be continued normally. - /// - /// - /// It is possible to call lzma_get_check() immediately after - /// lzma_code has returned . The result will - /// naturally be , but the possibility to call - /// lzma_get_check() may be convenient in some applications. - /// - /// - NoCheck = 2, - - /// - /// Cannot calculate the integrity check - /// - /// - /// - /// The usage of this return value is different in encoders - /// and decoders. - /// - /// - /// Encoders can return this value only from the initialization - /// function. If initialization fails with this value, the - /// encoding cannot be done, because there's no way to produce - /// output with the correct integrity check. - /// - /// - /// Decoders can return this value only from lzma_code() and - /// only if the flag was used when - /// initializing the decoder. The decoding can still be - /// continued normally even if the check type is unsupported, - /// but naturally the check will not be validated, and possible - /// errors may go undetected. - /// - /// - /// With decoder, it is possible to call lzma_get_check() - /// immediately after lzma_code() has returned - /// . This way it is possible to find - /// out what the unsupported Check ID was. - /// - /// - UnsupportedCheck = 3, - - /// - /// Integrity check type is now available - /// - /// - /// This value can be returned only by the lzma_code() function - /// and only if the decoder was initialized with the - /// flag. tells the - /// application that it may now call lzma_get_check() to find - /// out the Check ID. This can be used, for example, to - /// implement a decoder that accepts only files that have - /// strong enough integrity check. - /// - GetCheck = 4, - - /// - /// Cannot allocate memory - /// - /// - /// - /// Memory allocation failed, or the size of the allocation - /// would be greater than SIZE_MAX. - /// - /// - /// Due to internal implementation reasons, the coding cannot - /// be continued even if more memory were made available after - /// . - /// - /// - MemError = 5, - - /// - /// Memory usage limit was reached - /// - /// - /// Decoder would need more memory than allowed by the - /// specified memory usage limit. To continue decoding, - /// the memory usage limit has to be increased with - /// lzma_memlimit_set(). - /// - MemlimitError = 6, - - /// - /// File format not recognized - /// - /// - /// The decoder did not recognize the input as supported file - /// format. This error can occur, for example, when trying to - /// decode .lzma format file with , - /// because accepts only the .xz format. - /// - FormatError = 7, - - /// - /// Invalid or unsupported options - /// - /// - /// - /// Invalid or unsupported options, for example - /// - unsupported filter(s) or filter options; or - /// - reserved bits set in headers (decoder only). - /// - /// - /// Rebuilding liblzma with more features enabled, or - /// upgrading to a newer version of liblzma may help. - /// - /// - OptionsError = 8, - - /// - /// Data is corrupt - /// - /// - /// - /// The usage of this return value is different in encoders - /// and decoders. In both encoder and decoder, the coding - /// cannot continue after this error. - /// - /// - /// Encoders return this if size limits of the target file - /// format would be exceeded. These limits are huge, thus - /// getting this error from an encoder is mostly theoretical. - /// For example, the maximum compressed and uncompressed - /// size of a .xz Stream is roughly 8 EiB (2^63 bytes). - /// - /// - /// Decoders return this error if the input data is corrupt. - /// This can mean, for example, invalid CRC32 in headers - /// or invalid check of uncompressed data. - /// - /// - DataError = 9, - - /// - /// No progress is possible - /// - /// - /// - /// This error code is returned when the coder cannot consume - /// any new input and produce any new output.The most common - /// reason for this error is that the input stream being - /// decoded is truncated or corrupt. - /// - /// - /// This error is not fatal. Coding can be continued normally - /// by providing more input and/or more output space, if - /// possible. - /// - /// - /// Typically the first call to that can do no - /// progress returns instead of . Only - /// the second consecutive call doing no progress will return - /// . This is intentional. - /// - /// - /// With zlib, Z_BUF_ERROR may be returned even if the - /// application is doing nothing wrong, so apps will need - /// to handle Z_BUF_ERROR specially. The above hack - /// guarantees that liblzma never returns - /// to properly written applications unless the input file - /// is truncated or corrupt. This should simplify the - /// applications a little. - /// - /// - BufferError = 10, - - /// - /// Programming error - /// - /// - /// - /// This indicates that the arguments given to the function are - /// invalid or the internal state of the decoder is corrupt. - /// - Function arguments are invalid or the structures - /// pointed by the argument pointers are invalid - /// e.g. if has been set to and - /// > 0 when calling . - /// - lzma_/// functions have been called in wrong order - /// e.g. was called right after . - /// - If errors occur randomly, the reason might be flaky - /// hardware. - /// - /// - /// If you think that your code is correct, this error code - /// can be a sign of a bug in liblzma. See the documentation - /// how to report bugs. - /// - /// - ProgError = 11 - } -} diff --git a/Packaging.Targets/IO/LzmaStream.cs b/Packaging.Targets/IO/LzmaStream.cs deleted file mode 100644 index 6ed09a7a..00000000 --- a/Packaging.Targets/IO/LzmaStream.cs +++ /dev/null @@ -1,121 +0,0 @@ -using System; -using System.Runtime.InteropServices; - -namespace Packaging.Targets.IO -{ - /// - /// Passing data to and from liblzma - /// - /// - /// - /// The lzma_stream structure is used for - /// - passing pointers to input and output buffers to liblzma; - /// - defining custom memory hander functions; and - /// - holding a pointer to coder-specific internal data structures. - /// - /// - /// Typical usage: - /// - /// - /// - After allocating (on stack or with malloc()), it must be - /// initialized to LZMA_STREAM_INIT (see LZMA_STREAM_INIT for details). - /// - /// - /// - Initialize a coder to the lzma_stream, for example by using - /// lzma_easy_encoder() or lzma_auto_decoder(). Some notes: - /// - In contrast to zlib, and are - /// ignored by all initialization functions, thus it is safe - /// to not initialize them yet. - /// - The initialization functions always set strm->total_in and - /// strm->total_out to zero. - /// - If the initialization function fails, no memory is left allocated - /// that would require freeing with even if some memory was - /// associated with the structure when the initialization - /// function was called. - /// - /// - /// - Use to do the actual work. - /// - /// - /// - Once the coding has been finished, the existing lzma_stream can be - /// reused. It is OK to reuse with different initialization - /// function without calling first. Old allocations are - /// automatically freed. - /// - /// - /// - Finally, use to free the allocated memory. never - /// frees the structure itself. - /// - /// - /// Application may modify the values of and as it wants. - /// They are updated by liblzma to match the amount of data read and - /// written, but aren't used for anything else. - /// - /// - [StructLayout(LayoutKind.Sequential)] - internal struct LzmaStream - { - /// - /// Pointer to the next input byte. - /// - public IntPtr NextIn; - - /// - /// Number of available input bytes in next_in. - /// - public uint AvailIn; - - /// - /// Total number of bytes read by liblzma. - /// - public ulong TotalIn; - - /// - /// Pointer to the next output position. - /// - public IntPtr NextOut; - - /// - /// Amount of free space in next_out. - /// - public uint AvailOut; - - /// - /// Total number of bytes written by liblzma. - /// - public ulong TotalOut; - - /// - /// Custom memory allocation functions - /// - /// - /// In most cases this is NULL which makes liblzma use - /// the standard malloc() and free(). - /// - public IntPtr Allocator; - - /// - /// Internal state is not visible to applications. - /// -#pragma warning disable SA1214 // Readonly fields must appear before non-readonly fields - public readonly IntPtr InternalState; -#pragma warning restore SA1214 // Readonly fields must appear before non-readonly fields - - /// - /// Reserved space to allow possible future extensions without - /// breaking the ABI. Excluding the initialization of this structure, - /// you should not touch these, because the names of these variables - /// may change. - /// - private readonly IntPtr reservedPtr1; - private readonly IntPtr reservedPtr2; - private readonly IntPtr reservedPtr3; - private readonly IntPtr reservedPtr4; - private readonly ulong reservedInt1; - private readonly ulong reservedInt2; - private readonly uint reservedInt3; - private readonly uint reservedInt4; - private readonly uint reservedEnum1; - private readonly uint reservedEnum2; - } -} \ No newline at end of file diff --git a/Packaging.Targets/IO/LzmaStreamFlags.cs b/Packaging.Targets/IO/LzmaStreamFlags.cs deleted file mode 100644 index d235a3fa..00000000 --- a/Packaging.Targets/IO/LzmaStreamFlags.cs +++ /dev/null @@ -1,98 +0,0 @@ -using System.Runtime.InteropServices; - -namespace Packaging.Targets.IO -{ - /// - /// Options for encoding/decoding Stream Header and Stream Footer - /// - /// - [StructLayout(LayoutKind.Sequential)] - internal struct LzmaStreamFlags - { - /// - /// Stream Flags format version - /// - /// - /// - /// To prevent API and ABI breakages if new features are needed in - /// Stream Header or Stream Footer, a version number is used to - /// indicate which fields in this structure are in use.For now, - /// version must always be zero.With non-zero version, the - /// lzma_stream_header_encode() and lzma_stream_footer_encode() - /// will return . - /// - /// - /// - /// lzma_stream_header_decode() and - /// will always set this to the lowest value that supports all the - /// features indicated by the Stream Flags field.The application - /// must check that the version number set by the decoding functions - /// is supported by the application. Otherwise it is possible that - /// the application will decode the Stream incorrectly. - /// - /// - public readonly uint Version; - - /// - /// Backward Size - /// - /// - /// - /// Backward Size must be a multiple of four bytes.In this Stream - /// format version, Backward Size is the size of the Index field. - /// - /// - /// - /// Backward Size isn't actually part of the Stream Flags field, but - /// it is convenient to include in this structure anyway. Backward - /// Size is present only in the Stream Footer.There is no need to - /// initialize backward_size when encoding Stream Header. - /// - /// - /// - /// lzma_stream_header_decode() always sets backward_size to - /// LZMA_VLI_UNKNOWN so that it is convenient to use - /// lzma_stream_flags_compare() when both Stream Header and Stream - /// Footer have been decoded. - /// - /// - public ulong BackwardSize; - - /// - /// Check ID - /// - /// - /// This indicates the type of the integrity check calculated from - /// uncompressed data. - /// - public LzmaCheck Check; - - /// - /// - /// Reserved space to allow possible future extensions without - /// breaking the ABI.You should not touch these, because the - /// names of these variables may change. - /// - /// - /// - /// (We will never be able to use all of these since Stream Flags - /// is just two bytes plus Backward Size of four bytes.But it's - /// nice to have the proper types when they are needed.) - /// - /// - private readonly int reservedEnum1; - private readonly int reservedEnum2; - private readonly int reservedEnum3; - private readonly int reservedEnum4; - private readonly char reservedBool1; - private readonly char reservedBool2; - private readonly char reservedBool3; - private readonly char reservedBool4; - private readonly char reservedBool5; - private readonly char reservedBool6; - private readonly char reservedBool7; - private readonly char reservedBool8; - private readonly uint reservedInt1; - private readonly uint reservedInt2; - } -} diff --git a/Packaging.Targets/IO/NativeMethods.cs b/Packaging.Targets/IO/NativeMethods.cs deleted file mode 100644 index 9b73f340..00000000 --- a/Packaging.Targets/IO/NativeMethods.cs +++ /dev/null @@ -1,407 +0,0 @@ -using System; -using System.IO; -using System.Reflection; -using System.Runtime.InteropServices; -using FunctionLoader = Packaging.Targets.Native.FunctionLoader; - -namespace Packaging.Targets.IO -{ - /// - /// Provides access to the liblzma API. liblzma is part of the xz suite. - /// - /// - /// You can download pre-built binaries from Windows from https://tukaani.org/xz/. - /// - /// - /// - internal static class NativeMethods - { - /// - /// The name of the lzma library. - /// - /// - /// You can fetch liblzma from https://github.com/RomanBelkov/XZ.NET/blob/master/XZ.NET/liblzma.dll - /// - private const string LibraryName = @"lzma"; - - private static lzma_stream_decoder_delegate lzma_stream_decoder_ptr; - private static lzma_code_delegate lzma_code_ptr; - private static lzma_stream_footer_decode_delegate lzma_stream_footer_decode_ptr; - private static lzma_index_uncompressed_size_delegate lzma_index_uncompressed_size_ptr; - private static lzma_index_buffer_decode_delegate lzma_index_buffer_decode_ptr; - private static lzma_index_end_delegate lzma_index_end_ptr; - private static lzma_end_delegate lzma_end_ptr; - private static lzma_easy_encoder_delegate lzma_easy_encoder_ptr; - private static lzma_stream_encoder_mt_delegate lzma_stream_encoder_mt_ptr; - private static lzma_stream_buffer_bound_delegate lzma_stream_buffer_bound_ptr; - private static lzma_easy_buffer_encode_delegate lzma_easy_buffer_encode_ptr; - - static NativeMethods() - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - if (RuntimeInformation.OSArchitecture != Architecture.X64) - { - throw new InvalidOperationException(".NET packaging only supports 64-bit Windows processes"); - } - } - - var libraryPath = Path.GetDirectoryName(typeof(NativeMethods).GetTypeInfo().Assembly.Location); - var lzmaWindowsPath = Path.GetFullPath(Path.Combine(libraryPath, "../../runtimes/win7-x64/native/lzma.dll")); - - IntPtr library = FunctionLoader.LoadNativeLibrary( - new string[] { lzmaWindowsPath, "lzma.dll" }, // lzma.dll is used when running unit tests. - new string[] { "liblzma.so.5", "liblzma.so" }, - new string[] { "liblzma.dylib" }); - - if (library == IntPtr.Zero) - { - throw new FileLoadException("Could not load liblzma. On Linux, make sure you've installed liblzma-dev or an equivalent package."); - } - - lzma_stream_decoder_ptr = FunctionLoader.LoadFunctionDelegate(library, nameof(lzma_stream_decoder)); - lzma_code_ptr = FunctionLoader.LoadFunctionDelegate(library, nameof(lzma_code)); - lzma_stream_footer_decode_ptr = FunctionLoader.LoadFunctionDelegate(library, nameof(lzma_stream_footer_decode)); - lzma_index_uncompressed_size_ptr = FunctionLoader.LoadFunctionDelegate(library, nameof(lzma_index_uncompressed_size)); - lzma_index_buffer_decode_ptr = FunctionLoader.LoadFunctionDelegate(library, nameof(lzma_index_buffer_decode)); - lzma_index_end_ptr = FunctionLoader.LoadFunctionDelegate(library, nameof(lzma_index_end)); - lzma_end_ptr = FunctionLoader.LoadFunctionDelegate(library, nameof(lzma_end)); - lzma_easy_encoder_ptr = FunctionLoader.LoadFunctionDelegate(library, nameof(lzma_easy_encoder)); - lzma_stream_encoder_mt_ptr = FunctionLoader.LoadFunctionDelegate(library, nameof(lzma_stream_encoder_mt), throwOnError: false); - lzma_stream_buffer_bound_ptr = FunctionLoader.LoadFunctionDelegate(library, nameof(lzma_stream_buffer_bound)); - lzma_easy_buffer_encode_ptr = FunctionLoader.LoadFunctionDelegate(library, nameof(lzma_easy_buffer_encode)); - } - - private delegate LzmaResult lzma_stream_decoder_delegate(ref LzmaStream stream, ulong memLimit, LzmaDecodeFlags flags); - - private unsafe delegate LzmaResult lzma_easy_buffer_encode_delegate(uint preset, LzmaCheck check, void* allocator, byte[] @in, UIntPtr in_size, byte[] @out, UIntPtr* out_pos, UIntPtr out_size); - - private delegate UIntPtr lzma_stream_buffer_bound_delegate(UIntPtr uncompressed_size); - - private delegate LzmaResult lzma_stream_encoder_mt_delegate(ref LzmaStream stream, ref LzmaMT mt); - - private delegate LzmaResult lzma_easy_encoder_delegate(ref LzmaStream stream, uint preset, LzmaCheck check); - - private delegate void lzma_end_delegate(ref LzmaStream stream); - - private delegate void lzma_index_end_delegate(IntPtr i, IntPtr allocator); - - private delegate uint lzma_index_buffer_decode_delegate(ref IntPtr i, ref ulong memLimit, IntPtr allocator, byte[] indexBuffer, ref uint inPosition, ulong inSize); - - private delegate ulong lzma_index_uncompressed_size_delegate(IntPtr i); - - private delegate LzmaResult lzma_stream_footer_decode_delegate(ref LzmaStreamFlags options, byte[] inp); - - private delegate LzmaResult lzma_code_delegate(ref LzmaStream stream, LzmaAction action); - - /// - /// Gets a value indicating whether the underlying native library supports multithreading. - /// - public static bool SupportsMultiThreading - { - get { return lzma_stream_encoder_mt_ptr != null; } - } - - /// - /// Initialize .xz Stream decoder - /// - /// - /// Pointer to properly prepared - /// - /// - /// Memory usage limit as bytes. Use UINT64_MAX - /// to effectively disable the limiter. - /// - /// - /// Bitwise-or of zero or more of the decoder flags: - /// , , - /// , . - /// - /// - /// : Initialization was successful, - /// : Cannot allocate memory, - /// : Unsupported flags, - /// . - /// - /// - public static LzmaResult lzma_stream_decoder(ref LzmaStream stream, ulong memLimit, LzmaDecodeFlags flags) => lzma_stream_decoder_ptr(ref stream, memLimit, flags); - - /// - /// Encode or decode data - /// - /// - /// The for which to read the data. - /// - /// - /// The action to perform. - /// - /// - /// A value. - /// - /// - /// - /// Once the has been successfully initialized (e.g. with - /// lzma_stream_encoder()), the actual encoding or decoding is done - /// using this function.The application has to update , - /// , , and to pass input - /// to and get output from liblzma. - /// - /// - /// See the description of the coder-specific initialization function to find - /// out what values are supported by the coder. - /// - /// - public static LzmaResult lzma_code(ref LzmaStream stream, LzmaAction action) => lzma_code_ptr(ref stream, action); - - /// - /// Decode Stream Footer - /// - /// - /// Target for the decoded Stream Header options. - /// - /// - /// Beginning of the input buffer of - /// LZMA_STREAM_HEADER_SIZE bytes. - /// - /// - /// : Decoding was successful. - /// : Magic bytes don't match, thus the given buffer cannot be Stream Footer. - /// : CRC32 doesn't match, thus the Stream Footer is corrupt. - /// : Unsupported options are present in Stream Footer. - /// - /// - /// If Stream Header was already decoded successfully, but - /// decoding Stream Footer returns , the - /// application should probably report some other error message - /// than "file format not recognized", since the file more likely - /// is corrupt(possibly truncated). Stream decoder in liblzma - /// uses in this situation. - /// - /// - public static LzmaResult lzma_stream_footer_decode(ref LzmaStreamFlags options, byte[] inp) => lzma_stream_footer_decode_ptr(ref options, inp); - - /// - /// Get the uncompressed size of the file - /// - /// - public static ulong lzma_index_uncompressed_size(IntPtr i) => lzma_index_uncompressed_size_ptr(i); - - /// - /// Single-call .xz Index decoder - /// - /// If decoding succeeds, will point to a new lzma_index, which the application has to - /// later free with lzma_index_end(). If an error occurs, will be . The old value of - /// is always ignored and thus doesn't need to be initialized by the caller. - /// - /// - /// Pointer to how much memory the resulting lzma_index is allowed to require. The value - /// pointed by this pointer is modified if and only if is returned. - /// - /// - /// Pointer to lzma_allocator, or to use malloc() - /// - /// - /// Beginning of the input buffer - /// - /// - /// The next byte will be read from in[*in_pos]. *in_pos is updated only if decoding succeeds. - /// - /// - /// Size of the input buffer; the first byte that won't be read is in[in_size]. - /// - /// - /// : Decoding was successful. - /// , - /// : Memory usage limit was reached. The minimum required memlimit value was stored to* memlimit. - /// , - /// . - /// - /// - public static uint lzma_index_buffer_decode(ref IntPtr i, ref ulong memLimit, IntPtr allocator, byte[] indexBuffer, ref uint inPosition, ulong inSize) - => lzma_index_buffer_decode_ptr(ref i, ref memLimit, allocator, indexBuffer, ref inPosition, inSize); - - /// - /// Deallocate lzma_index - /// - /// - /// The index to deallocate - /// - /// - /// The allocated used to allocate the memory. - /// - /// - /// If is , this does nothing. - /// - /// - public static void lzma_index_end(IntPtr i, IntPtr allocator) => lzma_index_end_ptr(i, allocator); - - /// - /// Free memory allocated for the coder data structures - /// - /// - /// Pointer to lzma_stream that is at least initialized with LZMA_STREAM_INIT. - /// - /// - /// After , is guaranteed to be . - /// No other members of the structure are touched. - /// zlib indicates an error if application end()s unfinished stream structure. - /// liblzma doesn't do this, and assumes that - /// application knows what it is doing. - /// - /// - public static void lzma_end(ref LzmaStream stream) => lzma_end_ptr(ref stream); - - /// - /// - /// Initialize .xz Stream encoder using a preset number. - /// - /// - /// This function is intended for those who just want to use the basic features if liblzma(that is, most developers out there). - /// - /// - /// If initialization fails(return value is not LZMA_OK), all the memory allocated for *strm by liblzma is always freed.Thus, there is no need to call lzma_end() after failed initialization. - /// - /// If initialization succeeds, use lzma_code() to do the actual encoding.Valid values for `action' (the second argument of lzma_code()) are LZMA_RUN, LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, and LZMA_FINISH. In future, there may be compression levels or flags that don't support LZMA_SYNC_FLUSH. - /// - /// - /// - /// Pointer to lzma_stream that is at least initialized with LZMA_STREAM_INIT. - /// - /// - /// Compression preset to use. A preset consist of level number and zero or more flags. Usually flags aren't used, so preset is simply a number [0, 9] which match the options -0 ... -9 of the xz command line tool. Additional flags can be be set using bitwise-or with the preset level number, e.g. 6 | LZMA_PRESET_EXTREME. - /// - /// - /// Integrity check type to use. See check.h for available checks. The xz command line tool defaults to LZMA_CHECK_CRC64, which is a good choice if you are unsure. LZMA_CHECK_CRC32 is good too as long as the uncompressed file is not many gigabytes. - /// - /// - /// - /// - LZMA_OK: Initialization succeeded. Use lzma_code() to encode your data. - /// - /// - LZMA_MEM_ERROR: Memory allocation failed. - /// - /// - /// - LZMA_OPTIONS_ERROR: The given compression preset is not supported by this build of liblzma. - /// - /// - /// - LZMA_UNSUPPORTED_CHECK: The given check type is not supported by this liblzma build. - /// - /// - /// - LZMA_PROG_ERROR: One or more of the parameters have values that will never be valid. For example, strm == NULL. - /// - /// - public static LzmaResult lzma_easy_encoder(ref LzmaStream stream, uint preset, LzmaCheck check) => lzma_easy_encoder_ptr(ref stream, preset, check); - - /// - /// - /// Initialize multithreaded .xz Stream encoder - /// - /// - /// This provides the functionality of lzma_easy_encoder() and - /// lzma_stream_encoder() as a single function for multithreaded use. - /// - /// - /// The supported actions for lzma_code() are LZMA_RUN, LZMA_FULL_FLUSH, - /// LZMA_FULL_BARRIER, and LZMA_FINISH. Support for LZMA_SYNC_FLUSH might be - /// added in the future. - /// - /// - /// - /// Pointer to properly prepared lzma_stream - /// - /// - /// Pointer to multithreaded compression options - /// - /// - /// A value which indicates success or failure. - /// - public static LzmaResult lzma_stream_encoder_mt(ref LzmaStream stream, ref LzmaMT mt) - { - if (SupportsMultiThreading) - { - return lzma_stream_encoder_mt_ptr(ref stream, ref mt); - } - else - { - throw new PlatformNotSupportedException("lzma_stream_encoder_mt is not supported on this platform. Check SupportsMultiThreading to see whether you can use this functionality."); - } - } - - /// - /// - /// Calculate output buffer size for single-call Stream encoder - /// - /// - /// When trying to compress uncompressible data, the encoded size will be slightly bigger than the input data.This function calculates how much output buffer space is required to be sure that lzma_stream_buffer_encode() doesn't return LZMA_BUF_ERROR. - /// - /// - /// The calculated value is not exact, but it is guaranteed to be big enough.The actual maximum output space required may be slightly smaller (up to about 100 bytes). This should not be a problem in practice. - /// - /// - /// If the calculated maximum size doesn't fit into size_t or would make the Stream grow past LZMA_VLI_MAX (which should never happen in practice), zero is returned to indicate the error. - /// - /// - /// - /// The uncompressed size. - /// - /// - /// The buffer output size. - /// - /// - /// The limit calculated by this function applies only to single-call encoding. Multi-call encoding may (and probably will) have larger maximum expansion when encoding uncompressible data. Currently there is no function to calculate the maximum expansion of multi-call encoding. - /// - public static UIntPtr lzma_stream_buffer_bound(UIntPtr uncompressed_size) => lzma_stream_buffer_bound_ptr(uncompressed_size); - - /// - /// Single-call .xz Stream encoding using a preset number. - /// - /// - /// Compression preset to use. See the description in lzma_easy_encoder(). - /// - /// - /// Type of the integrity check to calculate from uncompressed data. - /// - /// - /// lzma_allocator for custom allocator functions. Set to NULL to use malloc() and free(). - /// - /// - /// Beginning of the input buffer - /// - /// - /// Size of the input buffer - /// - /// - /// Beginning of the output buffer - /// - /// - /// The next byte will be written to out[*out_pos]. *out_pos is updated only if encoding succeeds. - /// - /// - /// Size of the out buffer; the first byte into which no data is written to is out[out_size]. - /// - /// - /// - /// - LZMA_OK: Encoding was successful. - /// - /// - /// - LZMA_BUF_ERROR: Not enough output buffer space. - /// - /// - /// - LZMA_OPTIONS_ERROR - /// - /// - /// - LZMA_MEM_ERROR - /// - /// - /// - LZMA_DATA_ERROR - /// - /// - /// - LZMA_PROG_ERROR - /// - /// - /// - /// The maximum required output buffer size can be calculated with lzma_stream_buffer_bound() - /// - public static unsafe LzmaResult lzma_easy_buffer_encode(uint preset, LzmaCheck check, void* allocator, byte[] @in, UIntPtr in_size, byte[] @out, UIntPtr* out_pos, UIntPtr out_size) - => lzma_easy_buffer_encode_ptr(preset, check, allocator, @in, in_size, @out, out_pos, out_size); - } -} diff --git a/Packaging.Targets/IO/XZHelper.cs b/Packaging.Targets/IO/XZHelper.cs new file mode 100644 index 00000000..e6f3c418 --- /dev/null +++ b/Packaging.Targets/IO/XZHelper.cs @@ -0,0 +1,24 @@ +using Joveler.Compression.XZ; + +namespace Packaging.Targets.IO +{ + /// + /// Helper class to ensure XZ library is initialized once globally + /// + internal static class XZHelper + { + static XZHelper() + { + XZInit.GlobalInit(); + } + + /// + /// Ensures XZ library is initialized. This is a no-op after the first call + /// since initialization happens in static constructor. + /// + public static void EnsureInitialized() + { + // Static constructor will have already run by the time this method is called + } + } +} diff --git a/Packaging.Targets/IO/XZInputStream.cs b/Packaging.Targets/IO/XZInputStream.cs index 4a015540..062a1a95 100644 --- a/Packaging.Targets/IO/XZInputStream.cs +++ b/Packaging.Targets/IO/XZInputStream.cs @@ -1,4 +1,4 @@ -/* +/* * The MIT License (MIT) * Copyright (c) 2015 Roman Belkov, Kirill Melentyev @@ -23,30 +23,22 @@ */ using System; -using System.Collections.Generic; using System.IO; -using System.Runtime.InteropServices; +using Joveler.Compression.XZ; namespace Packaging.Targets.IO { /// /// Represents a which can decompress xz-compressed data. + /// This is a thin wrapper around Joveler.Compression.XZ that mimics the original XZInputStream behavior. /// public class XZInputStream : Stream { - /// - /// The size of the buffer - /// - private const int BufSize = 512; - - private readonly List internalBuffer = new List(); private readonly Stream innerStream; - private readonly IntPtr inbuf; - private readonly IntPtr outbuf; - private LzmaStream lzmaStream; - private long length; + private readonly XZStream xzStream; + private long position; - private bool disposed; + private long length; /// /// Initializes a new instance of the class. @@ -61,273 +53,174 @@ public XZInputStream(Stream stream) throw new ArgumentNullException(nameof(stream)); } + XZHelper.EnsureInitialized(); this.innerStream = stream; + var decompOpts = new XZDecompressOptions { LeaveOpen = true }; - var ret = NativeMethods.lzma_stream_decoder(ref this.lzmaStream, ulong.MaxValue, LzmaDecodeFlags.Concatenated); - - this.inbuf = Marshal.AllocHGlobal(BufSize); - this.outbuf = Marshal.AllocHGlobal(BufSize); - - this.lzmaStream.AvailIn = 0; - this.lzmaStream.NextOut = this.outbuf; - this.lzmaStream.AvailOut = BufSize; - - if (ret == LzmaResult.OK) - { - return; - } - - switch (ret) - { - case LzmaResult.MemError: - throw new Exception("Memory allocation failed"); - - case LzmaResult.OptionsError: - throw new Exception("Unsupported decompressor flags"); - - default: - throw new Exception("Unknown error, possibly a bug"); - } + this.xzStream = new XZStream(stream, decompOpts); } /// - public override bool CanRead - { - get - { - this.EnsureNotDisposed(); - return true; - } - } + public override bool CanRead => this.xzStream.CanRead; /// - public override bool CanSeek - { - get - { - this.EnsureNotDisposed(); - return false; - } - } + public override bool CanSeek => false; // XZ decompression streams don't support seeking /// - public override bool CanWrite - { - get - { - this.EnsureNotDisposed(); - return false; - } - } + public override bool CanWrite => false; // Input streams don't support writing /// public override long Length { get { - this.EnsureNotDisposed(); - - const int streamFooterSize = 12; - if (this.length == 0) { - var lzmaStreamFlags = default(LzmaStreamFlags); - var streamFooter = new byte[streamFooterSize]; - - this.innerStream.Seek(-streamFooterSize, SeekOrigin.End); - this.innerStream.Read(streamFooter, 0, streamFooterSize); - - NativeMethods.lzma_stream_footer_decode(ref lzmaStreamFlags, streamFooter); - var indexPointer = new byte[lzmaStreamFlags.BackwardSize]; - - this.innerStream.Seek(-streamFooterSize - (long)lzmaStreamFlags.BackwardSize, SeekOrigin.End); - this.innerStream.Read(indexPointer, 0, (int)lzmaStreamFlags.BackwardSize); - this.innerStream.Seek(0, SeekOrigin.Begin); - - var index = IntPtr.Zero; - var memLimit = ulong.MaxValue; - uint inPos = 0; - - NativeMethods.lzma_index_buffer_decode(ref index, ref memLimit, IntPtr.Zero, indexPointer, ref inPos, lzmaStreamFlags.BackwardSize); - - if (inPos != lzmaStreamFlags.BackwardSize) - { - NativeMethods.lzma_index_end(index, IntPtr.Zero); - throw new Exception("Index decoding failed!"); - } - - var uSize = NativeMethods.lzma_index_uncompressed_size(index); - - NativeMethods.lzma_index_end(index, IntPtr.Zero); - this.length = (long)uSize; - return this.length; - } - else - { - return this.length; + // Try to parse XZ format directly + this.length = this.ParseXZLength(); } + + return this.length; } } /// public override long Position { - get - { - this.EnsureNotDisposed(); - return this.position; - } - - set - { - this.EnsureNotDisposed(); - throw new NotSupportedException("XZ Stream does not support setting position"); - } + get => this.position; + set => throw new NotSupportedException("XZ Stream does not support setting position"); } /// - public override void Flush() - { - this.EnsureNotDisposed(); + public override void Flush() => this.xzStream.Flush(); - throw new NotSupportedException("XZ Stream does not support flush"); - } + /// + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException("XZ Stream does not support seek"); /// - public override long Seek(long offset, SeekOrigin origin) - { - this.EnsureNotDisposed(); + public override void SetLength(long value) => throw new NotSupportedException("XZ Stream does not support setting length"); - throw new NotSupportedException("XZ Stream does not support seek"); + /// + public override int Read(byte[] buffer, int offset, int count) + { + int bytesRead = this.xzStream.Read(buffer, offset, count); + this.position += bytesRead; + return bytesRead; } /// - public override void SetLength(long value) + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException("XZ Input stream does not support writing"); + + /// + protected override void Dispose(bool disposing) { - throw new NotSupportedException("XZ Stream does not support setting length"); + this.xzStream?.Dispose(); + base.Dispose(disposing); } - /// - /// Reads bytes from stream - /// - /// byte read or -1 on end of stream - public override int Read(byte[] buffer, int offset, int count) + private long ParseXZLength() { - this.EnsureNotDisposed(); + const int streamFooterSize = 12; - var action = LzmaAction.Run; + var streamFooter = new byte[streamFooterSize]; + this.innerStream.Seek(-streamFooterSize, SeekOrigin.End); + this.innerStream.Read(streamFooter, 0, streamFooterSize); - var readBuf = new byte[BufSize]; - var outManagedBuf = new byte[BufSize]; + var backwardSize = this.DecodeFooter(streamFooter); + var indexPointer = new byte[backwardSize]; - while (this.internalBuffer.Count < count) - { - if (this.lzmaStream.AvailIn == 0) - { - this.lzmaStream.AvailIn = (uint)this.innerStream.Read(readBuf, 0, readBuf.Length); - Marshal.Copy(readBuf, 0, this.inbuf, (int)this.lzmaStream.AvailIn); - this.lzmaStream.NextIn = this.inbuf; - - if (this.lzmaStream.AvailIn == 0) - { - action = LzmaAction.Finish; - } - } - - var ret = NativeMethods.lzma_code(ref this.lzmaStream, action); - - if (this.lzmaStream.AvailOut == 0 || ret == LzmaResult.StreamEnd) - { - var writeSize = BufSize - (int)this.lzmaStream.AvailOut; - Marshal.Copy(this.outbuf, outManagedBuf, 0, writeSize); + this.innerStream.Seek(-streamFooterSize - backwardSize, SeekOrigin.End); + this.innerStream.Read(indexPointer, 0, (int)backwardSize); + this.innerStream.Seek(0, SeekOrigin.Begin); - this.internalBuffer.AddRange(outManagedBuf); - var tail = outManagedBuf.Length - writeSize; - this.internalBuffer.RemoveRange(this.internalBuffer.Count - tail, tail); - - this.lzmaStream.NextOut = this.outbuf; - this.lzmaStream.AvailOut = BufSize; - } + return this.DecodeIndex(indexPointer); + } - if (ret != LzmaResult.OK) - { - if (ret == LzmaResult.StreamEnd) - { - break; - } + private long DecodeFooter(byte[] streamFooter) + { + // Verify footer magic "YZ" (bytes 10-11) + if (streamFooter[10] != 0x59 || streamFooter[11] != 0x5A) + { + throw new InvalidDataException("Invalid XZ stream footer"); + } - NativeMethods.lzma_end(ref this.lzmaStream); + // Extract backward size from footer (bytes 4-7, little-endian) + // This is the size of the index in multiples of 4, minus 1 + var backwardSizeEncoded = BitConverter.ToUInt32(streamFooter, 4); + var backwardSize = (backwardSizeEncoded + 1) * 4; - switch (ret) - { - case LzmaResult.MemError: - throw new Exception("Memory allocation failed"); + return (long)backwardSize; + } - case LzmaResult.FormatError: - throw new Exception("The input is not in the .xz format"); + private long DecodeIndex(byte[] indexData) + { + if (indexData.Length < 1) + { + throw new InvalidDataException("XZ index data is empty"); + } - case LzmaResult.OptionsError: - throw new Exception("Unsupported compression options"); + int pos = 0; - case LzmaResult.DataError: - throw new Exception("Compressed file is corrupt"); + // The index starts with a null byte (0x00) as an indicator + if (indexData[0] != 0x00) + { + throw new InvalidDataException($"XZ index should start with 0x00, but found 0x{indexData[0]:X2}"); + } - case LzmaResult.BufferError: - throw new Exception("Compressed file is truncated or otherwise corrupt"); + pos++; - default: - throw new Exception("Unknown error.Possibly a bug"); - } - } - } + // Read record count (variable-length integer) + var recordCount = this.ReadVarInt(indexData, ref pos); - if (this.internalBuffer.Count >= count) + if (recordCount == 0) { - this.internalBuffer.CopyTo(0, buffer, offset, count); - this.internalBuffer.RemoveRange(0, count); - this.position += count; - return count; + throw new InvalidDataException("XZ index contains no records"); } - else + + long totalUncompressedSize = 0; + + // Read each record + for (ulong i = 0; i < recordCount; i++) { - var intBufLength = this.internalBuffer.Count; - this.internalBuffer.CopyTo(0, buffer, offset, intBufLength); - this.internalBuffer.Clear(); - this.position += intBufLength; - return intBufLength; + // Each record contains unpadded size and uncompressed size + var unpaddedSize = this.ReadVarInt(indexData, ref pos); + var uncompressedSize = this.ReadVarInt(indexData, ref pos); + + totalUncompressedSize += (long)uncompressedSize; } - } - /// - public override void Write(byte[] buffer, int offset, int count) - { - throw new NotSupportedException("XZ Input stream does not support writing"); + return totalUncompressedSize; } - /// - protected override void Dispose(bool disposing) + private ulong ReadVarInt(byte[] data, ref int pos) { - if (this.disposed) + if (pos >= data.Length) { - return; + throw new InvalidDataException("Unexpected end of XZ index data while reading variable-length integer"); } - NativeMethods.lzma_end(ref this.lzmaStream); + ulong result = 0; + int shift = 0; - Marshal.FreeHGlobal(this.inbuf); - Marshal.FreeHGlobal(this.outbuf); - - base.Dispose(disposing); + while (pos < data.Length) + { + byte b = data[pos++]; + result |= (ulong)(b & 0x7F) << shift; - this.disposed = true; - } + if ((b & 0x80) == 0) + { + break; + } - private void EnsureNotDisposed() - { - if (this.disposed) - { - throw new ObjectDisposedException(nameof(XZInputStream)); + shift += 7; + if (shift >= 64) + { + throw new InvalidDataException("Variable-length integer too large in XZ index"); + } } + + return result; } } -} \ No newline at end of file +} diff --git a/Packaging.Targets/IO/XZOutputStream.cs b/Packaging.Targets/IO/XZOutputStream.cs index 1e769916..c5b72770 100644 --- a/Packaging.Targets/IO/XZOutputStream.cs +++ b/Packaging.Targets/IO/XZOutputStream.cs @@ -1,4 +1,4 @@ -/* +/* * The MIT License (MIT) * Copyright (c) 2015 Roman Belkov, Kirill Melentyev @@ -23,28 +23,29 @@ */ using System; -using System.Diagnostics; using System.IO; +using Joveler.Compression.XZ; namespace Packaging.Targets.IO { - public unsafe class XZOutputStream : Stream + /// + /// Represents a which can compress data using xz compression. + /// This is a clean wrapper around Joveler.Compression.XZ.XZStream that provides the original XZOutputStream API. + /// + public class XZOutputStream : Stream { /// /// Default compression preset. /// public const uint DefaultPreset = 6; - public const uint PresetExtremeFlag = (uint)1 << 31; - // You can tweak BufSize value to get optimal results - // of speed and chunk size - private const int BufSize = 4096; + /// + /// Default number of threads to use. + /// + public static readonly int DefaultThreads = Environment.ProcessorCount; - private readonly Stream innerStream; - private readonly bool leaveOpen; - private readonly byte[] outbuf; - private LzmaStream lzmaStream; - private bool disposed; + // Th stream we're wrapping. It is sealed, so we can't inherit from it + private readonly XZStream xzStream; public XZOutputStream(Stream s) : this(s, DefaultThreads) @@ -63,277 +64,84 @@ public XZOutputStream(Stream s, int threads, uint preset) public XZOutputStream(Stream s, int threads, uint preset, bool leaveOpen) { - this.innerStream = s; - this.leaveOpen = leaveOpen; - - LzmaResult ret; - if (threads == 1 || !NativeMethods.SupportsMultiThreading) + if (s == null) { - ret = NativeMethods.lzma_easy_encoder(ref this.lzmaStream, preset, LzmaCheck.Crc64); + throw new ArgumentNullException(nameof(s)); } - else - { - if (threads <= 0) - { - throw new ArgumentOutOfRangeException(nameof(threads)); - } - - if (threads > Environment.ProcessorCount) - { - Trace.TraceWarning("{0} threads required, but only {1} processors available", threads, Environment.ProcessorCount); - threads = Environment.ProcessorCount; - } - var mt = new LzmaMT() - { - preset = preset, - check = LzmaCheck.Crc64, - threads = (uint)threads - }; - ret = NativeMethods.lzma_stream_encoder_mt(ref this.lzmaStream, ref mt); - } - - if (ret == LzmaResult.OK) + if (threads <= 0) { - this.outbuf = new byte[BufSize]; - this.lzmaStream.AvailOut = BufSize; - return; + throw new ArgumentOutOfRangeException(nameof(threads)); } - GC.SuppressFinalize(this); - throw GetError(ret); - } - - ~XZOutputStream() - { - this.Dispose(false); - } - - public static int DefaultThreads => Environment.ProcessorCount; + XZHelper.EnsureInitialized(); - public static bool SupportsMultiThreading => NativeMethods.SupportsMultiThreading; + var compOpts = CreateCompressOptions(preset, leaveOpen); - /// - public override bool CanRead - { - get + // Use parallel compression if threads > 1 + if (threads > 1) { - this.EnsureNotDisposed(); - return false; + var threadOpts = CreateParallelOptions(threads); + this.xzStream = new XZStream(s, compOpts, threadOpts); } - } - - /// - public override bool CanSeek - { - get + else { - this.EnsureNotDisposed(); - return false; + this.xzStream = new XZStream(s, compOpts); } } /// - public override bool CanWrite - { - get - { - this.EnsureNotDisposed(); - return true; - } - } + public override bool CanRead => this.xzStream.CanRead; /// - public override long Length - { - get - { - this.EnsureNotDisposed(); - throw new NotSupportedException(); - } - } + public override bool CanSeek => this.xzStream.CanSeek; /// - public override long Position - { - get - { - this.EnsureNotDisposed(); - throw new NotSupportedException(); - } - - set - { - this.EnsureNotDisposed(); - throw new NotSupportedException(); - } - } - - /// - /// Single-call buffer encoding - /// - public static byte[] Encode(byte[] buffer, uint preset = DefaultPreset) - { - var res = new byte[(long)NativeMethods.lzma_stream_buffer_bound((UIntPtr)buffer.Length)]; - - UIntPtr outPos; - var ret = NativeMethods.lzma_easy_buffer_encode(preset, LzmaCheck.Crc64, null, buffer, (UIntPtr)buffer.Length, res, &outPos, (UIntPtr)res.Length); - if (ret != LzmaResult.OK) - { - throw GetError(ret); - } - - if ((long)outPos < res.Length) - { - Array.Resize(ref res, (int)(ulong)outPos); - } - - return res; - } + public override bool CanWrite => this.xzStream.CanWrite; /// - public override void Flush() - { - throw new NotSupportedException(); - } + public override long Length => this.xzStream.Length; /// - public override long Seek(long offset, SeekOrigin origin) - { - this.EnsureNotDisposed(); - throw new NotSupportedException(); - } + public override long Position { get => this.xzStream.Position; set => this.xzStream.Position = value; } /// - public override void SetLength(long value) - { - this.EnsureNotDisposed(); - throw new NotSupportedException(); - } + public override void Flush() => this.xzStream.Flush(); /// - public override int Read(byte[] buffer, int offset, int count) - { - this.EnsureNotDisposed(); - throw new NotSupportedException(); - } + public override long Seek(long offset, SeekOrigin origin) => this.xzStream.Seek(offset, origin); /// - public override void Write(byte[] buffer, int offset, int count) - { - this.EnsureNotDisposed(); - - if (count == 0) - { - return; - } - - var guard = buffer[checked((uint)offset + (uint)count) - 1]; - - if (this.lzmaStream.AvailIn != 0) - { - throw new InvalidOperationException(); - } - - this.lzmaStream.AvailIn = (uint)count; - do - { - LzmaResult ret; - fixed (byte* inbuf = &buffer[offset]) - { - this.lzmaStream.NextIn = (IntPtr)inbuf; - fixed (byte* outbuf = &this.outbuf[BufSize - this.lzmaStream.AvailOut]) - { - this.lzmaStream.NextOut = (IntPtr)outbuf; - ret = NativeMethods.lzma_code(ref this.lzmaStream, LzmaAction.Run); - } - - offset += (int)((ulong)this.lzmaStream.NextIn - (ulong)(IntPtr)inbuf); - } + public override void SetLength(long value) => this.xzStream.SetLength(value); - if (ret != LzmaResult.OK) - { - throw this.ThrowError(ret); - } + /// + public override int Read(byte[] buffer, int offset, int count) => this.xzStream.Read(buffer, offset, count); - if (this.lzmaStream.AvailOut == 0) - { - this.innerStream.Write(this.outbuf, 0, BufSize); - this.lzmaStream.AvailOut = BufSize; - } - } - while (this.lzmaStream.AvailIn != 0); - } + /// + public override void Write(byte[] buffer, int offset, int count) => this.xzStream.Write(buffer, offset, count); /// protected override void Dispose(bool disposing) { - // finish encoding only if all input has been successfully processed - if (this.lzmaStream.InternalState != IntPtr.Zero && this.lzmaStream.AvailIn == 0) - { - LzmaResult ret; - do - { - fixed (byte* outbuf = &this.outbuf[BufSize - (int)this.lzmaStream.AvailOut]) - { - this.lzmaStream.NextOut = (IntPtr)outbuf; - ret = NativeMethods.lzma_code(ref this.lzmaStream, LzmaAction.Finish); - } - - if (ret > LzmaResult.StreamEnd) - { - throw this.ThrowError(ret); - } - - var writeSize = BufSize - (int)this.lzmaStream.AvailOut; - if (writeSize != 0) - { - this.innerStream.Write(this.outbuf, 0, writeSize); - this.lzmaStream.AvailOut = BufSize; - } - } - while (ret != LzmaResult.StreamEnd); - } - - NativeMethods.lzma_end(ref this.lzmaStream); - - if (disposing && !this.leaveOpen) - { - this.innerStream?.Dispose(); - } - + this.xzStream?.Dispose(); base.Dispose(disposing); - - this.disposed = true; } - private static Exception GetError(LzmaResult ret) + private static XZCompressOptions CreateCompressOptions(uint preset, bool leaveOpen) { - switch (ret) + return new XZCompressOptions { - case LzmaResult.MemError: return new OutOfMemoryException("Memory allocation failed"); - case LzmaResult.OptionsError: return new ArgumentException("Specified preset is not supported"); - case LzmaResult.UnsupportedCheck: return new Exception("Specified integrity check is not supported"); - case LzmaResult.DataError: return new InvalidDataException("File size limits exceeded"); - default: return new Exception("Unknown error, possibly a bug: " + ret); - } + Level = (LzmaCompLevel)Math.Min((int)preset, 9), // Map preset to compression level + LeaveOpen = leaveOpen // Forward leaveOpen parameter directly + }; } - /// - /// Throws an exception if this stream is disposed of. - /// - private void EnsureNotDisposed() + private static XZParallelCompressOptions CreateParallelOptions(int threads) { - if (this.disposed) + return new XZParallelCompressOptions { - throw new ObjectDisposedException(nameof(XZOutputStream)); - } - } - - private Exception ThrowError(LzmaResult ret) - { - NativeMethods.lzma_end(ref this.lzmaStream); - return GetError(ret); + Threads = threads + }; } } -} \ No newline at end of file +} diff --git a/Packaging.Targets/Packaging.Targets.csproj b/Packaging.Targets/Packaging.Targets.csproj index 6d2d7cf9..467e5593 100644 --- a/Packaging.Targets/Packaging.Targets.csproj +++ b/Packaging.Targets/Packaging.Targets.csproj @@ -37,7 +37,8 @@ all - + +