Add KipReader class and add kip decompression to hactoolnet

build/CodeGen/results.csv

@@ -306,3 +306,10 @@ Module,DescriptionStart,DescriptionEnd,Name,Summary
 428,2,,NullArgument,
 428,3,,ArgumentOutOfRange,
 428,4,,BufferTooSmall,
+
+428,1000,1999,InvalidData,
+428,1001,1019,InvalidKip,
+428,1002,,InvalidKipFileSize,The size of the KIP file was smaller than expected.
+428,1003,,InvalidKipMagic,The magic value of the KIP file was not KIP1.
+428,1004,,InvalidKipSegmentSize,The size of the compressed KIP segment was smaller than expected.
+428,1005,,KipSegmentDecompressionFailed,An error occurred while decompressing a KIP segment.

src/LibHac/Common/ResultLibHac.cs

@@ -9,6 +9,8 @@
 // code generation portion of the build.
 //-----------------------------------------------------------------------------
 
+using System.Runtime.CompilerServices;
+
 namespace LibHac.Common
 {
     public static class ResultLibHac
@@ -23,5 +25,18 @@ namespace LibHac.Common
             public static Result.Base ArgumentOutOfRange => new Result.Base(ModuleLibHac, 3);
             /// <summary>Error code: 2428-0004; Inner value: 0x9ac</summary>
             public static Result.Base BufferTooSmall => new Result.Base(ModuleLibHac, 4);
+
+        /// <summary>Error code: 2428-1000; Range: 1000-1999; Inner value: 0x7d1ac</summary>
+        public static Result.Base InvalidData { [MethodImpl(MethodImplOptions.AggressiveInlining)] get => new Result.Base(ModuleLibHac, 1000, 1999); }
+            /// <summary>Error code: 2428-1001; Range: 1001-1019; Inner value: 0x7d3ac</summary>
+            public static Result.Base InvalidKip { [MethodImpl(MethodImplOptions.AggressiveInlining)] get => new Result.Base(ModuleLibHac, 1001, 1019); }
+                /// <summary>The size of the KIP file was smaller than expected.<br/>Error code: 2428-1002; Inner value: 0x7d5ac</summary>
+                public static Result.Base InvalidKipFileSize => new Result.Base(ModuleLibHac, 1002);
+                /// <summary>The magic value of the KIP file was not KIP1.<br/>Error code: 2428-1003; Inner value: 0x7d7ac</summary>
+                public static Result.Base InvalidKipMagic => new Result.Base(ModuleLibHac, 1003);
+                /// <summary>The size of the compressed KIP segment was smaller than expected.<br/>Error code: 2428-1004; Inner value: 0x7d9ac</summary>
+                public static Result.Base InvalidKipSegmentSize => new Result.Base(ModuleLibHac, 1004);
+                /// <summary>An error occurred while decompressing a KIP segment.<br/>Error code: 2428-1005; Inner value: 0x7dbac</summary>
+                public static Result.Base KipSegmentDecompressionFailed => new Result.Base(ModuleLibHac, 1005);
     }
 }

src/LibHac/Kip.cs

@@ -5,6 +5,7 @@ using LibHac.FsSystem;
 
 namespace LibHac
 {
+    [Obsolete("This class has been deprecated. LibHac.Loader.KipReader should be used instead.")]
     public class Kip
     {
         private const int HeaderSize = 0x100;

src/LibHac/Loader/KipHeader.cs

@@ -0,0 +1,76 @@
+using System;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using LibHac.Common;
+
+namespace LibHac.Loader
+{
+    [StructLayout(LayoutKind.Explicit, Size = 0x100)]
+    public struct KipHeader
+    {
+        public const uint Kip1Magic = 0x3150494B; // KIP1
+        public const int NameSize = 12;
+        public const int SegmentCount = 6;
+
+        [FieldOffset(0x00)] public uint Magic;
+
+        [FieldOffset(0x04)] private byte _name;
+
+        [FieldOffset(0x10)] public ulong ProgramId;
+        [FieldOffset(0x18)] public int Version;
+
+        [FieldOffset(0x1C)] public byte Priority;
+        [FieldOffset(0x1D)] public byte IdealCoreId;
+        [FieldOffset(0x1F)] public Flag Flags;
+
+        [FieldOffset(0x20)] public int TextMemoryOffset;
+        [FieldOffset(0x24)] public int TextSize;
+        [FieldOffset(0x28)] public int TextFileSize;
+
+        [FieldOffset(0x2C)] public int AffinityMask;
+
+        [FieldOffset(0x30)] public int RoMemoryOffset;
+        [FieldOffset(0x34)] public int RoSize;
+        [FieldOffset(0x38)] public int RoFileSize;
+
+        [FieldOffset(0x3C)] public int StackSize;
+
+        [FieldOffset(0x40)] public int DataMemoryOffset;
+        [FieldOffset(0x44)] public int DataSize;
+        [FieldOffset(0x48)] public int DataFileSize;
+
+        [FieldOffset(0x50)] public int BssMemoryOffset;
+        [FieldOffset(0x54)] public int BssSize;
+        [FieldOffset(0x58)] public int BssFileSize;
+
+        [FieldOffset(0x80)] private uint _capabilities;
+
+        public Span<byte> Name => SpanHelpers.CreateSpan(ref _name, NameSize);
+
+        public Span<SegmentHeader> Segments =>
+            SpanHelpers.CreateSpan(ref Unsafe.As<int, SegmentHeader>(ref TextMemoryOffset), SegmentCount);
+
+        public Span<uint> Capabilities => SpanHelpers.CreateSpan(ref _capabilities, 0x80 / sizeof(uint));
+
+        public bool IsValid => Magic == Kip1Magic;
+
+        [Flags]
+        public enum Flag : byte
+        {
+            TextCompress = 1 << 0,
+            RoCompress = 1 << 1,
+            DataCompress = 1 << 2,
+            Is64BitInstruction = 1 << 3,
+            ProcessAddressSpace64Bit = 1 << 4,
+            UseSecureMemory = 1 << 5
+        }
+
+        [StructLayout(LayoutKind.Sequential, Size = 0x10)]
+        public struct SegmentHeader
+        {
+            public int MemoryOffset;
+            public int Size;
+            public int FileSize;
+        }
+    }
+}

src/LibHac/Loader/KipReader.cs

@@ -0,0 +1,265 @@
+using System;
+using System.Buffers.Binary;
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using LibHac.Common;
+using LibHac.Fs;
+
+namespace LibHac.Loader
+{
+    public class KipReader
+    {
+        private IFile KipFile { get; set; }
+
+        private KipHeader _header;
+
+        public ReadOnlySpan<uint> Capabilities => _header.Capabilities;
+        public U8Span Name => new U8Span(_header.Name);
+
+        public ulong ProgramId => _header.ProgramId;
+        public int Version => _header.Version;
+        public byte Priority => _header.Priority;
+        public byte IdealCoreId => _header.IdealCoreId;
+
+        public bool IsTextCompressed => _header.Flags.HasFlag(KipHeader.Flag.TextCompress);
+        public bool IsRoCompressed => _header.Flags.HasFlag(KipHeader.Flag.RoCompress);
+        public bool IsDataCompressed => _header.Flags.HasFlag(KipHeader.Flag.DataCompress);
+        public bool Is64Bit => _header.Flags.HasFlag(KipHeader.Flag.Is64BitInstruction);
+        public bool Is64BitAddressSpace => _header.Flags.HasFlag(KipHeader.Flag.ProcessAddressSpace64Bit);
+        public bool UsesSecureMemory => _header.Flags.HasFlag(KipHeader.Flag.UseSecureMemory);
+
+        public ReadOnlySpan<KipHeader.SegmentHeader> Segments => _header.Segments;
+
+        public int AffinityMask => _header.AffinityMask;
+        public int StackSize => _header.StackSize;
+
+        public Result Initialize(IFile kipFile)
+        {
+            if (kipFile is null)
+                return ResultLibHac.NullArgument.Log();
+
+            Result rc = kipFile.Read(out long bytesRead, 0, SpanHelpers.AsByteSpan(ref _header), ReadOption.None);
+            if (rc.IsFailure()) return rc;
+
+            if (bytesRead != Unsafe.SizeOf<KipHeader>())
+                return ResultLibHac.InvalidKipFileSize.Log();
+
+            if (!_header.IsValid)
+                return ResultLibHac.InvalidKipMagic.Log();
+
+            KipFile = kipFile;
+            return Result.Success;
+        }
+
+        public Result GetSegmentSize(SegmentType segment, out int size)
+        {
+            switch (segment)
+            {
+                case SegmentType.Text:
+                case SegmentType.Ro:
+                case SegmentType.Data:
+                case SegmentType.Bss:
+                case SegmentType.Reserved1:
+                case SegmentType.Reserved2:
+                    size = _header.Segments[(int)segment].Size;
+                    return Result.Success;
+                default:
+                    size = default;
+                    return ResultLibHac.ArgumentOutOfRange.Log();
+            }
+        }
+
+        public int GetUncompressedSize()
+        {
+            int size = Unsafe.SizeOf<KipHeader>();
+
+            for (int i = 0; i < Segments.Length; i++)
+            {
+                if (Segments[i].FileSize != 0)
+                {
+                    size += Segments[i].Size;
+                }
+            }
+
+            return size;
+        }
+
+        public Result ReadSegment(SegmentType segment, Span<byte> buffer)
+        {
+            Result rc = GetSegmentSize(segment, out int segmentSize);
+            if (rc.IsFailure()) return rc;
+
+            if (buffer.Length < segmentSize)
+                return ResultLibHac.BufferTooSmall.Log();
+
+            KipHeader.SegmentHeader segmentHeader = Segments[(int)segment];
+
+            // Return early for empty segments.
+            if (segmentHeader.Size == 0)
+                return Result.Success;
+
+            // The segment is all zeros if it has no data.
+            if (segmentHeader.FileSize == 0)
+            {
+                buffer.Slice(0, segmentHeader.Size).Clear();
+                return Result.Success;
+            }
+
+            int offset = CalculateSegmentOffset((int)segment);
+
+            // Read the segment data.
+            rc = KipFile.Read(out long bytesRead, offset, buffer.Slice(0, segmentHeader.FileSize), ReadOption.None);
+            if (rc.IsFailure()) return rc;
+
+            if (bytesRead != segmentHeader.FileSize)
+                return ResultLibHac.InvalidKipFileSize.Log();
+
+            // Decompress if necessary.
+            bool isCompressed = segment switch
+            {
+                SegmentType.Text => IsTextCompressed,
+                SegmentType.Ro => IsRoCompressed,
+                SegmentType.Data => IsDataCompressed,
+                _ => false
+            };
+
+            if (isCompressed)
+            {
+                rc = DecompressBlz(buffer, segmentHeader.FileSize);
+                if (rc.IsFailure()) return rc;
+            }
+
+            return Result.Success;
+        }
+
+        public Result ReadUncompressedKip(Span<byte> buffer)
+        {
+            if (buffer.Length < GetUncompressedSize())
+                return ResultLibHac.BufferTooSmall.Log();
+
+            Span<byte> segmentBuffer = buffer.Slice(Unsafe.SizeOf<KipHeader>());
+
+            // Read each of the segments into the buffer.
+            for (int i = 0; i < Segments.Length; i++)
+            {
+                if (Segments[i].FileSize != 0)
+                {
+                    Result rc = ReadSegment((SegmentType)i, segmentBuffer);
+                    if (rc.IsFailure()) return rc;
+
+                    segmentBuffer = segmentBuffer.Slice(Segments[i].Size);
+                }
+            }
+
+            // Copy the header to the buffer and update the sizes and flags.
+            ref KipHeader header = ref Unsafe.As<byte, KipHeader>(ref MemoryMarshal.GetReference(buffer));
+            header = _header;
+
+            // Remove any compression flags.
+            const KipHeader.Flag compressFlagsMask =
+                ~(KipHeader.Flag.TextCompress | KipHeader.Flag.RoCompress | KipHeader.Flag.DataCompress);
+
+            header.Flags &= compressFlagsMask;
+
+            // Update each segment's uncompressed size.
+            foreach (ref KipHeader.SegmentHeader segment in header.Segments)
+            {
+                if (segment.FileSize != 0)
+                {
+                    segment.FileSize = segment.Size;
+                }
+            }
+
+            return Result.Success;
+        }
+
+        private int CalculateSegmentOffset(int index)
+        {
+            Debug.Assert((uint)index <= (uint)SegmentType.Reserved2);
+
+            int offset = Unsafe.SizeOf<KipHeader>();
+            ReadOnlySpan<KipHeader.SegmentHeader> segments = Segments;
+
+            for (int i = 0; i < index; i++)
+            {
+                offset += segments[i].FileSize;
+            }
+
+            return offset;
+        }
+
+        private static Result DecompressBlz(Span<byte> buffer, int compressedDataSize)
+        {
+            const int segmentFooterSize = 12;
+
+            if (buffer.Length < segmentFooterSize)
+                return ResultLibHac.InvalidKipSegmentSize.Log();
+
+            // Parse the footer, endian agnostic.
+            Span<byte> footer = buffer.Slice(compressedDataSize - segmentFooterSize);
+            int totalCompSize = BinaryPrimitives.ReadInt32LittleEndian(footer);
+            int footerSize = BinaryPrimitives.ReadInt32LittleEndian(footer.Slice(4));
+            int additionalSize = BinaryPrimitives.ReadInt32LittleEndian(footer.Slice(8));
+
+            if (buffer.Length < totalCompSize + additionalSize)
+                return ResultLibHac.BufferTooSmall.Log();
+
+            Span<byte> data = buffer;
+
+            int inOffset = totalCompSize - footerSize;
+            int outOffset = totalCompSize + additionalSize;
+
+            while (outOffset != 0)
+            {
+                byte control = data[--inOffset];
+
+                // Each bit in the control byte is a flag indicating compressed or not compressed.
+                for (int i = 0; i < 8; i++)
+                {
+                    if ((control & 0x80) != 0)
+                    {
+                        if (inOffset < 2)
+                            return ResultLibHac.KipSegmentDecompressionFailed.Log();
+
+                        inOffset -= 2;
+                        ushort segmentValue = BinaryPrimitives.ReadUInt16LittleEndian(data.Slice(inOffset));
+                        int segmentOffset = (segmentValue & 0x0FFF) + 3;
+                        int segmentSize = Math.Min(((segmentValue >> 12) & 0xF) + 3, outOffset);
+
+                        outOffset -= segmentSize;
+
+                        for (int j = 0; j < segmentSize; j++)
+                        {
+                            data[outOffset + j] = data[outOffset + segmentOffset + j];
+                        }
+                    }
+                    else
+                    {
+                        if (inOffset < 1)
+                            return ResultLibHac.KipSegmentDecompressionFailed.Log();
+
+                        // Copy directly.
+                        data[--outOffset] = data[--inOffset];
+                    }
+                    control <<= 1;
+
+                    if (outOffset == 0)
+                        return Result.Success;
+                }
+            }
+
+            return Result.Success;
+        }
+
+        public enum SegmentType
+        {
+            Text = 0,
+            Ro = 1,
+            Data = 2,
+            Bss = 3,
+            Reserved1 = 4,
+            Reserved2 = 5
+        }
+    }
+}

src/LibHac/Lz4.cs

@@ -82,6 +82,7 @@ namespace LibHac
             int cmpPos = 0;
             int decPos = 0;
 
+            // ReSharper disable once VariableHidesOuterVariable
             int GetLength(int length, ReadOnlySpan<byte> cmp)
             {
                 byte sum;

src/hactoolnet/CliParser.cs

@@ -38,6 +38,7 @@ namespace hactoolnet
             new CliOption("outdir", 1, (o, a) => o.OutDir = a[0]),
             new CliOption("outfile", 1, (o, a) => o.OutFile = a[0]),
             new CliOption("plaintext", 1, (o, a) => o.PlaintextOut = a[0]),
+            new CliOption("uncompressed", 1, (o, a) => o.UncompressedOut = a[0]),
             new CliOption("nspout", 1, (o, a) => o.NspOut = a[0]),
             new CliOption("sdseed", 1, (o, a) => o.SdSeed = a[0]),
             new CliOption("sdpath", 1, (o, a) => o.SdPath = a[0]),
@@ -221,6 +222,8 @@ namespace hactoolnet
             sb.AppendLine("  --romfsdir <dir>     Specify RomFS directory path.");
             sb.AppendLine("  --listromfs          List files in RomFS.");
             sb.AppendLine("  --basenca            Set Base NCA to use with update partitions.");
+            sb.AppendLine("KIP1 options:");
+            sb.AppendLine("  --uncompressed <f>   Specify file path for saving uncompressed KIP1.");
             sb.AppendLine("RomFS options:");
             sb.AppendLine("  --romfsdir <dir>     Specify RomFS directory path.");
             sb.AppendLine("  --listromfs          List files in RomFS.");

src/hactoolnet/Options.cs

@@ -29,6 +29,7 @@ namespace hactoolnet
         public string OutDir;
         public string OutFile;
         public string PlaintextOut;
+        public string UncompressedOut;
         public string SdSeed;
         public string NspOut;
         public string SdPath;

src/hactoolnet/ProcessKip.cs

@@ -1,6 +1,8 @@
 using System.IO;
 using LibHac;
+using LibHac.Fs;
 using LibHac.FsSystem;
+using LibHac.Loader;
 
 namespace hactoolnet
 {
@@ -8,10 +10,19 @@ namespace hactoolnet
     {
         public static void ProcessKip1(Context ctx)
         {
-            using (var file = new LocalStorage(ctx.Options.InFile, FileAccess.Read))
+            using (var file = new LocalFile(ctx.Options.InFile, OpenMode.Read))
             {
-                var kip = new Kip(file);
-                kip.OpenRawFile();
+                var kip = new KipReader();
+                kip.Initialize(file).ThrowIfFailure();
+
+                if (!string.IsNullOrWhiteSpace(ctx.Options.UncompressedOut))
+                {
+                    var uncompressed = new byte[kip.GetUncompressedSize()];
+
+                    kip.ReadUncompressedKip(uncompressed).ThrowIfFailure();
+                    
+                    File.WriteAllBytes(ctx.Options.UncompressedOut, uncompressed);
+                }
             }
         }