From ed6c3c2bed66e4b0e102dcf12c11e69458c20f94 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Thu, 20 Sep 2018 16:16:40 -0500 Subject: [PATCH] Add the option to enable IVFC integrity checks --- ...HierarchicalIntegrityVerificationStream.cs | 79 +++++++++++++ LibHac/IntegrityVerificationStream.cs | 98 ++++++++++++++++ LibHac/Nca.cs | 105 ++++++++++++------ LibHac/Savefile/Savefile.cs | 3 +- LibHac/SwitchFs.cs | 4 +- hactoolnet/CliParser.cs | 4 +- hactoolnet/Options.cs | 1 + hactoolnet/ProcessNca.cs | 14 +-- hactoolnet/ProcessSwitchFs.cs | 8 +- hactoolnet/ProcessXci.cs | 8 +- 10 files changed, 268 insertions(+), 56 deletions(-) create mode 100644 LibHac/HierarchicalIntegrityVerificationStream.cs create mode 100644 LibHac/IntegrityVerificationStream.cs diff --git a/LibHac/HierarchicalIntegrityVerificationStream.cs b/LibHac/HierarchicalIntegrityVerificationStream.cs new file mode 100644 index 00000000..cc70f05f --- /dev/null +++ b/LibHac/HierarchicalIntegrityVerificationStream.cs @@ -0,0 +1,79 @@ +using System; +using System.IO; +using LibHac.Streams; + +namespace LibHac +{ + public class HierarchicalIntegrityVerificationStream : Stream + { + public Stream[] Levels { get; } + public Stream DataLevel { get; } + public bool EnableIntegrityChecks { get; } + + public HierarchicalIntegrityVerificationStream(IntegrityVerificationInfo[] levelInfo, bool enableIntegrityChecks) + { + Levels = new Stream[levelInfo.Length]; + EnableIntegrityChecks = enableIntegrityChecks; + + Levels[0] = levelInfo[0].Data; + + for (int i = 1; i < Levels.Length; i++) + { + var levelData = new IntegrityVerificationStream(levelInfo[i].Data, Levels[i - 1], + levelInfo[i].BlockSizePower, enableIntegrityChecks); + + Levels[i] = new RandomAccessSectorStream(levelData); + } + + DataLevel = Levels[Levels.Length - 1]; + } + + public override void Flush() + { + throw new NotImplementedException(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + switch (origin) + { + case SeekOrigin.Begin: + Position = offset; + break; + case SeekOrigin.Current: + Position += offset; + break; + case SeekOrigin.End: + Position = Length - offset; + break; + } + + return Position; + } + + public override void SetLength(long value) + { + throw new NotImplementedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + return DataLevel.Read(buffer, offset, count); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotImplementedException(); + } + + public override bool CanRead => DataLevel.CanRead; + public override bool CanSeek => DataLevel.CanSeek; + public override bool CanWrite => false; + public override long Length => DataLevel.Length; + public override long Position + { + get => DataLevel.Position; + set => DataLevel.Position = value; + } + } +} diff --git a/LibHac/IntegrityVerificationStream.cs b/LibHac/IntegrityVerificationStream.cs new file mode 100644 index 00000000..4bfd745b --- /dev/null +++ b/LibHac/IntegrityVerificationStream.cs @@ -0,0 +1,98 @@ +using System; +using System.IO; +using System.Security.Cryptography; +using LibHac.Streams; + +namespace LibHac +{ + public class IntegrityVerificationStream : SectorStream + { + private const int DigestSize = 0x20; + + private Stream HashStream { get; } + public bool EnableIntegrityChecks { get; } + + private readonly byte[] _hashBuffer = new byte[DigestSize]; + private readonly SHA256 _hash = SHA256.Create(); + + public IntegrityVerificationStream(Stream dataStream, Stream hashStream, int blockSizePower, bool enableIntegrityChecks) + : base(dataStream, 1 << blockSizePower) + { + HashStream = hashStream; + EnableIntegrityChecks = enableIntegrityChecks; + } + + public override void Flush() + { + throw new NotImplementedException(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + switch (origin) + { + case SeekOrigin.Begin: + Position = offset; + break; + case SeekOrigin.Current: + Position += offset; + break; + case SeekOrigin.End: + Position = Length - offset; + break; + } + + return Position; + } + + public override void SetLength(long value) + { + throw new NotImplementedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + HashStream.Position = CurrentSector * DigestSize; + HashStream.Read(_hashBuffer, 0, DigestSize); + + // If a hash is zero the data for the entire block is zero + if (_hashBuffer.IsEmpty()) + { + Array.Clear(buffer, 0, SectorSize); + } + + int bytesRead = base.Read(buffer, 0, count); + + if (bytesRead < SectorSize) + { + // Pad out unused portion of block + Array.Clear(buffer, bytesRead, SectorSize - bytesRead); + } + + if (EnableIntegrityChecks && !Util.ArraysEqual(_hashBuffer, _hash.ComputeHash(buffer))) + { + throw new InvalidDataException("Hash error!"); + } + + return bytesRead; + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotImplementedException(); + } + + public override bool CanRead => true; + public override bool CanSeek => true; + public override bool CanWrite => false; + } + + /// + /// Information for creating an + /// + public class IntegrityVerificationInfo + { + public Stream Data { get; set; } + public int BlockSizePower { get; set; } + } +} diff --git a/LibHac/Nca.cs b/LibHac/Nca.cs index db312f0c..cd124c1b 100644 --- a/LibHac/Nca.cs +++ b/LibHac/Nca.cs @@ -61,7 +61,7 @@ namespace LibHac foreach (var pfsSection in Sections.Where(x => x != null && x.Type == SectionType.Pfs0)) { - var sectionStream = OpenSection(pfsSection.SectionNum, false); + Stream sectionStream = OpenSection(pfsSection.SectionNum, false, false); if (sectionStream == null) continue; var pfs = new Pfs(sectionStream); @@ -76,10 +76,10 @@ namespace LibHac return StreamSource.CreateStream(); } - public Stream OpenSection(int index, bool raw) + private Stream OpenRawSection(int index) { - if (Sections[index] == null) throw new ArgumentOutOfRangeException(nameof(index)); - var sect = Sections[index]; + NcaSection sect = Sections[index]; + if (sect == null) throw new ArgumentOutOfRangeException(nameof(index)); if (sect.SuperblockHashValidity == Validity.Invalid) return null; @@ -91,54 +91,87 @@ namespace LibHac switch (sect.Header.CryptType) { case SectionCryptType.None: - break; + return rawStream; case SectionCryptType.XTS: - break; + throw new NotImplementedException("NCA sections using XTS are not supported"); case SectionCryptType.CTR: - rawStream = new RandomAccessSectorStream(new Aes128CtrStream(rawStream, DecryptedKeys[2], offset, sect.Header.Ctr), false); - break; + return new RandomAccessSectorStream(new Aes128CtrStream(rawStream, DecryptedKeys[2], offset, sect.Header.Ctr), false); case SectionCryptType.BKTR: rawStream = new RandomAccessSectorStream( new BktrCryptoStream(rawStream, DecryptedKeys[2], 0, size, offset, sect.Header.Ctr, sect.Header.Bktr), false); - if (BaseNca == null) - { - return rawStream; - } - else - { - var baseSect = BaseNca.Sections.FirstOrDefault(x => x.Type == SectionType.Romfs); - if (baseSect == null) throw new InvalidDataException("Base NCA has no RomFS section"); + if (BaseNca == null) return rawStream; + + NcaSection baseSect = BaseNca.Sections.FirstOrDefault(x => x.Type == SectionType.Romfs); + if (baseSect == null) throw new InvalidDataException("Base NCA has no RomFS section"); + + Stream baseStream = BaseNca.OpenSection(baseSect.SectionNum, true, false); + return new Bktr(rawStream, baseStream, sect); - var baseStream = BaseNca.OpenSection(baseSect.SectionNum, true); - rawStream = new Bktr(rawStream, baseStream, sect); - } - break; default: throw new ArgumentOutOfRangeException(); } + } + + public Stream OpenSection(int index, bool raw, bool enableIntegrityChecks) + { + Stream rawStream = OpenRawSection(index); + NcaSection sect = Sections[index]; if (raw) return rawStream; switch (sect.Header.Type) { case SectionType.Pfs0: - offset = sect.Pfs0.Superblock.Pfs0Offset; - size = sect.Pfs0.Superblock.Pfs0Size; - break; + PfsSuperblock pfs0Superblock = sect.Pfs0.Superblock; + + return new SubStream(rawStream, pfs0Superblock.Pfs0Offset, pfs0Superblock.Pfs0Size); case SectionType.Romfs: - offset = sect.Header.Romfs.IvfcHeader.LevelHeaders[Romfs.IvfcMaxLevel - 1].LogicalOffset; - size = sect.Header.Romfs.IvfcHeader.LevelHeaders[Romfs.IvfcMaxLevel - 1].HashDataSize; - break; case SectionType.Bktr: - offset = sect.Header.Bktr.IvfcHeader.LevelHeaders[Romfs.IvfcMaxLevel - 1].LogicalOffset; - size = sect.Header.Bktr.IvfcHeader.LevelHeaders[Romfs.IvfcMaxLevel - 1].HashDataSize; - break; + + var romfsStreamSource = new SharedStreamSource(rawStream); + + IvfcHeader ivfc; + if (sect.Header.Type == SectionType.Romfs) + { + ivfc = sect.Header.Romfs.IvfcHeader; + } + else + { + ivfc = sect.Header.Bktr.IvfcHeader; + } + + return InitIvfcForRomfs(ivfc, romfsStreamSource, enableIntegrityChecks); default: throw new ArgumentOutOfRangeException(); } - return new SubStream(rawStream, offset, size); + } + private static HierarchicalIntegrityVerificationStream InitIvfcForRomfs(IvfcHeader ivfc, + SharedStreamSource romfsStreamSource, bool enableIntegrityChecks) + { + var initInfo = new IntegrityVerificationInfo[ivfc.NumLevels]; + + // Set the master hash + initInfo[0] = new IntegrityVerificationInfo + { + Data = new MemoryStream(ivfc.MasterHash), + BlockSizePower = 0 + }; + + for (int i = 1; i < ivfc.NumLevels; i++) + { + IvfcLevelHeader level = ivfc.LevelHeaders[i - 1]; + Stream data = romfsStreamSource.CreateStream(level.LogicalOffset, level.HashDataSize); + + initInfo[i] = new IntegrityVerificationInfo + { + Data = data, + BlockSizePower = level.BlockSize + }; + } + + return new HierarchicalIntegrityVerificationStream(initInfo, enableIntegrityChecks); } public void SetBaseNca(Nca baseNca) => BaseNca = baseNca; @@ -264,7 +297,7 @@ namespace LibHac return; } - var stream = OpenSection(index, true); + Stream stream = OpenSection(index, true, false); if (stream == null) return; if (expected == null) return; @@ -287,7 +320,7 @@ namespace LibHac { if (Sections[index] == null) throw new ArgumentOutOfRangeException(nameof(index)); var sect = Sections[index]; - var stream = OpenSection(index, true); + var stream = OpenSection(index, true, false); logger?.LogMessage($"Verifying section {index}..."); switch (sect.Type) @@ -390,12 +423,12 @@ namespace LibHac public static class NcaExtensions { - public static void ExportSection(this Nca nca, int index, string filename, bool raw = false, IProgressReport logger = null) + public static void ExportSection(this Nca nca, int index, string filename, bool raw = false, bool verify = false, IProgressReport logger = null) { if (index < 0 || index > 3) throw new IndexOutOfRangeException(); if (nca.Sections[index] == null) return; - var section = nca.OpenSection(index, raw); + var section = nca.OpenSection(index, raw, verify); var dir = Path.GetDirectoryName(filename); if (!string.IsNullOrWhiteSpace(dir)) Directory.CreateDirectory(dir); @@ -405,13 +438,13 @@ namespace LibHac } } - public static void ExtractSection(this Nca nca, int index, string outputDir, IProgressReport logger = null) + public static void ExtractSection(this Nca nca, int index, string outputDir, bool verify = false, IProgressReport logger = null) { if (index < 0 || index > 3) throw new IndexOutOfRangeException(); if (nca.Sections[index] == null) return; var section = nca.Sections[index]; - var stream = nca.OpenSection(index, false); + var stream = nca.OpenSection(index, false, verify); switch (section.Type) { diff --git a/LibHac/Savefile/Savefile.cs b/LibHac/Savefile/Savefile.cs index b7389d8d..6acf567a 100644 --- a/LibHac/Savefile/Savefile.cs +++ b/LibHac/Savefile/Savefile.cs @@ -127,8 +127,7 @@ namespace LibHac.Savefile { if (file.BlockIndex < 0) { - //todo replace - return JournalStreamSource.CreateStream(0, 0); + return Stream.Null; } return OpenFatBlock(file.BlockIndex, file.FileSize); diff --git a/LibHac/SwitchFs.cs b/LibHac/SwitchFs.cs index d429187a..60d86662 100644 --- a/LibHac/SwitchFs.cs +++ b/LibHac/SwitchFs.cs @@ -135,7 +135,7 @@ namespace LibHac var title = new Title(); // Meta contents always have 1 Partition FS section with 1 file in it - Stream sect = nca.OpenSection(0, false); + Stream sect = nca.OpenSection(0, false, true); var pfs0 = new Pfs(sect); var file = pfs0.OpenFile(pfs0.Files[0]); @@ -175,7 +175,7 @@ namespace LibHac { foreach (var title in Titles.Values.Where(x => x.ControlNca != null)) { - var romfs = new Romfs(title.ControlNca.OpenSection(0, false)); + var romfs = new Romfs(title.ControlNca.OpenSection(0, false, true)); var control = romfs.GetFile("/control.nacp"); var reader = new BinaryReader(new MemoryStream(control)); diff --git a/hactoolnet/CliParser.cs b/hactoolnet/CliParser.cs index 62bf0dde..84a213d7 100644 --- a/hactoolnet/CliParser.cs +++ b/hactoolnet/CliParser.cs @@ -13,6 +13,7 @@ namespace hactoolnet new CliOption("intype", 't', 1, (o, a) => o.InFileType = ParseFileType(a[0])), new CliOption("raw", 'r', 0, (o, a) => o.Raw = true), new CliOption("verify", 'y', 0, (o, a) => o.Validate = true), + new CliOption("enablehash", 'h', 0, (o, a) => o.EnableHash = true), new CliOption("keyset", 'k', 1, (o, a) => o.Keyfile = a[0]), new CliOption("titlekeys", 1, (o, a) => o.TitleKeyFile = a[0]), new CliOption("consolekeys", 1, (o, a) => o.ConsoleKeyFile = a[0]), @@ -145,7 +146,8 @@ namespace hactoolnet sb.AppendLine("Usage: hactoolnet.exe [options...] "); sb.AppendLine("Options:"); sb.AppendLine(" -r, --raw Keep raw data, don\'t unpack."); - sb.AppendLine(" -y, --verify Verify hashes."); + sb.AppendLine(" -y, --verify Verify all hashes in the input file."); + sb.AppendLine(" -h, --enablehash Enable hash checks when reading the input file."); sb.AppendLine(" -k, --keyset Load keys from an external file."); sb.AppendLine(" -t, --intype=type Specify input file type [nca, xci, romfs, pk11, pk21, ini1, kip1, switchfs, save, keygen]"); sb.AppendLine(" --titlekeys Load title keys from an external file."); diff --git a/hactoolnet/Options.cs b/hactoolnet/Options.cs index 44d1845a..e7fba555 100644 --- a/hactoolnet/Options.cs +++ b/hactoolnet/Options.cs @@ -9,6 +9,7 @@ namespace hactoolnet public FileType InFileType = FileType.Nca; public bool Raw; public bool Validate; + public bool EnableHash; public string Keyfile; public string TitleKeyFile; public string ConsoleKeyFile; diff --git a/hactoolnet/ProcessNca.cs b/hactoolnet/ProcessNca.cs index b01a336f..8557b3b0 100644 --- a/hactoolnet/ProcessNca.cs +++ b/hactoolnet/ProcessNca.cs @@ -25,12 +25,12 @@ namespace hactoolnet { if (ctx.Options.SectionOut[i] != null) { - nca.ExportSection(i, ctx.Options.SectionOut[i], ctx.Options.Raw, ctx.Logger); + nca.ExportSection(i, ctx.Options.SectionOut[i], ctx.Options.Raw, ctx.Options.EnableHash, ctx.Logger); } if (ctx.Options.SectionOutDir[i] != null) { - nca.ExtractSection(i, ctx.Options.SectionOutDir[i], ctx.Logger); + nca.ExtractSection(i, ctx.Options.SectionOutDir[i], ctx.Options.EnableHash, ctx.Logger); } if (ctx.Options.Validate && nca.Sections[i] != null) @@ -41,7 +41,7 @@ namespace hactoolnet if (ctx.Options.ListRomFs && nca.Sections[1] != null) { - var romfs = new Romfs(nca.OpenSection(1, false)); + var romfs = new Romfs(nca.OpenSection(1, false, ctx.Options.EnableHash)); foreach (var romfsFile in romfs.Files) { @@ -67,12 +67,12 @@ namespace hactoolnet if (ctx.Options.RomfsOut != null) { - nca.ExportSection(section.SectionNum, ctx.Options.RomfsOut, ctx.Options.Raw, ctx.Logger); + nca.ExportSection(section.SectionNum, ctx.Options.RomfsOut, ctx.Options.Raw, ctx.Options.EnableHash, ctx.Logger); } if (ctx.Options.RomfsOutDir != null) { - var romfs = new Romfs(nca.OpenSection(section.SectionNum, false)); + var romfs = new Romfs(nca.OpenSection(section.SectionNum, false, ctx.Options.EnableHash)); romfs.Extract(ctx.Options.RomfsOutDir, ctx.Logger); } } @@ -89,12 +89,12 @@ namespace hactoolnet if (ctx.Options.ExefsOut != null) { - nca.ExportSection(section.SectionNum, ctx.Options.ExefsOut, ctx.Options.Raw, ctx.Logger); + nca.ExportSection(section.SectionNum, ctx.Options.ExefsOut, ctx.Options.Raw, ctx.Options.EnableHash, ctx.Logger); } if (ctx.Options.ExefsOutDir != null) { - nca.ExtractSection(section.SectionNum, ctx.Options.ExefsOutDir, ctx.Logger); + nca.ExtractSection(section.SectionNum, ctx.Options.ExefsOutDir, ctx.Options.EnableHash, ctx.Logger); } } diff --git a/hactoolnet/ProcessSwitchFs.cs b/hactoolnet/ProcessSwitchFs.cs index 065682bf..540dd61b 100644 --- a/hactoolnet/ProcessSwitchFs.cs +++ b/hactoolnet/ProcessSwitchFs.cs @@ -54,12 +54,12 @@ namespace hactoolnet if (ctx.Options.ExefsOutDir != null) { - title.MainNca.ExtractSection(section.SectionNum, ctx.Options.ExefsOutDir, ctx.Logger); + title.MainNca.ExtractSection(section.SectionNum, ctx.Options.ExefsOutDir, ctx.Options.EnableHash, ctx.Logger); } if (ctx.Options.ExefsOut != null) { - title.MainNca.ExportSection(section.SectionNum, ctx.Options.ExefsOut, ctx.Options.Raw, ctx.Logger); + title.MainNca.ExportSection(section.SectionNum, ctx.Options.ExefsOut, ctx.Options.Raw, ctx.Options.EnableHash, ctx.Logger); } } @@ -94,13 +94,13 @@ namespace hactoolnet if (ctx.Options.RomfsOutDir != null) { - var romfs = new Romfs(title.MainNca.OpenSection(section.SectionNum, false)); + var romfs = new Romfs(title.MainNca.OpenSection(section.SectionNum, false, ctx.Options.EnableHash)); romfs.Extract(ctx.Options.RomfsOutDir, ctx.Logger); } if (ctx.Options.RomfsOut != null) { - title.MainNca.ExportSection(section.SectionNum, ctx.Options.RomfsOut, ctx.Options.Raw, ctx.Logger); + title.MainNca.ExportSection(section.SectionNum, ctx.Options.RomfsOut, ctx.Options.Raw, ctx.Options.EnableHash, ctx.Logger); } } diff --git a/hactoolnet/ProcessXci.cs b/hactoolnet/ProcessXci.cs index 7824d94b..612910b9 100644 --- a/hactoolnet/ProcessXci.cs +++ b/hactoolnet/ProcessXci.cs @@ -79,12 +79,12 @@ namespace hactoolnet if (ctx.Options.ExefsOutDir != null) { - mainNca.ExtractSection(exefsSection.SectionNum, ctx.Options.ExefsOutDir, ctx.Logger); + mainNca.ExtractSection(exefsSection.SectionNum, ctx.Options.ExefsOutDir, ctx.Options.EnableHash, ctx.Logger); } if (ctx.Options.ExefsOut != null) { - mainNca.ExportSection(exefsSection.SectionNum, ctx.Options.ExefsOut, ctx.Options.Raw, ctx.Logger); + mainNca.ExportSection(exefsSection.SectionNum, ctx.Options.ExefsOut, ctx.Options.Raw, ctx.Options.EnableHash, ctx.Logger); } } @@ -108,13 +108,13 @@ namespace hactoolnet if (ctx.Options.RomfsOutDir != null) { - var romfs = new Romfs(mainNca.OpenSection(romfsSection.SectionNum, false)); + var romfs = new Romfs(mainNca.OpenSection(romfsSection.SectionNum, false, ctx.Options.EnableHash)); romfs.Extract(ctx.Options.RomfsOutDir, ctx.Logger); } if (ctx.Options.RomfsOut != null) { - mainNca.ExportSection(romfsSection.SectionNum, ctx.Options.RomfsOut, ctx.Options.Raw, ctx.Logger); + mainNca.ExportSection(romfsSection.SectionNum, ctx.Options.RomfsOut, ctx.Options.Raw, ctx.Options.EnableHash, ctx.Logger); } } }