From 4a43c330b6cc50510d4aad364ccef06d74a0f59e Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Mon, 15 Oct 2018 20:53:49 -0500 Subject: [PATCH] Add RemapStorage class --- LibHac/Save/Header.cs | 17 +++--- LibHac/Save/RemapStorage.cs | 116 ++++++++++++++++++++++++++++++++++++ LibHac/Save/RemapStream.cs | 42 +++---------- LibHac/Save/SaveFs.cs | 2 +- LibHac/Save/Savefile.cs | 111 +++++++++++++--------------------- hactoolnet/ProcessSave.cs | 44 ++++++++++---- 6 files changed, 208 insertions(+), 124 deletions(-) create mode 100644 LibHac/Save/RemapStorage.cs diff --git a/LibHac/Save/Header.cs b/LibHac/Save/Header.cs index e3f9fda6..17afa2f8 100644 --- a/LibHac/Save/Header.cs +++ b/LibHac/Save/Header.cs @@ -192,11 +192,11 @@ namespace LibHac.Save public class RemapHeader { - public string Magic { get; set; } - public uint MagicNum { get; set; } - public int MapEntryCount { get; set; } - public int MapSegmentCount { get; set; } - public int Field10 { get; set; } + public string Magic { get; } + public uint MagicNum { get; } + public int MapEntryCount { get; } + public int MapSegmentCount { get; } + public int SegmentBits { get; } public RemapHeader(BinaryReader reader) { @@ -204,7 +204,7 @@ namespace LibHac.Save MagicNum = reader.ReadUInt32(); MapEntryCount = reader.ReadInt32(); MapSegmentCount = reader.ReadInt32(); - Field10 = reader.ReadInt32(); + SegmentBits = reader.ReadInt32(); } } @@ -320,7 +320,8 @@ namespace LibHac.Save public long PhysicalOffset { get; } public long Size { get; } public int Alignment { get; } - public int StorageType { get; } + public int Field1C { get; } + public long VirtualOffsetEnd => VirtualOffset + Size; public long PhysicalOffsetEnd => PhysicalOffset + Size; internal RemapSegment Segment { get; set; } @@ -332,7 +333,7 @@ namespace LibHac.Save PhysicalOffset = reader.ReadInt64(); Size = reader.ReadInt64(); Alignment = reader.ReadInt32(); - StorageType = reader.ReadInt32(); + Field1C = reader.ReadInt32(); } } diff --git a/LibHac/Save/RemapStorage.cs b/LibHac/Save/RemapStorage.cs new file mode 100644 index 00000000..e61cfa4b --- /dev/null +++ b/LibHac/Save/RemapStorage.cs @@ -0,0 +1,116 @@ +using System; +using System.IO; +using LibHac.Streams; + +namespace LibHac.Save +{ + public class RemapStorage + { + private SharedStreamSource StreamSource { get; } + private RemapHeader Header { get; } + public MapEntry[] MapEntries { get; set; } + public RemapSegment[] Segments { get; set; } + + /// + /// Creates a new + /// + /// A of the main data of the RemapStream. + /// The object takes complete ownership of the Stream. + /// The header for this RemapStorage. + /// The remapping entries for this RemapStorage. + public RemapStorage(Stream data, RemapHeader header, MapEntry[] mapEntries) + { + StreamSource = new SharedStreamSource(data); + Header = header; + MapEntries = mapEntries; + + Segments = InitSegments(Header, MapEntries); + } + + public Stream OpenStream(long offset, long size) + { + int segmentIdx = GetSegmentFromVirtualOffset(offset); + long segmentOffset = GetOffsetFromVirtualOffset(offset); + + if (segmentIdx > Segments.Length) + { + throw new ArgumentOutOfRangeException(nameof(offset)); + } + + RemapSegment segment = Segments[GetSegmentFromVirtualOffset(offset)]; + + if (segmentOffset > segment.Length) + { + throw new ArgumentOutOfRangeException(nameof(offset)); + } + + Stream stream = new RemapStream(StreamSource.CreateStream(), segment); + + return new SubStream(stream, offset, size); + } + + public Stream OpenSegmentStream(int segment) + { + long offset = ToVirtualOffset(segment, 0); + long size = Segments[segment].Length; + + return OpenStream(offset, size); + } + + private static RemapSegment[] InitSegments(RemapHeader header, MapEntry[] mapEntries) + { + var segments = new RemapSegment[header.MapSegmentCount]; + int entryIdx = 0; + + for (int i = 0; i < header.MapSegmentCount; i++) + { + var seg = new RemapSegment(); + seg.Entries.Add(mapEntries[entryIdx]); + seg.Offset = mapEntries[entryIdx].VirtualOffset; + mapEntries[entryIdx].Segment = seg; + entryIdx++; + + while (entryIdx < mapEntries.Length && + mapEntries[entryIdx - 1].VirtualOffsetEnd == mapEntries[entryIdx].VirtualOffset) + { + mapEntries[entryIdx].Segment = seg; + mapEntries[entryIdx - 1].Next = mapEntries[entryIdx]; + seg.Entries.Add(mapEntries[entryIdx]); + entryIdx++; + } + + seg.Length = seg.Entries[seg.Entries.Count - 1].VirtualOffsetEnd - seg.Entries[0].VirtualOffset; + segments[i] = seg; + } + + return segments; + } + + private int GetSegmentFromVirtualOffset(long virtualOffset) + { + return (int)((ulong)virtualOffset >> (64 - Header.SegmentBits)); + } + + private long GetOffsetFromVirtualOffset(long virtualOffset) + { + return virtualOffset & GetOffsetMask(); + } + + private long ToVirtualOffset(int segment, long offset) + { + long seg = (segment << (64 - Header.SegmentBits)) & GetSegmentMask(); + long off = offset & GetOffsetMask(); + return seg | off; + } + + private long GetOffsetMask() + { + return (1 << (64 - Header.SegmentBits)) - 1; + } + + private long GetSegmentMask() + { + return ~GetOffsetMask(); + } + } +} diff --git a/LibHac/Save/RemapStream.cs b/LibHac/Save/RemapStream.cs index e1adab68..d74b12ec 100644 --- a/LibHac/Save/RemapStream.cs +++ b/LibHac/Save/RemapStream.cs @@ -9,40 +9,15 @@ namespace LibHac.Save { private long _position; private Stream BaseStream { get; } - public MapEntry[] MapEntries { get; set; } - public MapEntry CurrentEntry { get; set; } - public RemapSegment[] Segments { get; set; } + private RemapSegment Segment { get; } + private MapEntry CurrentEntry { get; set; } - public RemapStream(Stream baseStream, MapEntry[] entries, int segmentCount) + public RemapStream(Stream baseStream, RemapSegment segment) { BaseStream = baseStream; - MapEntries = entries; - Segments = new RemapSegment[segmentCount]; - - int entryIdx = 0; - for (int i = 0; i < segmentCount; i++) - { - var seg = new RemapSegment(); - seg.Entries.Add(MapEntries[entryIdx]); - seg.Offset = MapEntries[entryIdx].VirtualOffset; - MapEntries[entryIdx].Segment = seg; - entryIdx++; - - while (entryIdx < MapEntries.Length && - MapEntries[entryIdx - 1].VirtualOffsetEnd == MapEntries[entryIdx].VirtualOffset) - { - MapEntries[entryIdx].Segment = seg; - MapEntries[entryIdx - 1].Next = MapEntries[entryIdx]; - seg.Entries.Add(MapEntries[entryIdx]); - entryIdx++; - } - - seg.Length = seg.Entries[seg.Entries.Count - 1].VirtualOffsetEnd - seg.Entries[0].VirtualOffset; - Segments[i] = seg; - } - - CurrentEntry = GetMapEntry(0); - UpdateBaseStreamPosition(); + Segment = segment; + CurrentEntry = segment.Entries[0]; + Length = segment.Length; } public override int Read(byte[] buffer, int offset, int count) @@ -103,8 +78,7 @@ namespace LibHac.Save private MapEntry GetMapEntry(long offset) { - // todo: is O(n) search a possible performance issue? - MapEntry entry = MapEntries.FirstOrDefault(x => offset >= x.VirtualOffset && offset < x.VirtualOffsetEnd); + MapEntry entry = Segment.Entries.FirstOrDefault(x => offset >= x.VirtualOffset && offset < x.VirtualOffsetEnd); if (entry == null) throw new ArgumentOutOfRangeException(nameof(offset)); return entry; } @@ -120,7 +94,7 @@ namespace LibHac.Save public override bool CanRead => true; public override bool CanSeek => true; public override bool CanWrite => false; - public override long Length { get; } = -1; + public override long Length { get; } public override long Position { diff --git a/LibHac/Save/SaveFs.cs b/LibHac/Save/SaveFs.cs index 533db211..3ee0d7dd 100644 --- a/LibHac/Save/SaveFs.cs +++ b/LibHac/Save/SaveFs.cs @@ -6,8 +6,8 @@ namespace LibHac.Save { public class SaveFs { - private AllocationTable AllocationTable { get; } private SharedStreamSource StreamSource { get; } + private AllocationTable AllocationTable { get; } private SaveHeader Header { get; } public DirectoryEntry RootDirectory { get; private set; } diff --git a/LibHac/Save/Savefile.cs b/LibHac/Save/Savefile.cs index af3bd616..4135fb11 100644 --- a/LibHac/Save/Savefile.cs +++ b/LibHac/Save/Savefile.cs @@ -8,32 +8,18 @@ namespace LibHac.Save public class Savefile { public Header Header { get; } - private RemapStream FileRemap { get; } public SharedStreamSource SavefileSource { get; } - public SharedStreamSource FileRemapSource { get; } - private RemapStream MetaRemap { get; } - public SharedStreamSource MetaRemapSource { get; } + private JournalStream JournalStream { get; } public SharedStreamSource JournalStreamSource { get; } private HierarchicalIntegrityVerificationStream IvfcStream { get; } public SharedStreamSource IvfcStreamSource { get; } public SaveFs SaveFs { get; } - public Stream DuplexL1A { get; } - public Stream DuplexL1B { get; } - public Stream DuplexDataA { get; } - public Stream DuplexDataB { get; } - public LayeredDuplexFs DuplexData { get; } - public Stream JournalData { get; } + public RemapStorage DataRemapStorage { get; } + public RemapStorage MetaRemapStorage { get; } - public Stream JournalTable { get; } - public Stream JournalBitmapUpdatedPhysical { get; } - public Stream JournalBitmapUpdatedVirtual { get; } - public Stream JournalBitmapUnassigned { get; } - public Stream JournalLayer1Hash { get; } - public Stream JournalLayer2Hash { get; } - public Stream JournalLayer3Hash { get; } - public Stream JournalFat { get; } + public LayeredDuplexFs DuplexData { get; } public DirectoryEntry RootDirectory => SaveFs.RootDirectory; public FileEntry[] Files => SaveFs.Files; @@ -48,69 +34,59 @@ namespace LibHac.Save Header = new Header(keyset, reader); FsLayout layout = Header.Layout; - FileRemap = new RemapStream( - SavefileSource.CreateStream(layout.FileMapDataOffset, layout.FileMapDataSize), - Header.FileMapEntries, Header.FileRemap.MapSegmentCount); + DataRemapStorage = new RemapStorage(SavefileSource.CreateStream(layout.FileMapDataOffset, layout.FileMapDataSize), + Header.FileRemap, Header.FileMapEntries); - FileRemapSource = new SharedStreamSource(FileRemap); + DuplexData = InitDuplexStream(DataRemapStorage, Header); - var duplexLayers = new DuplexFsLayerInfo[3]; + MetaRemapStorage = new RemapStorage(DuplexData, Header.MetaRemap, Header.MetaMapEntries); - duplexLayers[0] = new DuplexFsLayerInfo - { - DataA = new MemoryStream(Header.DuplexMasterA), - DataB = new MemoryStream(Header.DuplexMasterB), - Info = Header.Duplex.Layers[0] - }; + Stream journalTable = MetaRemapStorage.OpenStream(layout.JournalTableOffset, layout.JournalTableSize); - duplexLayers[1] = new DuplexFsLayerInfo - { - DataA = FileRemapSource.CreateStream(layout.DuplexL1OffsetA, layout.DuplexL1Size), - DataB = FileRemapSource.CreateStream(layout.DuplexL1OffsetB, layout.DuplexL1Size), - Info = Header.Duplex.Layers[1] - }; + MappingEntry[] journalMap = JournalStream.ReadMappingEntries(journalTable, Header.Journal.MainDataBlockCount); - duplexLayers[2] = new DuplexFsLayerInfo - { - DataA = FileRemapSource.CreateStream(layout.DuplexDataOffsetA, layout.DuplexDataSize), - DataB = FileRemapSource.CreateStream(layout.DuplexDataOffsetB, layout.DuplexDataSize), - Info = Header.Duplex.Layers[2] - }; - - DuplexL1A = FileRemapSource.CreateStream(layout.DuplexL1OffsetA, layout.DuplexL1Size); - DuplexL1B = FileRemapSource.CreateStream(layout.DuplexL1OffsetB, layout.DuplexL1Size); - DuplexDataA = FileRemapSource.CreateStream(layout.DuplexDataOffsetA, layout.DuplexDataSize); - DuplexDataB = FileRemapSource.CreateStream(layout.DuplexDataOffsetB, layout.DuplexDataSize); - JournalData = FileRemapSource.CreateStream(layout.JournalDataOffset, layout.JournalDataSizeB + layout.SizeReservedArea); - - DuplexData = new LayeredDuplexFs(duplexLayers, Header.Layout.DuplexIndex == 1); - MetaRemap = new RemapStream(DuplexData, Header.MetaMapEntries, Header.MetaRemap.MapSegmentCount); - MetaRemapSource = new SharedStreamSource(MetaRemap); - - JournalTable = MetaRemapSource.CreateStream(layout.JournalTableOffset, layout.JournalTableSize); - JournalBitmapUpdatedPhysical = MetaRemapSource.CreateStream(layout.JournalBitmapUpdatedPhysicalOffset, layout.JournalBitmapUpdatedPhysicalSize); - JournalBitmapUpdatedVirtual = MetaRemapSource.CreateStream(layout.JournalBitmapUpdatedVirtualOffset, layout.JournalBitmapUpdatedVirtualSize); - JournalBitmapUnassigned = MetaRemapSource.CreateStream(layout.JournalBitmapUnassignedOffset, layout.JournalBitmapUnassignedSize); - JournalLayer1Hash = MetaRemapSource.CreateStream(layout.IvfcL1Offset, layout.IvfcL1Size); - JournalLayer2Hash = MetaRemapSource.CreateStream(layout.IvfcL2Offset, layout.IvfcL2Size); - JournalLayer3Hash = MetaRemapSource.CreateStream(layout.IvfcL3Offset, layout.IvfcL3Size); - JournalFat = MetaRemapSource.CreateStream(layout.FatOffset, layout.FatSize); - - MappingEntry[] journalMap = JournalStream.ReadMappingEntries(JournalTable, Header.Journal.MainDataBlockCount); - - SharedStream journalData = FileRemapSource.CreateStream(layout.JournalDataOffset, + Stream journalData = DataRemapStorage.OpenStream(layout.JournalDataOffset, layout.JournalDataSizeB + layout.SizeReservedArea); JournalStream = new JournalStream(journalData, journalMap, (int)Header.Journal.BlockSize); JournalStreamSource = new SharedStreamSource(JournalStream); IvfcStream = InitIvfcStream(integrityCheckLevel); - SaveFs = new SaveFs(IvfcStream, MetaRemapSource.CreateStream(layout.FatOffset, layout.FatSize), Header.Save); + SaveFs = new SaveFs(IvfcStream, MetaRemapStorage.OpenStream(layout.FatOffset, layout.FatSize), Header.Save); IvfcStreamSource = new SharedStreamSource(IvfcStream); } } + private static LayeredDuplexFs InitDuplexStream(RemapStorage baseStorage, Header header) + { + FsLayout layout = header.Layout; + var duplexLayers = new DuplexFsLayerInfo[3]; + + duplexLayers[0] = new DuplexFsLayerInfo + { + DataA = new MemoryStream(header.DuplexMasterA), + DataB = new MemoryStream(header.DuplexMasterB), + Info = header.Duplex.Layers[0] + }; + + duplexLayers[1] = new DuplexFsLayerInfo + { + DataA = baseStorage.OpenStream(layout.DuplexL1OffsetA, layout.DuplexL1Size), + DataB = baseStorage.OpenStream(layout.DuplexL1OffsetB, layout.DuplexL1Size), + Info = header.Duplex.Layers[1] + }; + + duplexLayers[2] = new DuplexFsLayerInfo + { + DataA = baseStorage.OpenStream(layout.DuplexDataOffsetA, layout.DuplexDataSize), + DataB = baseStorage.OpenStream(layout.DuplexDataOffsetB, layout.DuplexDataSize), + Info = header.Duplex.Layers[2] + }; + + return new LayeredDuplexFs(duplexLayers, layout.DuplexIndex == 1); + } + private HierarchicalIntegrityVerificationStream InitIvfcStream(IntegrityCheckLevel integrityCheckLevel) { IvfcHeader ivfc = Header.Ivfc; @@ -130,8 +106,8 @@ namespace LibHac.Save IvfcLevelHeader level = ivfc.LevelHeaders[i - 1]; Stream data = i == ivfcLevels - 1 - ? (Stream)JournalStream - : MetaRemapSource.CreateStream(level.LogicalOffset, level.HashDataSize); + ? JournalStream + : MetaRemapStorage.OpenStream(level.LogicalOffset, level.HashDataSize); initInfo[i] = new IntegrityVerificationInfo { @@ -155,7 +131,6 @@ namespace LibHac.Save return SaveFs.OpenFile(file); } - public bool FileExists(string filename) => SaveFs.FileExists(filename); public bool SignHeader(Keyset keyset) diff --git a/hactoolnet/ProcessSave.cs b/hactoolnet/ProcessSave.cs index 10d103b5..12b9a931 100644 --- a/hactoolnet/ProcessSave.cs +++ b/hactoolnet/ProcessSave.cs @@ -26,25 +26,43 @@ namespace hactoolnet string dir = ctx.Options.DebugOutDir; Directory.CreateDirectory(dir); + FsLayout layout = save.Header.Layout; + File.WriteAllBytes(Path.Combine(dir, "L0_0_MasterHashA"), save.Header.MasterHashA); File.WriteAllBytes(Path.Combine(dir, "L0_1_MasterHashB"), save.Header.MasterHashB); File.WriteAllBytes(Path.Combine(dir, "L0_2_DuplexMasterA"), save.Header.DuplexMasterA); File.WriteAllBytes(Path.Combine(dir, "L0_3_DuplexMasterB"), save.Header.DuplexMasterB); - save.DuplexL1A.WriteAllBytes(Path.Combine(dir, "L0_4_DuplexL1A"), ctx.Logger); - save.DuplexL1B.WriteAllBytes(Path.Combine(dir, "L0_5_DuplexL1B"), ctx.Logger); - save.DuplexDataA.WriteAllBytes(Path.Combine(dir, "L0_6_DuplexDataA"), ctx.Logger); - save.DuplexDataB.WriteAllBytes(Path.Combine(dir, "L0_7_DuplexDataB"), ctx.Logger); - save.JournalData.WriteAllBytes(Path.Combine(dir, "L0_9_JournalData"), ctx.Logger); + Stream duplexL1A = save.DataRemapStorage.OpenStream(layout.DuplexL1OffsetA, layout.DuplexL1Size); + Stream duplexL1B = save.DataRemapStorage.OpenStream(layout.DuplexL1OffsetB, layout.DuplexL1Size); + Stream duplexDataA = save.DataRemapStorage.OpenStream(layout.DuplexDataOffsetA, layout.DuplexDataSize); + Stream duplexDataB = save.DataRemapStorage.OpenStream(layout.DuplexDataOffsetB, layout.DuplexDataSize); + Stream journalData = save.DataRemapStorage.OpenStream(layout.JournalDataOffset, layout.JournalDataSizeB + layout.SizeReservedArea); + + duplexL1A.WriteAllBytes(Path.Combine(dir, "L0_4_DuplexL1A"), ctx.Logger); + duplexL1B.WriteAllBytes(Path.Combine(dir, "L0_5_DuplexL1B"), ctx.Logger); + duplexDataA.WriteAllBytes(Path.Combine(dir, "L0_6_DuplexDataA"), ctx.Logger); + duplexDataB.WriteAllBytes(Path.Combine(dir, "L0_7_DuplexDataB"), ctx.Logger); + journalData.WriteAllBytes(Path.Combine(dir, "L0_9_JournalData"), ctx.Logger); save.DuplexData.WriteAllBytes(Path.Combine(dir, "L1_0_DuplexData"), ctx.Logger); - save.JournalTable.WriteAllBytes(Path.Combine(dir, "L2_0_JournalTable"), ctx.Logger); - save.JournalBitmapUpdatedPhysical.WriteAllBytes(Path.Combine(dir, "L2_1_JournalBitmapUpdatedPhysical"), ctx.Logger); - save.JournalBitmapUpdatedVirtual.WriteAllBytes(Path.Combine(dir, "L2_2_JournalBitmapUpdatedVirtual"), ctx.Logger); - save.JournalBitmapUnassigned.WriteAllBytes(Path.Combine(dir, "L2_3_JournalBitmapUnassigned"), ctx.Logger); - save.JournalLayer1Hash.WriteAllBytes(Path.Combine(dir, "L2_4_Layer1Hash"), ctx.Logger); - save.JournalLayer2Hash.WriteAllBytes(Path.Combine(dir, "L2_5_Layer2Hash"), ctx.Logger); - save.JournalLayer3Hash.WriteAllBytes(Path.Combine(dir, "L2_6_Layer3Hash"), ctx.Logger); - save.JournalFat.WriteAllBytes(Path.Combine(dir, "L2_7_FAT"), ctx.Logger); + + Stream journalTable = save.MetaRemapStorage.OpenStream(layout.JournalTableOffset, layout.JournalTableSize); + Stream journalBitmapUpdatedPhysical = save.MetaRemapStorage.OpenStream(layout.JournalBitmapUpdatedPhysicalOffset, layout.JournalBitmapUpdatedPhysicalSize); + Stream journalBitmapUpdatedVirtual = save.MetaRemapStorage.OpenStream(layout.JournalBitmapUpdatedVirtualOffset, layout.JournalBitmapUpdatedVirtualSize); + Stream journalBitmapUnassigned = save.MetaRemapStorage.OpenStream(layout.JournalBitmapUnassignedOffset, layout.JournalBitmapUnassignedSize); + Stream journalLayer1Hash = save.MetaRemapStorage.OpenStream(layout.IvfcL1Offset, layout.IvfcL1Size); + Stream journalLayer2Hash = save.MetaRemapStorage.OpenStream(layout.IvfcL2Offset, layout.IvfcL2Size); + Stream journalLayer3Hash = save.MetaRemapStorage.OpenStream(layout.IvfcL3Offset, layout.IvfcL3Size); + Stream journalFat = save.MetaRemapStorage.OpenStream(layout.FatOffset, layout.FatSize); + + journalTable.WriteAllBytes(Path.Combine(dir, "L2_0_JournalTable"), ctx.Logger); + journalBitmapUpdatedPhysical.WriteAllBytes(Path.Combine(dir, "L2_1_JournalBitmapUpdatedPhysical"), ctx.Logger); + journalBitmapUpdatedVirtual.WriteAllBytes(Path.Combine(dir, "L2_2_JournalBitmapUpdatedVirtual"), ctx.Logger); + journalBitmapUnassigned.WriteAllBytes(Path.Combine(dir, "L2_3_JournalBitmapUnassigned"), ctx.Logger); + journalLayer1Hash.WriteAllBytes(Path.Combine(dir, "L2_4_Layer1Hash"), ctx.Logger); + journalLayer2Hash.WriteAllBytes(Path.Combine(dir, "L2_5_Layer2Hash"), ctx.Logger); + journalLayer3Hash.WriteAllBytes(Path.Combine(dir, "L2_6_Layer3Hash"), ctx.Logger); + journalFat.WriteAllBytes(Path.Combine(dir, "L2_7_FAT"), ctx.Logger); save.IvfcStreamSource.CreateStream().WriteAllBytes(Path.Combine(dir, "L3_0_SaveData"), ctx.Logger); }