Add RemapStorage class

This commit is contained in:
Alex Barney 2018-10-15 20:53:49 -05:00
parent 75cb28364f
commit 4a43c330b6
6 changed files with 208 additions and 124 deletions

View File

@ -192,11 +192,11 @@ namespace LibHac.Save
public class RemapHeader
{
public string Magic { get; set; }
public uint MagicNum { get; set; }
public int MapEntryCount { get; set; }
public int MapSegmentCount { get; set; }
public int Field10 { get; set; }
public string Magic { get; }
public uint MagicNum { get; }
public int MapEntryCount { get; }
public int MapSegmentCount { get; }
public int SegmentBits { get; }
public RemapHeader(BinaryReader reader)
{
@ -204,7 +204,7 @@ namespace LibHac.Save
MagicNum = reader.ReadUInt32();
MapEntryCount = reader.ReadInt32();
MapSegmentCount = reader.ReadInt32();
Field10 = reader.ReadInt32();
SegmentBits = reader.ReadInt32();
}
}
@ -320,7 +320,8 @@ namespace LibHac.Save
public long PhysicalOffset { get; }
public long Size { get; }
public int Alignment { get; }
public int StorageType { get; }
public int Field1C { get; }
public long VirtualOffsetEnd => VirtualOffset + Size;
public long PhysicalOffsetEnd => PhysicalOffset + Size;
internal RemapSegment Segment { get; set; }
@ -332,7 +333,7 @@ namespace LibHac.Save
PhysicalOffset = reader.ReadInt64();
Size = reader.ReadInt64();
Alignment = reader.ReadInt32();
StorageType = reader.ReadInt32();
Field1C = reader.ReadInt32();
}
}

116
LibHac/Save/RemapStorage.cs Normal file
View File

@ -0,0 +1,116 @@
using System;
using System.IO;
using LibHac.Streams;
namespace LibHac.Save
{
public class RemapStorage
{
private SharedStreamSource StreamSource { get; }
private RemapHeader Header { get; }
public MapEntry[] MapEntries { get; set; }
public RemapSegment[] Segments { get; set; }
/// <summary>
/// Creates a new <see cref="RemapStorage"/>
/// </summary>
/// <param name="data">A <see cref="Stream"/> of the main data of the RemapStream.
/// The <see cref="RemapStorage"/> object takes complete ownership of the Stream.</param>
/// <param name="header">The header for this RemapStorage.</param>
/// <param name="mapEntries">The remapping entries for this RemapStorage.</param>
public RemapStorage(Stream data, RemapHeader header, MapEntry[] mapEntries)
{
StreamSource = new SharedStreamSource(data);
Header = header;
MapEntries = mapEntries;
Segments = InitSegments(Header, MapEntries);
}
public Stream OpenStream(long offset, long size)
{
int segmentIdx = GetSegmentFromVirtualOffset(offset);
long segmentOffset = GetOffsetFromVirtualOffset(offset);
if (segmentIdx > Segments.Length)
{
throw new ArgumentOutOfRangeException(nameof(offset));
}
RemapSegment segment = Segments[GetSegmentFromVirtualOffset(offset)];
if (segmentOffset > segment.Length)
{
throw new ArgumentOutOfRangeException(nameof(offset));
}
Stream stream = new RemapStream(StreamSource.CreateStream(), segment);
return new SubStream(stream, offset, size);
}
public Stream OpenSegmentStream(int segment)
{
long offset = ToVirtualOffset(segment, 0);
long size = Segments[segment].Length;
return OpenStream(offset, size);
}
private static RemapSegment[] InitSegments(RemapHeader header, MapEntry[] mapEntries)
{
var segments = new RemapSegment[header.MapSegmentCount];
int entryIdx = 0;
for (int i = 0; i < header.MapSegmentCount; i++)
{
var seg = new RemapSegment();
seg.Entries.Add(mapEntries[entryIdx]);
seg.Offset = mapEntries[entryIdx].VirtualOffset;
mapEntries[entryIdx].Segment = seg;
entryIdx++;
while (entryIdx < mapEntries.Length &&
mapEntries[entryIdx - 1].VirtualOffsetEnd == mapEntries[entryIdx].VirtualOffset)
{
mapEntries[entryIdx].Segment = seg;
mapEntries[entryIdx - 1].Next = mapEntries[entryIdx];
seg.Entries.Add(mapEntries[entryIdx]);
entryIdx++;
}
seg.Length = seg.Entries[seg.Entries.Count - 1].VirtualOffsetEnd - seg.Entries[0].VirtualOffset;
segments[i] = seg;
}
return segments;
}
private int GetSegmentFromVirtualOffset(long virtualOffset)
{
return (int)((ulong)virtualOffset >> (64 - Header.SegmentBits));
}
private long GetOffsetFromVirtualOffset(long virtualOffset)
{
return virtualOffset & GetOffsetMask();
}
private long ToVirtualOffset(int segment, long offset)
{
long seg = (segment << (64 - Header.SegmentBits)) & GetSegmentMask();
long off = offset & GetOffsetMask();
return seg | off;
}
private long GetOffsetMask()
{
return (1 << (64 - Header.SegmentBits)) - 1;
}
private long GetSegmentMask()
{
return ~GetOffsetMask();
}
}
}

View File

@ -9,40 +9,15 @@ namespace LibHac.Save
{
private long _position;
private Stream BaseStream { get; }
public MapEntry[] MapEntries { get; set; }
public MapEntry CurrentEntry { get; set; }
public RemapSegment[] Segments { get; set; }
private RemapSegment Segment { get; }
private MapEntry CurrentEntry { get; set; }
public RemapStream(Stream baseStream, MapEntry[] entries, int segmentCount)
public RemapStream(Stream baseStream, RemapSegment segment)
{
BaseStream = baseStream;
MapEntries = entries;
Segments = new RemapSegment[segmentCount];
int entryIdx = 0;
for (int i = 0; i < segmentCount; i++)
{
var seg = new RemapSegment();
seg.Entries.Add(MapEntries[entryIdx]);
seg.Offset = MapEntries[entryIdx].VirtualOffset;
MapEntries[entryIdx].Segment = seg;
entryIdx++;
while (entryIdx < MapEntries.Length &&
MapEntries[entryIdx - 1].VirtualOffsetEnd == MapEntries[entryIdx].VirtualOffset)
{
MapEntries[entryIdx].Segment = seg;
MapEntries[entryIdx - 1].Next = MapEntries[entryIdx];
seg.Entries.Add(MapEntries[entryIdx]);
entryIdx++;
}
seg.Length = seg.Entries[seg.Entries.Count - 1].VirtualOffsetEnd - seg.Entries[0].VirtualOffset;
Segments[i] = seg;
}
CurrentEntry = GetMapEntry(0);
UpdateBaseStreamPosition();
Segment = segment;
CurrentEntry = segment.Entries[0];
Length = segment.Length;
}
public override int Read(byte[] buffer, int offset, int count)
@ -103,8 +78,7 @@ namespace LibHac.Save
private MapEntry GetMapEntry(long offset)
{
// todo: is O(n) search a possible performance issue?
MapEntry entry = MapEntries.FirstOrDefault(x => offset >= x.VirtualOffset && offset < x.VirtualOffsetEnd);
MapEntry entry = Segment.Entries.FirstOrDefault(x => offset >= x.VirtualOffset && offset < x.VirtualOffsetEnd);
if (entry == null) throw new ArgumentOutOfRangeException(nameof(offset));
return entry;
}
@ -120,7 +94,7 @@ namespace LibHac.Save
public override bool CanRead => true;
public override bool CanSeek => true;
public override bool CanWrite => false;
public override long Length { get; } = -1;
public override long Length { get; }
public override long Position
{

View File

@ -6,8 +6,8 @@ namespace LibHac.Save
{
public class SaveFs
{
private AllocationTable AllocationTable { get; }
private SharedStreamSource StreamSource { get; }
private AllocationTable AllocationTable { get; }
private SaveHeader Header { get; }
public DirectoryEntry RootDirectory { get; private set; }

View File

@ -8,32 +8,18 @@ namespace LibHac.Save
public class Savefile
{
public Header Header { get; }
private RemapStream FileRemap { get; }
public SharedStreamSource SavefileSource { get; }
public SharedStreamSource FileRemapSource { get; }
private RemapStream MetaRemap { get; }
public SharedStreamSource MetaRemapSource { get; }
private JournalStream JournalStream { get; }
public SharedStreamSource JournalStreamSource { get; }
private HierarchicalIntegrityVerificationStream IvfcStream { get; }
public SharedStreamSource IvfcStreamSource { get; }
public SaveFs SaveFs { get; }
public Stream DuplexL1A { get; }
public Stream DuplexL1B { get; }
public Stream DuplexDataA { get; }
public Stream DuplexDataB { get; }
public LayeredDuplexFs DuplexData { get; }
public Stream JournalData { get; }
public RemapStorage DataRemapStorage { get; }
public RemapStorage MetaRemapStorage { get; }
public Stream JournalTable { get; }
public Stream JournalBitmapUpdatedPhysical { get; }
public Stream JournalBitmapUpdatedVirtual { get; }
public Stream JournalBitmapUnassigned { get; }
public Stream JournalLayer1Hash { get; }
public Stream JournalLayer2Hash { get; }
public Stream JournalLayer3Hash { get; }
public Stream JournalFat { get; }
public LayeredDuplexFs DuplexData { get; }
public DirectoryEntry RootDirectory => SaveFs.RootDirectory;
public FileEntry[] Files => SaveFs.Files;
@ -48,69 +34,59 @@ namespace LibHac.Save
Header = new Header(keyset, reader);
FsLayout layout = Header.Layout;
FileRemap = new RemapStream(
SavefileSource.CreateStream(layout.FileMapDataOffset, layout.FileMapDataSize),
Header.FileMapEntries, Header.FileRemap.MapSegmentCount);
DataRemapStorage = new RemapStorage(SavefileSource.CreateStream(layout.FileMapDataOffset, layout.FileMapDataSize),
Header.FileRemap, Header.FileMapEntries);
FileRemapSource = new SharedStreamSource(FileRemap);
DuplexData = InitDuplexStream(DataRemapStorage, Header);
var duplexLayers = new DuplexFsLayerInfo[3];
MetaRemapStorage = new RemapStorage(DuplexData, Header.MetaRemap, Header.MetaMapEntries);
duplexLayers[0] = new DuplexFsLayerInfo
{
DataA = new MemoryStream(Header.DuplexMasterA),
DataB = new MemoryStream(Header.DuplexMasterB),
Info = Header.Duplex.Layers[0]
};
Stream journalTable = MetaRemapStorage.OpenStream(layout.JournalTableOffset, layout.JournalTableSize);
duplexLayers[1] = new DuplexFsLayerInfo
{
DataA = FileRemapSource.CreateStream(layout.DuplexL1OffsetA, layout.DuplexL1Size),
DataB = FileRemapSource.CreateStream(layout.DuplexL1OffsetB, layout.DuplexL1Size),
Info = Header.Duplex.Layers[1]
};
MappingEntry[] journalMap = JournalStream.ReadMappingEntries(journalTable, Header.Journal.MainDataBlockCount);
duplexLayers[2] = new DuplexFsLayerInfo
{
DataA = FileRemapSource.CreateStream(layout.DuplexDataOffsetA, layout.DuplexDataSize),
DataB = FileRemapSource.CreateStream(layout.DuplexDataOffsetB, layout.DuplexDataSize),
Info = Header.Duplex.Layers[2]
};
DuplexL1A = FileRemapSource.CreateStream(layout.DuplexL1OffsetA, layout.DuplexL1Size);
DuplexL1B = FileRemapSource.CreateStream(layout.DuplexL1OffsetB, layout.DuplexL1Size);
DuplexDataA = FileRemapSource.CreateStream(layout.DuplexDataOffsetA, layout.DuplexDataSize);
DuplexDataB = FileRemapSource.CreateStream(layout.DuplexDataOffsetB, layout.DuplexDataSize);
JournalData = FileRemapSource.CreateStream(layout.JournalDataOffset, layout.JournalDataSizeB + layout.SizeReservedArea);
DuplexData = new LayeredDuplexFs(duplexLayers, Header.Layout.DuplexIndex == 1);
MetaRemap = new RemapStream(DuplexData, Header.MetaMapEntries, Header.MetaRemap.MapSegmentCount);
MetaRemapSource = new SharedStreamSource(MetaRemap);
JournalTable = MetaRemapSource.CreateStream(layout.JournalTableOffset, layout.JournalTableSize);
JournalBitmapUpdatedPhysical = MetaRemapSource.CreateStream(layout.JournalBitmapUpdatedPhysicalOffset, layout.JournalBitmapUpdatedPhysicalSize);
JournalBitmapUpdatedVirtual = MetaRemapSource.CreateStream(layout.JournalBitmapUpdatedVirtualOffset, layout.JournalBitmapUpdatedVirtualSize);
JournalBitmapUnassigned = MetaRemapSource.CreateStream(layout.JournalBitmapUnassignedOffset, layout.JournalBitmapUnassignedSize);
JournalLayer1Hash = MetaRemapSource.CreateStream(layout.IvfcL1Offset, layout.IvfcL1Size);
JournalLayer2Hash = MetaRemapSource.CreateStream(layout.IvfcL2Offset, layout.IvfcL2Size);
JournalLayer3Hash = MetaRemapSource.CreateStream(layout.IvfcL3Offset, layout.IvfcL3Size);
JournalFat = MetaRemapSource.CreateStream(layout.FatOffset, layout.FatSize);
MappingEntry[] journalMap = JournalStream.ReadMappingEntries(JournalTable, Header.Journal.MainDataBlockCount);
SharedStream journalData = FileRemapSource.CreateStream(layout.JournalDataOffset,
Stream journalData = DataRemapStorage.OpenStream(layout.JournalDataOffset,
layout.JournalDataSizeB + layout.SizeReservedArea);
JournalStream = new JournalStream(journalData, journalMap, (int)Header.Journal.BlockSize);
JournalStreamSource = new SharedStreamSource(JournalStream);
IvfcStream = InitIvfcStream(integrityCheckLevel);
SaveFs = new SaveFs(IvfcStream, MetaRemapSource.CreateStream(layout.FatOffset, layout.FatSize), Header.Save);
SaveFs = new SaveFs(IvfcStream, MetaRemapStorage.OpenStream(layout.FatOffset, layout.FatSize), Header.Save);
IvfcStreamSource = new SharedStreamSource(IvfcStream);
}
}
private static LayeredDuplexFs InitDuplexStream(RemapStorage baseStorage, Header header)
{
FsLayout layout = header.Layout;
var duplexLayers = new DuplexFsLayerInfo[3];
duplexLayers[0] = new DuplexFsLayerInfo
{
DataA = new MemoryStream(header.DuplexMasterA),
DataB = new MemoryStream(header.DuplexMasterB),
Info = header.Duplex.Layers[0]
};
duplexLayers[1] = new DuplexFsLayerInfo
{
DataA = baseStorage.OpenStream(layout.DuplexL1OffsetA, layout.DuplexL1Size),
DataB = baseStorage.OpenStream(layout.DuplexL1OffsetB, layout.DuplexL1Size),
Info = header.Duplex.Layers[1]
};
duplexLayers[2] = new DuplexFsLayerInfo
{
DataA = baseStorage.OpenStream(layout.DuplexDataOffsetA, layout.DuplexDataSize),
DataB = baseStorage.OpenStream(layout.DuplexDataOffsetB, layout.DuplexDataSize),
Info = header.Duplex.Layers[2]
};
return new LayeredDuplexFs(duplexLayers, layout.DuplexIndex == 1);
}
private HierarchicalIntegrityVerificationStream InitIvfcStream(IntegrityCheckLevel integrityCheckLevel)
{
IvfcHeader ivfc = Header.Ivfc;
@ -130,8 +106,8 @@ namespace LibHac.Save
IvfcLevelHeader level = ivfc.LevelHeaders[i - 1];
Stream data = i == ivfcLevels - 1
? (Stream)JournalStream
: MetaRemapSource.CreateStream(level.LogicalOffset, level.HashDataSize);
? JournalStream
: MetaRemapStorage.OpenStream(level.LogicalOffset, level.HashDataSize);
initInfo[i] = new IntegrityVerificationInfo
{
@ -155,7 +131,6 @@ namespace LibHac.Save
return SaveFs.OpenFile(file);
}
public bool FileExists(string filename) => SaveFs.FileExists(filename);
public bool SignHeader(Keyset keyset)

View File

@ -26,25 +26,43 @@ namespace hactoolnet
string dir = ctx.Options.DebugOutDir;
Directory.CreateDirectory(dir);
FsLayout layout = save.Header.Layout;
File.WriteAllBytes(Path.Combine(dir, "L0_0_MasterHashA"), save.Header.MasterHashA);
File.WriteAllBytes(Path.Combine(dir, "L0_1_MasterHashB"), save.Header.MasterHashB);
File.WriteAllBytes(Path.Combine(dir, "L0_2_DuplexMasterA"), save.Header.DuplexMasterA);
File.WriteAllBytes(Path.Combine(dir, "L0_3_DuplexMasterB"), save.Header.DuplexMasterB);
save.DuplexL1A.WriteAllBytes(Path.Combine(dir, "L0_4_DuplexL1A"), ctx.Logger);
save.DuplexL1B.WriteAllBytes(Path.Combine(dir, "L0_5_DuplexL1B"), ctx.Logger);
save.DuplexDataA.WriteAllBytes(Path.Combine(dir, "L0_6_DuplexDataA"), ctx.Logger);
save.DuplexDataB.WriteAllBytes(Path.Combine(dir, "L0_7_DuplexDataB"), ctx.Logger);
save.JournalData.WriteAllBytes(Path.Combine(dir, "L0_9_JournalData"), ctx.Logger);
Stream duplexL1A = save.DataRemapStorage.OpenStream(layout.DuplexL1OffsetA, layout.DuplexL1Size);
Stream duplexL1B = save.DataRemapStorage.OpenStream(layout.DuplexL1OffsetB, layout.DuplexL1Size);
Stream duplexDataA = save.DataRemapStorage.OpenStream(layout.DuplexDataOffsetA, layout.DuplexDataSize);
Stream duplexDataB = save.DataRemapStorage.OpenStream(layout.DuplexDataOffsetB, layout.DuplexDataSize);
Stream journalData = save.DataRemapStorage.OpenStream(layout.JournalDataOffset, layout.JournalDataSizeB + layout.SizeReservedArea);
duplexL1A.WriteAllBytes(Path.Combine(dir, "L0_4_DuplexL1A"), ctx.Logger);
duplexL1B.WriteAllBytes(Path.Combine(dir, "L0_5_DuplexL1B"), ctx.Logger);
duplexDataA.WriteAllBytes(Path.Combine(dir, "L0_6_DuplexDataA"), ctx.Logger);
duplexDataB.WriteAllBytes(Path.Combine(dir, "L0_7_DuplexDataB"), ctx.Logger);
journalData.WriteAllBytes(Path.Combine(dir, "L0_9_JournalData"), ctx.Logger);
save.DuplexData.WriteAllBytes(Path.Combine(dir, "L1_0_DuplexData"), ctx.Logger);
save.JournalTable.WriteAllBytes(Path.Combine(dir, "L2_0_JournalTable"), ctx.Logger);
save.JournalBitmapUpdatedPhysical.WriteAllBytes(Path.Combine(dir, "L2_1_JournalBitmapUpdatedPhysical"), ctx.Logger);
save.JournalBitmapUpdatedVirtual.WriteAllBytes(Path.Combine(dir, "L2_2_JournalBitmapUpdatedVirtual"), ctx.Logger);
save.JournalBitmapUnassigned.WriteAllBytes(Path.Combine(dir, "L2_3_JournalBitmapUnassigned"), ctx.Logger);
save.JournalLayer1Hash.WriteAllBytes(Path.Combine(dir, "L2_4_Layer1Hash"), ctx.Logger);
save.JournalLayer2Hash.WriteAllBytes(Path.Combine(dir, "L2_5_Layer2Hash"), ctx.Logger);
save.JournalLayer3Hash.WriteAllBytes(Path.Combine(dir, "L2_6_Layer3Hash"), ctx.Logger);
save.JournalFat.WriteAllBytes(Path.Combine(dir, "L2_7_FAT"), ctx.Logger);
Stream journalTable = save.MetaRemapStorage.OpenStream(layout.JournalTableOffset, layout.JournalTableSize);
Stream journalBitmapUpdatedPhysical = save.MetaRemapStorage.OpenStream(layout.JournalBitmapUpdatedPhysicalOffset, layout.JournalBitmapUpdatedPhysicalSize);
Stream journalBitmapUpdatedVirtual = save.MetaRemapStorage.OpenStream(layout.JournalBitmapUpdatedVirtualOffset, layout.JournalBitmapUpdatedVirtualSize);
Stream journalBitmapUnassigned = save.MetaRemapStorage.OpenStream(layout.JournalBitmapUnassignedOffset, layout.JournalBitmapUnassignedSize);
Stream journalLayer1Hash = save.MetaRemapStorage.OpenStream(layout.IvfcL1Offset, layout.IvfcL1Size);
Stream journalLayer2Hash = save.MetaRemapStorage.OpenStream(layout.IvfcL2Offset, layout.IvfcL2Size);
Stream journalLayer3Hash = save.MetaRemapStorage.OpenStream(layout.IvfcL3Offset, layout.IvfcL3Size);
Stream journalFat = save.MetaRemapStorage.OpenStream(layout.FatOffset, layout.FatSize);
journalTable.WriteAllBytes(Path.Combine(dir, "L2_0_JournalTable"), ctx.Logger);
journalBitmapUpdatedPhysical.WriteAllBytes(Path.Combine(dir, "L2_1_JournalBitmapUpdatedPhysical"), ctx.Logger);
journalBitmapUpdatedVirtual.WriteAllBytes(Path.Combine(dir, "L2_2_JournalBitmapUpdatedVirtual"), ctx.Logger);
journalBitmapUnassigned.WriteAllBytes(Path.Combine(dir, "L2_3_JournalBitmapUnassigned"), ctx.Logger);
journalLayer1Hash.WriteAllBytes(Path.Combine(dir, "L2_4_Layer1Hash"), ctx.Logger);
journalLayer2Hash.WriteAllBytes(Path.Combine(dir, "L2_5_Layer2Hash"), ctx.Logger);
journalLayer3Hash.WriteAllBytes(Path.Combine(dir, "L2_6_Layer3Hash"), ctx.Logger);
journalFat.WriteAllBytes(Path.Combine(dir, "L2_7_FAT"), ctx.Logger);
save.IvfcStreamSource.CreateStream().WriteAllBytes(Path.Combine(dir, "L3_0_SaveData"), ctx.Logger);
}