From 079ffa6d3baa3e0655f399713e0962d74d172cd4 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Fri, 29 Mar 2019 16:25:22 -0500 Subject: [PATCH 01/14] Implement adding entries to HierarchicalSaveFileTable --- .../IO/Save/HierarchicalSaveFileTable.cs | 85 +++++++++- src/LibHac/IO/Save/SaveFsList.cs | 146 +++++++++++++++++- 2 files changed, 219 insertions(+), 12 deletions(-) diff --git a/src/LibHac/IO/Save/HierarchicalSaveFileTable.cs b/src/LibHac/IO/Save/HierarchicalSaveFileTable.cs index 3d3bb82a..e7d22f9f 100644 --- a/src/LibHac/IO/Save/HierarchicalSaveFileTable.cs +++ b/src/LibHac/IO/Save/HierarchicalSaveFileTable.cs @@ -16,7 +16,11 @@ namespace LibHac.IO.Save public bool TryOpenFile(string path, out SaveFileInfo fileInfo) { - FindPathRecursive(Util.GetUtf8Bytes(path), out SaveEntryKey key); + if (!FindPathRecursive(Util.GetUtf8Bytes(path), out SaveEntryKey key)) + { + fileInfo = default; + return false; + } if (FileTable.TryGetValue(ref key, out FileSaveEntry value)) { @@ -83,9 +87,77 @@ namespace LibHac.IO.Save return true; } + public void AddFile(string path, ref SaveFileInfo fileInfo) + { + path = PathTools.Normalize(path); + ReadOnlySpan pathBytes = Util.GetUtf8Bytes(path); + + if (path == "/") throw new ArgumentException("Path cannot be empty"); + + CreateFileRecursiveInternal(pathBytes, ref fileInfo); + } + + private void CreateFileRecursiveInternal(ReadOnlySpan path, ref SaveFileInfo fileInfo) + { + var parser = new PathParser(path); + var key = new SaveEntryKey(parser.GetCurrent(), 0); + + int prevIndex = 0; + + while (!parser.IsFinished()) + { + int index = DirectoryTable.GetIndexFromKey(ref key).Index; + + if (index < 0) + { + var newEntry = new DirectorySaveEntry(); + index = DirectoryTable.Add(ref key, ref newEntry); + + if (prevIndex > 0) + { + DirectoryTable.GetValue(prevIndex, out DirectorySaveEntry parentEntry); + + newEntry.NextSibling = parentEntry.Pos.NextDirectory; + parentEntry.Pos.NextDirectory = index; + + DirectoryTable.SetValue(prevIndex, ref parentEntry); + DirectoryTable.SetValue(index, ref newEntry); + } + } + + prevIndex = index; + key.Parent = index; + parser.TryGetNext(out key.Name); + } + + { + int index = FileTable.GetIndexFromKey(ref key).Index; + var fileEntry = new FileSaveEntry(); + + if (index < 0) + { + index = FileTable.Add(ref key, ref fileEntry); + + DirectoryTable.GetValue(prevIndex, out DirectorySaveEntry parentEntry); + + fileEntry.NextSibling = (int)parentEntry.Pos.NextFile; + parentEntry.Pos.NextFile = index; + + DirectoryTable.SetValue(prevIndex, ref parentEntry); + } + + fileEntry.Info = fileInfo; + FileTable.SetValue(index, ref fileEntry); + } + } + public bool TryOpenDirectory(string path, out SaveFindPosition position) { - FindPathRecursive(Util.GetUtf8Bytes(path), out SaveEntryKey key); + if (!FindPathRecursive(Util.GetUtf8Bytes(path), out SaveEntryKey key)) + { + position = default; + return false; + } if (DirectoryTable.TryGetValue(ref key, out DirectorySaveEntry value)) { @@ -97,16 +169,21 @@ namespace LibHac.IO.Save return false; } - private void FindPathRecursive(ReadOnlySpan path, out SaveEntryKey key) + private bool FindPathRecursive(ReadOnlySpan path, out SaveEntryKey key) { var parser = new PathParser(path); key = new SaveEntryKey(parser.GetCurrent(), 0); while (!parser.IsFinished()) { - key.Parent = DirectoryTable.GetOffsetFromKey(ref key); + key.Parent = DirectoryTable.GetIndexFromKey(ref key).Index; + + if (key.Parent < 0) return false; + parser.TryGetNext(out key.Name); } + + return true; } [StructLayout(LayoutKind.Sequential, Pack = 1)] diff --git a/src/LibHac/IO/Save/SaveFsList.cs b/src/LibHac/IO/Save/SaveFsList.cs index ad7af15a..21c57d4d 100644 --- a/src/LibHac/IO/Save/SaveFsList.cs +++ b/src/LibHac/IO/Save/SaveFsList.cs @@ -20,7 +20,7 @@ namespace LibHac.IO.Save Storage = tableStorage; } - public int GetOffsetFromKey(ref SaveEntryKey key) + public (int Index, int PreviousIndex) GetIndexFromKey(ref SaveEntryKey key) { Span entryBytes = stackalloc byte[_sizeOfEntry]; Span name = entryBytes.Slice(4, MaxNameLength); @@ -29,26 +29,92 @@ namespace LibHac.IO.Save int capacity = GetListCapacity(); ReadEntry(UsedListHeadIndex, entryBytes); + int prevIndex = UsedListHeadIndex; + int index = entry.Next; - while (entry.Next > 0) + while (index > 0) { - if (entry.Next > capacity) throw new IndexOutOfRangeException("Save entry index out of range"); + if (index > capacity) throw new IndexOutOfRangeException("Save entry index out of range"); - int entryId = entry.Next; - ReadEntry(entry.Next, out entry); + ReadEntry(index, out entry); if (entry.Parent == key.Parent && Util.StringSpansEqual(name, key.Name)) { - return entryId; + return (index, prevIndex); } + + prevIndex = index; + index = entry.Next; } - return -1; + return (-1, -1); + } + + public int Add(ref SaveEntryKey key, ref T value) + { + int index = GetIndexFromKey(ref key).Index; + + if (index != -1) + { + SetValue(index, ref value); + return index; + } + + index = AllocateEntry(); + + ReadEntry(index, out SaveFsEntry entry); + entry.Value = value; + WriteEntry(index, ref entry, ref key); + + return index; + } + + private int AllocateEntry() + { + ReadEntry(FreeListHeadIndex, out SaveFsEntry freeListHead); + ReadEntry(UsedListHeadIndex, out SaveFsEntry usedListHead); + + if (freeListHead.Next != 0) + { + ReadEntry(freeListHead.Next, out SaveFsEntry firstFreeEntry); + + int allocatedIndex = freeListHead.Next; + + freeListHead.Next = firstFreeEntry.Next; + firstFreeEntry.Next = usedListHead.Next; + usedListHead.Next = allocatedIndex; + + WriteEntry(FreeListHeadIndex, ref freeListHead); + WriteEntry(UsedListHeadIndex, ref usedListHead); + WriteEntry(allocatedIndex, ref firstFreeEntry); + + return allocatedIndex; + } + + int length = GetListLength(); + int capacity = GetListCapacity(); + + if (capacity == 0 || length >= capacity) + { + throw new NotImplementedException(); + } + + SetListLength(length + 1); + + ReadEntry(length, out SaveFsEntry newEntry); + + newEntry.Next = usedListHead.Next; + usedListHead.Next = length; + + WriteEntry(UsedListHeadIndex, ref usedListHead); + WriteEntry(length, ref newEntry); + + return length; } public bool TryGetValue(ref SaveEntryKey key, out T value) { - int index = GetOffsetFromKey(ref key); + int index = GetIndexFromKey(ref key).Index; if (index < 0) { @@ -123,6 +189,18 @@ namespace LibHac.IO.Save value = entry.Value; } + public void SetValue(int index, ref T value) + { + Span entryBytes = stackalloc byte[_sizeOfEntry]; + ref SaveFsEntry entry = ref GetEntryFromBytes(entryBytes); + + ReadEntry(index, out entry); + + entry.Value = value; + + WriteEntry(index, ref entry); + } + private int GetListCapacity() { Span buf = stackalloc byte[sizeof(int)]; @@ -139,6 +217,22 @@ namespace LibHac.IO.Save return MemoryMarshal.Read(buf); } + private void SetListCapacity(int capacity) + { + Span buf = stackalloc byte[sizeof(int)]; + MemoryMarshal.Write(buf, ref capacity); + + Storage.Write(buf, 4); + } + + private void SetListLength(int length) + { + Span buf = stackalloc byte[sizeof(int)]; + MemoryMarshal.Write(buf, ref length); + + Storage.Write(buf, 0); + } + private void ReadEntry(int index, out SaveFsEntry entry) { Span bytes = stackalloc byte[_sizeOfEntry]; @@ -147,6 +241,34 @@ namespace LibHac.IO.Save entry = GetEntryFromBytes(bytes); } + private void WriteEntry(int index, ref SaveFsEntry entry, ref SaveEntryKey key) + { + Span bytes = stackalloc byte[_sizeOfEntry]; + Span nameSpan = bytes.Slice(4, MaxNameLength); + + // Copy needed for .NET Framework compat + ref SaveFsEntry newEntry = ref GetEntryFromBytes(bytes); + newEntry = entry; + + newEntry.Parent = key.Parent; + key.Name.CopyTo(nameSpan); + + nameSpan.Slice(key.Name.Length).Fill(0); + + WriteEntry(index, bytes); + } + + private void WriteEntry(int index, ref SaveFsEntry entry) + { + Span bytes = stackalloc byte[_sizeOfEntry]; + + // Copy needed for .NET Framework compat + ref SaveFsEntry newEntry = ref GetEntryFromBytes(bytes); + newEntry = entry; + + WriteEntry(index, bytes); + } + private void ReadEntry(int index, Span entry) { Debug.Assert(entry.Length == _sizeOfEntry); @@ -155,6 +277,14 @@ namespace LibHac.IO.Save Storage.Read(entry, offset); } + private void WriteEntry(int index, Span entry) + { + Debug.Assert(entry.Length == _sizeOfEntry); + + int offset = index * _sizeOfEntry; + Storage.Write(entry, offset); + } + private ref SaveFsEntry GetEntryFromBytes(Span entry) { return ref MemoryMarshal.Cast(entry)[0]; From 90fc0e096c89bb3d56f1f5375fa387040f689d04 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Mon, 1 Apr 2019 21:15:08 -0500 Subject: [PATCH 02/14] Use generics for save file table entries --- .../IO/Save/HierarchicalSaveFileTable.cs | 55 ++++++++----------- src/LibHac/IO/Save/SaveFsEntries.cs | 5 +- src/LibHac/IO/Save/SaveFsList.cs | 2 +- 3 files changed, 27 insertions(+), 35 deletions(-) diff --git a/src/LibHac/IO/Save/HierarchicalSaveFileTable.cs b/src/LibHac/IO/Save/HierarchicalSaveFileTable.cs index e7d22f9f..fd7b9ed9 100644 --- a/src/LibHac/IO/Save/HierarchicalSaveFileTable.cs +++ b/src/LibHac/IO/Save/HierarchicalSaveFileTable.cs @@ -5,13 +5,13 @@ namespace LibHac.IO.Save { public class HierarchicalSaveFileTable { - private SaveFsList FileTable { get; } - private SaveFsList DirectoryTable { get; } + private SaveFsList> FileTable { get; } + private SaveFsList> DirectoryTable { get; } public HierarchicalSaveFileTable(IStorage dirTable, IStorage fileTable) { - FileTable = new SaveFsList(fileTable); - DirectoryTable = new SaveFsList(dirTable); + FileTable = new SaveFsList>(fileTable); + DirectoryTable = new SaveFsList>(dirTable); } public bool TryOpenFile(string path, out SaveFileInfo fileInfo) @@ -22,9 +22,9 @@ namespace LibHac.IO.Save return false; } - if (FileTable.TryGetValue(ref key, out FileSaveEntry value)) + if (FileTable.TryGetValue(ref key, out TableEntry value)) { - fileInfo = value.Info; + fileInfo = value.Value; return true; } @@ -43,7 +43,7 @@ namespace LibHac.IO.Save Span nameBytes = stackalloc byte[FileTable.MaxNameLength]; - bool success = FileTable.TryGetValue((int)position.NextFile, out FileSaveEntry entry, ref nameBytes); + bool success = FileTable.TryGetValue((int)position.NextFile, out TableEntry entry, ref nameBytes); // todo error message if (!success) @@ -54,7 +54,7 @@ namespace LibHac.IO.Save } position.NextFile = entry.NextSibling; - info = entry.Info; + info = entry.Value; name = Util.GetUtf8StringNullTerminated(nameBytes); @@ -69,9 +69,9 @@ namespace LibHac.IO.Save return false; } - Span nameBytes = stackalloc byte[FileTable.MaxNameLength]; + Span nameBytes = stackalloc byte[DirectoryTable.MaxNameLength]; - bool success = DirectoryTable.TryGetValue(position.NextDirectory, out DirectorySaveEntry entry, ref nameBytes); + bool success = DirectoryTable.TryGetValue(position.NextDirectory, out TableEntry entry, ref nameBytes); // todo error message if (!success) @@ -110,15 +110,15 @@ namespace LibHac.IO.Save if (index < 0) { - var newEntry = new DirectorySaveEntry(); + var newEntry = new TableEntry(); index = DirectoryTable.Add(ref key, ref newEntry); if (prevIndex > 0) { - DirectoryTable.GetValue(prevIndex, out DirectorySaveEntry parentEntry); + DirectoryTable.GetValue(prevIndex, out TableEntry parentEntry); - newEntry.NextSibling = parentEntry.Pos.NextDirectory; - parentEntry.Pos.NextDirectory = index; + newEntry.NextSibling = parentEntry.Value.NextDirectory; + parentEntry.Value.NextDirectory = index; DirectoryTable.SetValue(prevIndex, ref parentEntry); DirectoryTable.SetValue(index, ref newEntry); @@ -132,21 +132,21 @@ namespace LibHac.IO.Save { int index = FileTable.GetIndexFromKey(ref key).Index; - var fileEntry = new FileSaveEntry(); + var fileEntry = new TableEntry(); if (index < 0) { index = FileTable.Add(ref key, ref fileEntry); - DirectoryTable.GetValue(prevIndex, out DirectorySaveEntry parentEntry); + DirectoryTable.GetValue(prevIndex, out TableEntry parentEntry); - fileEntry.NextSibling = (int)parentEntry.Pos.NextFile; - parentEntry.Pos.NextFile = index; + fileEntry.NextSibling = (int)parentEntry.Value.NextFile; + parentEntry.Value.NextFile = index; DirectoryTable.SetValue(prevIndex, ref parentEntry); } - fileEntry.Info = fileInfo; + fileEntry.Value = fileInfo; FileTable.SetValue(index, ref fileEntry); } } @@ -159,9 +159,9 @@ namespace LibHac.IO.Save return false; } - if (DirectoryTable.TryGetValue(ref key, out DirectorySaveEntry value)) + if (DirectoryTable.TryGetValue(ref key, out TableEntry entry)) { - position = value.Pos; + position = entry.Value; return true; } @@ -187,19 +187,10 @@ namespace LibHac.IO.Save } [StructLayout(LayoutKind.Sequential, Pack = 1)] - private struct DirectorySaveEntry + private struct TableEntry where T : struct { public int NextSibling; - public SaveFindPosition Pos; - public long Field10; - } - - [StructLayout(LayoutKind.Sequential, Pack = 1)] - private struct FileSaveEntry - { - public int NextSibling; - public SaveFileInfo Info; - public long Field10; + public T Value; } } } diff --git a/src/LibHac/IO/Save/SaveFsEntries.cs b/src/LibHac/IO/Save/SaveFsEntries.cs index 3eb5b9d1..ee4dd8d5 100644 --- a/src/LibHac/IO/Save/SaveFsEntries.cs +++ b/src/LibHac/IO/Save/SaveFsEntries.cs @@ -15,17 +15,18 @@ namespace LibHac.IO.Save } } - [StructLayout(LayoutKind.Sequential, Pack = 1)] + [StructLayout(LayoutKind.Sequential, Pack = 1, Size = 0x14)] public struct SaveFileInfo { public int StartBlock; public long Length; + public long Reserved; } /// /// Represents the current position when enumerating a directory's contents. /// - [StructLayout(LayoutKind.Sequential, Pack = 1)] + [StructLayout(LayoutKind.Sequential, Pack = 1, Size = 0x14)] public struct SaveFindPosition { /// The ID of the next directory to be enumerated. diff --git a/src/LibHac/IO/Save/SaveFsList.cs b/src/LibHac/IO/Save/SaveFsList.cs index 21c57d4d..dcb49103 100644 --- a/src/LibHac/IO/Save/SaveFsList.cs +++ b/src/LibHac/IO/Save/SaveFsList.cs @@ -5,7 +5,7 @@ using System.Runtime.InteropServices; namespace LibHac.IO.Save { - internal class SaveFsList where T : unmanaged + internal class SaveFsList where T : struct { private const int FreeListHeadIndex = 0; private const int UsedListHeadIndex = 1; From 6112e35eb2357601af97729f3f29582ab6f8eb52 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Tue, 2 Apr 2019 17:21:38 -0500 Subject: [PATCH 03/14] Read directly from save allocation table when requested --- src/LibHac/IO/Save/AllocationTable.cs | 47 ++++++++++++------- src/LibHac/IO/Save/AllocationTableIterator.cs | 18 +++---- 2 files changed, 39 insertions(+), 26 deletions(-) diff --git a/src/LibHac/IO/Save/AllocationTable.cs b/src/LibHac/IO/Save/AllocationTable.cs index 654f7dc3..03abeb9c 100644 --- a/src/LibHac/IO/Save/AllocationTable.cs +++ b/src/LibHac/IO/Save/AllocationTable.cs @@ -1,13 +1,16 @@ -using System.IO; +using System; +using System.IO; +using System.Runtime.InteropServices; namespace LibHac.IO.Save { public class AllocationTable { + private const int EntrySize = 8; + private IStorage BaseStorage { get; } private IStorage HeaderStorage { get; } - public AllocationTableEntry[] Entries { get; } public AllocationTableHeader Header { get; } public AllocationTable(IStorage storage, IStorage header) @@ -15,33 +18,43 @@ namespace LibHac.IO.Save BaseStorage = storage; HeaderStorage = header; Header = new AllocationTableHeader(HeaderStorage); + } - Stream tableStream = storage.AsStream(); + public void ReadEntry(int index, out AllocationTableEntry entry) + { + Span bytes = stackalloc byte[EntrySize]; + int offset = index * EntrySize; - // The first entry in the table is reserved. Block 0 is at table index 1 - int blockCount = (int)(Header.AllocationTableBlockCount) + 1; + BaseStorage.Read(bytes, offset); - Entries = new AllocationTableEntry[blockCount]; - tableStream.Position = 0; - var reader = new BinaryReader(tableStream); + entry = GetEntryFromBytes(bytes); + } - for (int i = 0; i < blockCount; i++) - { - int parent = reader.ReadInt32(); - int child = reader.ReadInt32(); + public void WriteEntry(int index, ref AllocationTableEntry entry) + { + Span bytes = stackalloc byte[EntrySize]; + int offset = index * EntrySize; - Entries[i] = new AllocationTableEntry { Next = child, Prev = parent }; - } + ref AllocationTableEntry newEntry = ref GetEntryFromBytes(bytes); + newEntry = entry; + + BaseStorage.Write(bytes, offset); + } + + private ref AllocationTableEntry GetEntryFromBytes(Span entry) + { + return ref MemoryMarshal.Cast(entry)[0]; } public IStorage GetBaseStorage() => BaseStorage.AsReadOnly(); public IStorage GetHeaderStorage() => HeaderStorage.AsReadOnly(); } - public class AllocationTableEntry + [StructLayout(LayoutKind.Sequential)] + public struct AllocationTableEntry { - public int Prev { get; set; } - public int Next { get; set; } + public int Prev; + public int Next; public bool IsListStart() { diff --git a/src/LibHac/IO/Save/AllocationTableIterator.cs b/src/LibHac/IO/Save/AllocationTableIterator.cs index e64a6882..22c1cfea 100644 --- a/src/LibHac/IO/Save/AllocationTableIterator.cs +++ b/src/LibHac/IO/Save/AllocationTableIterator.cs @@ -22,11 +22,11 @@ namespace LibHac.IO.Save public bool BeginIteration(int initialBlock) { - AllocationTableEntry tableEntry = Fat.Entries[initialBlock + 1]; + Fat.ReadEntry(initialBlock + 1, out AllocationTableEntry tableEntry); if (!tableEntry.IsListStart() && initialBlock != -1) { - return false; + return false; } if (tableEntry.IsSingleBlockSegment()) @@ -35,7 +35,7 @@ namespace LibHac.IO.Save } else { - AllocationTableEntry lengthEntry = Fat.Entries[initialBlock + 2]; + Fat.ReadEntry(initialBlock + 2, out AllocationTableEntry lengthEntry); CurrentSegmentSize = lengthEntry.Next - initialBlock; } @@ -46,11 +46,11 @@ namespace LibHac.IO.Save public bool MoveNext() { - AllocationTableEntry currentEntry = Fat.Entries[PhysicalBlock + 1]; + Fat.ReadEntry(PhysicalBlock + 1, out AllocationTableEntry currentEntry); if (currentEntry.IsListEnd()) return false; int newBlock = currentEntry.Next & 0x7FFFFFFF; - AllocationTableEntry newEntry = Fat.Entries[newBlock]; + Fat.ReadEntry(newBlock, out AllocationTableEntry newEntry); VirtualBlock += CurrentSegmentSize; if (newEntry.IsSingleBlockSegment()) @@ -59,7 +59,7 @@ namespace LibHac.IO.Save } else { - AllocationTableEntry lengthEntry = Fat.Entries[newBlock + 1]; + Fat.ReadEntry(newBlock + 1, out AllocationTableEntry lengthEntry); CurrentSegmentSize = lengthEntry.Next - (newBlock - 1); } @@ -69,11 +69,11 @@ namespace LibHac.IO.Save public bool MovePrevious() { - AllocationTableEntry currentEntry = Fat.Entries[PhysicalBlock + 1]; + Fat.ReadEntry(PhysicalBlock + 1, out AllocationTableEntry currentEntry); if (currentEntry.IsListStart()) return false; int newBlock = currentEntry.Prev & 0x7FFFFFFF; - AllocationTableEntry newEntry = Fat.Entries[newBlock]; + Fat.ReadEntry(newBlock, out AllocationTableEntry newEntry); if (newEntry.IsSingleBlockSegment()) { @@ -81,7 +81,7 @@ namespace LibHac.IO.Save } else { - AllocationTableEntry lengthEntry = Fat.Entries[newBlock + 1]; + Fat.ReadEntry(newBlock + 1, out AllocationTableEntry lengthEntry); CurrentSegmentSize = lengthEntry.Next - (newBlock - 1); } From d7dd540b21c95f09bd37ee2265365c69ef0aa164 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Wed, 3 Apr 2019 16:55:14 -0500 Subject: [PATCH 04/14] Move some logic to AllocationTable --- src/LibHac/IO/Save/AllocationTable.cs | 68 ++++++++++++++++--- src/LibHac/IO/Save/AllocationTableIterator.cs | 67 +++++------------- 2 files changed, 77 insertions(+), 58 deletions(-) diff --git a/src/LibHac/IO/Save/AllocationTable.cs b/src/LibHac/IO/Save/AllocationTable.cs index 03abeb9c..904b724b 100644 --- a/src/LibHac/IO/Save/AllocationTable.cs +++ b/src/LibHac/IO/Save/AllocationTable.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using System.IO; using System.Runtime.InteropServices; @@ -13,6 +14,9 @@ namespace LibHac.IO.Save public AllocationTableHeader Header { get; } + public IStorage GetBaseStorage() => BaseStorage.AsReadOnly(); + public IStorage GetHeaderStorage() => HeaderStorage.AsReadOnly(); + public AllocationTable(IStorage storage, IStorage header) { BaseStorage = storage; @@ -20,20 +24,68 @@ namespace LibHac.IO.Save Header = new AllocationTableHeader(HeaderStorage); } - public void ReadEntry(int index, out AllocationTableEntry entry) + public void ReadEntry(int blockIndex, out int next, out int previous, out int length) + { + int entryIndex = BlockToEntryIndex(blockIndex); + + Span entries = stackalloc AllocationTableEntry[2]; + ReadEntries(entryIndex, entries); + + if (entries[0].IsSingleBlockSegment()) + { + length = 1; + } + else + { + length = entries[1].Next - entryIndex; + } + + if (entries[0].IsListEnd()) + { + next = -1; + } + else + { + next = EntryIndexToBlock(entries[0].Next & 0x7FFFFFFF); + } + + if (entries[0].IsListStart()) + { + previous = -1; + } + else + { + previous = EntryIndexToBlock(entries[0].Prev & 0x7FFFFFFF); + } + } + + private void ReadEntries(int entryIndex, Span entries) + { + Debug.Assert(entries.Length >= 2); + + bool isLastBlock = entryIndex == BlockToEntryIndex(Header.AllocationTableBlockCount) - 1; + int entriesToRead = isLastBlock ? 1 : 2; + int offset = entryIndex * EntrySize; + + Span buffer = MemoryMarshal.Cast(entries.Slice(0, entriesToRead)); + + BaseStorage.Read(buffer, offset); + } + + private void ReadEntry(int entryIndex, out AllocationTableEntry entry) { Span bytes = stackalloc byte[EntrySize]; - int offset = index * EntrySize; + int offset = entryIndex * EntrySize; BaseStorage.Read(bytes, offset); entry = GetEntryFromBytes(bytes); } - public void WriteEntry(int index, ref AllocationTableEntry entry) + private void WriteEntry(int entryIndex, ref AllocationTableEntry entry) { Span bytes = stackalloc byte[EntrySize]; - int offset = index * EntrySize; + int offset = entryIndex * EntrySize; ref AllocationTableEntry newEntry = ref GetEntryFromBytes(bytes); newEntry = entry; @@ -41,13 +93,13 @@ namespace LibHac.IO.Save BaseStorage.Write(bytes, offset); } - private ref AllocationTableEntry GetEntryFromBytes(Span entry) + private static ref AllocationTableEntry GetEntryFromBytes(Span entry) { return ref MemoryMarshal.Cast(entry)[0]; } - public IStorage GetBaseStorage() => BaseStorage.AsReadOnly(); - public IStorage GetHeaderStorage() => HeaderStorage.AsReadOnly(); + private static int EntryIndexToBlock(int entryIndex) => entryIndex - 1; + private static int BlockToEntryIndex(int blockIndex) => blockIndex + 1; } [StructLayout(LayoutKind.Sequential)] @@ -81,7 +133,7 @@ namespace LibHac.IO.Save { public long BlockSize { get; } public long AllocationTableOffset { get; } - public long AllocationTableBlockCount { get; } + public int AllocationTableBlockCount { get; } public long DataOffset { get; } public long DataBlockCount { get; } public int DirectoryTableBlock { get; } diff --git a/src/LibHac/IO/Save/AllocationTableIterator.cs b/src/LibHac/IO/Save/AllocationTableIterator.cs index 22c1cfea..a7c8511b 100644 --- a/src/LibHac/IO/Save/AllocationTableIterator.cs +++ b/src/LibHac/IO/Save/AllocationTableIterator.cs @@ -8,7 +8,11 @@ namespace LibHac.IO.Save public int VirtualBlock { get; private set; } public int PhysicalBlock { get; private set; } - public int CurrentSegmentSize { get; private set; } + public int CurrentSegmentSize => _currentSegmentSize; + + private int _nextBlock; + private int _prevBlock; + private int _currentSegmentSize; public AllocationTableIterator(AllocationTable table, int initialBlock) { @@ -22,71 +26,34 @@ namespace LibHac.IO.Save public bool BeginIteration(int initialBlock) { - Fat.ReadEntry(initialBlock + 1, out AllocationTableEntry tableEntry); - - if (!tableEntry.IsListStart() && initialBlock != -1) - { - return false; - } - - if (tableEntry.IsSingleBlockSegment()) - { - CurrentSegmentSize = 1; - } - else - { - Fat.ReadEntry(initialBlock + 2, out AllocationTableEntry lengthEntry); - CurrentSegmentSize = lengthEntry.Next - initialBlock; - } - PhysicalBlock = initialBlock; + Fat.ReadEntry(initialBlock, out _nextBlock, out _prevBlock, out _currentSegmentSize); - return true; + return _prevBlock == -1; } public bool MoveNext() { - Fat.ReadEntry(PhysicalBlock + 1, out AllocationTableEntry currentEntry); - if (currentEntry.IsListEnd()) return false; - int newBlock = currentEntry.Next & 0x7FFFFFFF; + if (_nextBlock == -1) return false; - Fat.ReadEntry(newBlock, out AllocationTableEntry newEntry); - VirtualBlock += CurrentSegmentSize; + VirtualBlock += _currentSegmentSize; + PhysicalBlock = _nextBlock; - if (newEntry.IsSingleBlockSegment()) - { - CurrentSegmentSize = 1; - } - else - { - Fat.ReadEntry(newBlock + 1, out AllocationTableEntry lengthEntry); - CurrentSegmentSize = lengthEntry.Next - (newBlock - 1); - } - - PhysicalBlock = newBlock - 1; + Fat.ReadEntry(_nextBlock, out _nextBlock, out _prevBlock, out _currentSegmentSize); + return true; } public bool MovePrevious() { - Fat.ReadEntry(PhysicalBlock + 1, out AllocationTableEntry currentEntry); - if (currentEntry.IsListStart()) return false; - int newBlock = currentEntry.Prev & 0x7FFFFFFF; + if (_prevBlock == -1) return false; - Fat.ReadEntry(newBlock, out AllocationTableEntry newEntry); + PhysicalBlock = _prevBlock; - if (newEntry.IsSingleBlockSegment()) - { - CurrentSegmentSize = 1; - } - else - { - Fat.ReadEntry(newBlock + 1, out AllocationTableEntry lengthEntry); - CurrentSegmentSize = lengthEntry.Next - (newBlock - 1); - } + Fat.ReadEntry(_prevBlock, out _nextBlock, out _prevBlock, out _currentSegmentSize); - VirtualBlock -= CurrentSegmentSize; - PhysicalBlock = newBlock - 1; + VirtualBlock -= _currentSegmentSize; + return true; } From c89b8be8874bcaa22352d71e2aae8e07ca4848f5 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Wed, 3 Apr 2019 18:04:04 -0500 Subject: [PATCH 05/14] Add joining allocation table lists --- src/LibHac/IO/Save/AllocationTable.cs | 127 +++++++++++++++++++++++++- 1 file changed, 124 insertions(+), 3 deletions(-) diff --git a/src/LibHac/IO/Save/AllocationTable.cs b/src/LibHac/IO/Save/AllocationTable.cs index 904b724b..c92825ee 100644 --- a/src/LibHac/IO/Save/AllocationTable.cs +++ b/src/LibHac/IO/Save/AllocationTable.cs @@ -59,6 +59,47 @@ namespace LibHac.IO.Save } } + public void Join(int frontListBlockIndex, int backListBlockIndex) + { + int frontEntryIndex = BlockToEntryIndex(frontListBlockIndex); + int backEntryIndex = BlockToEntryIndex(backListBlockIndex); + + int frontTailIndex = GetListTail(frontEntryIndex); + + AllocationTableEntry frontTail = ReadEntry(frontTailIndex); + AllocationTableEntry backHead = ReadEntry(backEntryIndex); + + frontTail.SetNext(backEntryIndex); + backHead.SetPrev(frontTailIndex); + + WriteEntry(frontTailIndex, frontTail); + WriteEntry(backEntryIndex, backHead); + } + + public int GetListLength(int blockIndex) + { + int index = blockIndex; + int totalLength = 0; + + int tableSize = Header.AllocationTableBlockCount; + int nodesIterated = 0; + + while (index != -1) + { + ReadEntry(index, out index, out int _, out int length); + + totalLength += length; + nodesIterated++; + + if (nodesIterated > tableSize) + { + throw new InvalidDataException("Cycle detected in allocation table."); + } + } + + return totalLength; + } + private void ReadEntries(int entryIndex, Span entries) { Debug.Assert(entries.Length >= 2); @@ -72,17 +113,17 @@ namespace LibHac.IO.Save BaseStorage.Read(buffer, offset); } - private void ReadEntry(int entryIndex, out AllocationTableEntry entry) + private AllocationTableEntry ReadEntry(int entryIndex) { Span bytes = stackalloc byte[EntrySize]; int offset = entryIndex * EntrySize; BaseStorage.Read(bytes, offset); - entry = GetEntryFromBytes(bytes); + return GetEntryFromBytes(bytes); } - private void WriteEntry(int entryIndex, ref AllocationTableEntry entry) + private void WriteEntry(int entryIndex, AllocationTableEntry entry) { Span bytes = stackalloc byte[EntrySize]; int offset = entryIndex * EntrySize; @@ -93,6 +134,52 @@ namespace LibHac.IO.Save BaseStorage.Write(bytes, offset); } + private int GetListHead(int entryIndex) + { + int headIndex = entryIndex; + int tableSize = Header.AllocationTableBlockCount; + int nodesTraversed = 0; + + AllocationTableEntry entry = ReadEntry(entryIndex); + + while (!entry.IsListStart()) + { + nodesTraversed++; + headIndex = entry.Prev & 0x7FFFFFFF; + entry = ReadEntry(headIndex); + + if (nodesTraversed > tableSize) + { + throw new InvalidDataException("Cycle detected in allocation table."); + } + } + + return headIndex; + } + + private int GetListTail(int entryIndex) + { + int tailIndex = entryIndex; + int tableSize = Header.AllocationTableBlockCount; + int nodesTraversed = 0; + + AllocationTableEntry entry = ReadEntry(entryIndex); + + while (!entry.IsListEnd()) + { + nodesTraversed++; + tailIndex = entry.Next & 0x7FFFFFFF; + entry = ReadEntry(tailIndex); + + if (nodesTraversed > tableSize) + { + throw new InvalidDataException("Cycle detected in allocation table."); + } + } + + return tailIndex; + } + private static ref AllocationTableEntry GetEntryFromBytes(Span entry) { return ref MemoryMarshal.Cast(entry)[0]; @@ -123,10 +210,44 @@ namespace LibHac.IO.Save return Next < 0; } + public void MakeMultiBlockSegment() + { + Next |= unchecked((int)0x80000000); + } + + public void MakeSingleBlockSegment() + { + Next &= 0x7FFFFFFF; + } + public bool IsSingleBlockSegment() { return Next >= 0; } + + public void MakeListStart() + { + Prev = int.MinValue; + } + + public void MakeRangeEntry() + { + Prev |= unchecked((int)0x80000000); + } + + public void SetNext(int value) + { + Debug.Assert(value >= 0); + + Next = Next & unchecked((int)0x80000000) | value; + } + + public void SetPrev(int value) + { + Debug.Assert(value >= 0); + + Prev = value; + } } public class AllocationTableHeader From b1997806c1cd34cabdd690f9faf34e8d6daf0c40 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Thu, 11 Apr 2019 18:08:17 -0400 Subject: [PATCH 06/14] Add savedata CreateFile --- src/LibHac/IO/Save/AllocationTable.cs | 191 ++++++++++++++++++- src/LibHac/IO/Save/SaveDataFileSystemCore.cs | 11 +- 2 files changed, 197 insertions(+), 5 deletions(-) diff --git a/src/LibHac/IO/Save/AllocationTable.cs b/src/LibHac/IO/Save/AllocationTable.cs index c92825ee..7b3d3b4d 100644 --- a/src/LibHac/IO/Save/AllocationTable.cs +++ b/src/LibHac/IO/Save/AllocationTable.cs @@ -7,6 +7,7 @@ namespace LibHac.IO.Save { public class AllocationTable { + private const int FreeListEntryIndex = 0; private const int EntrySize = 8; private IStorage BaseStorage { get; } @@ -37,7 +38,7 @@ namespace LibHac.IO.Save } else { - length = entries[1].Next - entryIndex; + length = entries[1].Next - entryIndex + 1; } if (entries[0].IsListEnd()) @@ -46,7 +47,7 @@ namespace LibHac.IO.Save } else { - next = EntryIndexToBlock(entries[0].Next & 0x7FFFFFFF); + next = EntryIndexToBlock(entries[0].GetNext()); } if (entries[0].IsListStart()) @@ -55,10 +56,44 @@ namespace LibHac.IO.Save } else { - previous = EntryIndexToBlock(entries[0].Prev & 0x7FFFFFFF); + previous = EntryIndexToBlock(entries[0].GetPrev()); } } + public int GetFreeListBlockIndex() + { + AllocationTableEntry freeList = ReadEntry(FreeListEntryIndex); + return EntryIndexToBlock(freeList.GetNext()); + } + + public void SetFreeListBlockIndex(int headBlockIndex) + { + var freeList = new AllocationTableEntry() { Next = BlockToEntryIndex(headBlockIndex) }; + WriteEntry(FreeListEntryIndex, freeList); + } + + public int Allocate(int blockCount) + { + if (blockCount <= 0) + { + throw new ArgumentOutOfRangeException(nameof(blockCount)); + } + + int freeList = GetFreeListBlockIndex(); + + int newFreeList = Trim(freeList, blockCount); + if (newFreeList == -1) return -1; + + SetFreeListBlockIndex(newFreeList); + + return freeList; + } + + /// + /// Combines 2 lists into one list. The second list will be attached to the end of the first list. + /// + /// The index of the first block. + /// The index of the second block. public void Join(int frontListBlockIndex, int backListBlockIndex) { int frontEntryIndex = BlockToEntryIndex(frontListBlockIndex); @@ -76,6 +111,136 @@ namespace LibHac.IO.Save WriteEntry(backEntryIndex, backHead); } + /// + /// Trims an existing list to the specified length and returns the excess blocks as a new list. + /// + /// The starting block of the list to trim. + /// The length in blocks that the list will be shortened to. + /// The index of the head node of the removed blocks. + public int Trim(int listHeadBlockIndex, int newListLength) + { + int blocksRemaining = newListLength; + int next = BlockToEntryIndex(listHeadBlockIndex); + int listAIndex = -1; + int listBIndex = -1; + + while (blocksRemaining > 0) + { + if (next < 0) + { + return -1; + } + + int currentEntryIndex = next; + + ReadEntry(EntryIndexToBlock(currentEntryIndex), out next, out int _, out int segmentLength); + + int start = EntryIndexToBlock(currentEntryIndex); + int end = start + segmentLength - 1; + + next = BlockToEntryIndex(next); + + if (segmentLength == blocksRemaining) + { + listAIndex = currentEntryIndex; + listBIndex = next; + } + else if (segmentLength > blocksRemaining) + { + Split(EntryIndexToBlock(currentEntryIndex), blocksRemaining); + + listAIndex = currentEntryIndex; + listBIndex = currentEntryIndex + blocksRemaining; + } + + blocksRemaining -= segmentLength; + } + + if (listAIndex == -1 || listBIndex == -1) return -1; + + AllocationTableEntry listANode = ReadEntry(listAIndex); + AllocationTableEntry listBNode = ReadEntry(listBIndex); + + listANode.SetNext(0); + listBNode.MakeListStart(); + + WriteEntry(listAIndex, listANode); + WriteEntry(listBIndex, listBNode); + + return EntryIndexToBlock(listBIndex); + } + + /// + /// Splits a single list segment into 2 segments. The sequence of blocks in the full list will remain the same. + /// + /// The block index of the segment to split. + /// The length of the first subsegment. + public void Split(int segmentBlockIndex, int firstSubSegmentLength) + { + Debug.Assert(firstSubSegmentLength > 0); + + int segAIndex = BlockToEntryIndex(segmentBlockIndex); + + AllocationTableEntry segA = ReadEntry(segAIndex); + if (!segA.IsMultiBlockSegment()) throw new ArgumentException("Cannot split a single-entry segment."); + + AllocationTableEntry segARange = ReadEntry(segAIndex + 1); + int originalLength = segARange.GetNext() - segARange.GetPrev() + 1; + + if (firstSubSegmentLength >= originalLength) + { + throw new ArgumentOutOfRangeException(nameof(firstSubSegmentLength), + $"Requested sub-segment length ({firstSubSegmentLength}) must be less than the full segment length ({originalLength})"); + } + + int segBIndex = segAIndex + firstSubSegmentLength; + + int segALength = firstSubSegmentLength; + int segBLength = originalLength - segALength; + + var segB = new AllocationTableEntry(); + + // Insert segment B between segments A and C + segB.SetPrev(segAIndex); + segB.SetNext(segA.GetNext()); + segA.SetNext(segBIndex); + + if (!segB.IsListEnd()) + { + AllocationTableEntry segC = ReadEntry(segB.GetNext()); + segC.SetPrev(segBIndex); + WriteEntry(segB.GetNext(), segC); + } + + // Write the new range entries if needed + if (segBLength > 1) + { + segB.MakeMultiBlockSegment(); + + var segBRange = new AllocationTableEntry(); + segBRange.SetRange(segBIndex, segBIndex + segBLength - 1); + + WriteEntry(segBIndex + 1, segBRange); + WriteEntry(segBIndex + segBLength - 1, segBRange); + } + + WriteEntry(segBIndex, segB); + + if (segALength == 1) + { + segA.MakeSingleBlockSegment(); + } + else + { + segARange.SetRange(segAIndex, segAIndex + segALength - 1); + + WriteEntry(segAIndex + 1, segARange); + WriteEntry(segAIndex + segALength - 1, segARange); + } + + WriteEntry(segAIndex, segA); + } + public int GetListLength(int blockIndex) { int index = blockIndex; @@ -195,6 +360,16 @@ namespace LibHac.IO.Save public int Prev; public int Next; + public int GetPrev() + { + return Prev & 0x7FFFFFFF; + } + + public int GetNext() + { + return Next & 0x7FFFFFFF; + } + public bool IsListStart() { return Prev == int.MinValue; @@ -248,6 +423,16 @@ namespace LibHac.IO.Save Prev = value; } + + public void SetRange(int startIndex, int endIndex) + { + Debug.Assert(startIndex > 0); + Debug.Assert(endIndex > 0); + + Next = endIndex; + Prev = startIndex; + MakeRangeEntry(); + } } public class AllocationTableHeader diff --git a/src/LibHac/IO/Save/SaveDataFileSystemCore.cs b/src/LibHac/IO/Save/SaveDataFileSystemCore.cs index 4a714b8f..be2e857b 100644 --- a/src/LibHac/IO/Save/SaveDataFileSystemCore.cs +++ b/src/LibHac/IO/Save/SaveDataFileSystemCore.cs @@ -19,7 +19,7 @@ namespace LibHac.IO.Save AllocationTable = new AllocationTable(allocationTable, header.Slice(0x18, 0x30)); Header = new SaveHeader(HeaderStorage); - + // todo: Query the FAT for the file size when none is given AllocationTableStorage dirTableStorage = OpenFatBlock(AllocationTable.Header.DirectoryTableBlock, 1000000); AllocationTableStorage fileTableStorage = OpenFatBlock(AllocationTable.Header.FileTableBlock, 1000000); @@ -34,7 +34,14 @@ namespace LibHac.IO.Save public void CreateFile(string path, long size, CreateFileOptions options) { - throw new System.NotImplementedException(); + path = PathTools.Normalize(path); + + int blockCount = (int)Util.DivideByRoundUp(size, AllocationTable.Header.BlockSize); + int startBlock = AllocationTable.Allocate(blockCount); + + var fileEntry = new SaveFileInfo { StartBlock = startBlock, Length = size }; + + FileTable.AddFile(path, ref fileEntry); } public void DeleteDirectory(string path) From 9c0e6030e5fa405d87f539f8f3bf878259295d71 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Fri, 12 Apr 2019 22:52:33 -0400 Subject: [PATCH 07/14] Add DeleteFile to savedata --- src/LibHac/IO/Save/AllocationTable.cs | 44 +++++++++++++-- .../IO/Save/HierarchicalSaveFileTable.cs | 53 ++++++++++++++++++- src/LibHac/IO/Save/SaveDataFileSystemCore.cs | 14 ++++- src/LibHac/IO/Save/SaveFsEntries.cs | 2 +- src/LibHac/IO/Save/SaveFsList.cs | 25 +++++++++ 5 files changed, 129 insertions(+), 9 deletions(-) diff --git a/src/LibHac/IO/Save/AllocationTable.cs b/src/LibHac/IO/Save/AllocationTable.cs index 7b3d3b4d..e560b6c7 100644 --- a/src/LibHac/IO/Save/AllocationTable.cs +++ b/src/LibHac/IO/Save/AllocationTable.cs @@ -62,13 +62,23 @@ namespace LibHac.IO.Save public int GetFreeListBlockIndex() { - AllocationTableEntry freeList = ReadEntry(FreeListEntryIndex); - return EntryIndexToBlock(freeList.GetNext()); + return EntryIndexToBlock(GetFreeListEntryIndex()); } public void SetFreeListBlockIndex(int headBlockIndex) { - var freeList = new AllocationTableEntry() { Next = BlockToEntryIndex(headBlockIndex) }; + SetFreeListEntryIndex(BlockToEntryIndex(headBlockIndex)); + } + + public int GetFreeListEntryIndex() + { + AllocationTableEntry freeList = ReadEntry(FreeListEntryIndex); + return freeList.GetNext(); + } + + public void SetFreeListEntryIndex(int headBlockIndex) + { + var freeList = new AllocationTableEntry { Next = headBlockIndex }; WriteEntry(FreeListEntryIndex, freeList); } @@ -89,11 +99,35 @@ namespace LibHac.IO.Save return freeList; } + public void Free(int listBlockIndex) + { + int listEntryIndex = BlockToEntryIndex(listBlockIndex); + AllocationTableEntry listEntry = ReadEntry(listEntryIndex); + + if (!listEntry.IsListStart()) + { + throw new ArgumentOutOfRangeException(nameof(listBlockIndex), "The block to free must be the start of a list."); + } + + int freeListIndex = GetFreeListEntryIndex(); + + // Free list is empty + if (freeListIndex == 0) + { + SetFreeListEntryIndex(listEntryIndex); + return; + } + + Join(listBlockIndex, EntryIndexToBlock(freeListIndex)); + + SetFreeListBlockIndex(listBlockIndex); + } + /// /// Combines 2 lists into one list. The second list will be attached to the end of the first list. /// - /// The index of the first block. - /// The index of the second block. + /// The index of the start block of the first list. + /// The index of the start block of the second list. public void Join(int frontListBlockIndex, int backListBlockIndex) { int frontEntryIndex = BlockToEntryIndex(frontListBlockIndex); diff --git a/src/LibHac/IO/Save/HierarchicalSaveFileTable.cs b/src/LibHac/IO/Save/HierarchicalSaveFileTable.cs index fd7b9ed9..4c45b465 100644 --- a/src/LibHac/IO/Save/HierarchicalSaveFileTable.cs +++ b/src/LibHac/IO/Save/HierarchicalSaveFileTable.cs @@ -1,4 +1,5 @@ using System; +using System.IO; using System.Runtime.InteropServices; namespace LibHac.IO.Save @@ -43,7 +44,7 @@ namespace LibHac.IO.Save Span nameBytes = stackalloc byte[FileTable.MaxNameLength]; - bool success = FileTable.TryGetValue((int)position.NextFile, out TableEntry entry, ref nameBytes); + bool success = FileTable.TryGetValue(position.NextFile, out TableEntry entry, ref nameBytes); // todo error message if (!success) @@ -140,7 +141,7 @@ namespace LibHac.IO.Save DirectoryTable.GetValue(prevIndex, out TableEntry parentEntry); - fileEntry.NextSibling = (int)parentEntry.Value.NextFile; + fileEntry.NextSibling = parentEntry.Value.NextFile; parentEntry.Value.NextFile = index; DirectoryTable.SetValue(prevIndex, ref parentEntry); @@ -151,6 +152,54 @@ namespace LibHac.IO.Save } } + public void DeleteFile(string path) + { + path = PathTools.Normalize(path); + ReadOnlySpan pathBytes = Util.GetUtf8Bytes(path); + + FindPathRecursive(pathBytes, out SaveEntryKey key); + int parentIndex = key.Parent; + + DirectoryTable.GetValue(parentIndex, out TableEntry parentEntry); + + int toDeleteIndex = FileTable.GetIndexFromKey(ref key).Index; + if (toDeleteIndex < 0) throw new FileNotFoundException(); + + FileTable.GetValue(toDeleteIndex, out TableEntry toDeleteEntry); + + if (parentEntry.Value.NextFile == toDeleteIndex) + { + parentEntry.Value.NextFile = toDeleteEntry.NextSibling; + DirectoryTable.SetValue(parentIndex, ref parentEntry); + FileTable.Remove(ref key); + return; + } + + int prevIndex = parentEntry.Value.NextFile; + FileTable.GetValue(prevIndex, out TableEntry prevEntry); + int curIndex = prevEntry.NextSibling; + + while (curIndex != 0) + { + FileTable.GetValue(curIndex, out TableEntry curEntry); + + if (curIndex == toDeleteIndex) + { + prevEntry.NextSibling = curEntry.NextSibling; + FileTable.SetValue(prevIndex, ref prevEntry); + + FileTable.Remove(ref key); + return; + } + + prevIndex = curIndex; + prevEntry = curEntry; + curIndex = prevEntry.NextSibling; + } + + throw new FileNotFoundException(); + } + public bool TryOpenDirectory(string path, out SaveFindPosition position) { if (!FindPathRecursive(Util.GetUtf8Bytes(path), out SaveEntryKey key)) diff --git a/src/LibHac/IO/Save/SaveDataFileSystemCore.cs b/src/LibHac/IO/Save/SaveDataFileSystemCore.cs index be2e857b..12966a8f 100644 --- a/src/LibHac/IO/Save/SaveDataFileSystemCore.cs +++ b/src/LibHac/IO/Save/SaveDataFileSystemCore.cs @@ -51,7 +51,19 @@ namespace LibHac.IO.Save public void DeleteFile(string path) { - throw new System.NotImplementedException(); + path = PathTools.Normalize(path); + + if (!FileTable.TryOpenFile(path, out SaveFileInfo fileInfo)) + { + throw new FileNotFoundException(); + } + + if (fileInfo.StartBlock != int.MinValue) + { + AllocationTable.Free(fileInfo.StartBlock); + } + + FileTable.DeleteFile(path); } public IDirectory OpenDirectory(string path, OpenDirectoryMode mode) diff --git a/src/LibHac/IO/Save/SaveFsEntries.cs b/src/LibHac/IO/Save/SaveFsEntries.cs index ee4dd8d5..625e5ba5 100644 --- a/src/LibHac/IO/Save/SaveFsEntries.cs +++ b/src/LibHac/IO/Save/SaveFsEntries.cs @@ -32,6 +32,6 @@ namespace LibHac.IO.Save /// The ID of the next directory to be enumerated. public int NextDirectory; /// The ID of the next file to be enumerated. - public long NextFile; + public int NextFile; } } diff --git a/src/LibHac/IO/Save/SaveFsList.cs b/src/LibHac/IO/Save/SaveFsList.cs index dcb49103..8142c49c 100644 --- a/src/LibHac/IO/Save/SaveFsList.cs +++ b/src/LibHac/IO/Save/SaveFsList.cs @@ -112,6 +112,18 @@ namespace LibHac.IO.Save return length; } + private void Free(int entryIndex) + { + ReadEntry(FreeListHeadIndex, out SaveFsEntry freeEntry); + ReadEntry(entryIndex, out SaveFsEntry entry); + + entry.Next = freeEntry.Next; + freeEntry.Next = entryIndex; + + WriteEntry(FreeListHeadIndex, ref freeEntry); + WriteEntry(entryIndex, ref entry); + } + public bool TryGetValue(ref SaveEntryKey key, out T value) { int index = GetIndexFromKey(ref key).Index; @@ -201,6 +213,19 @@ namespace LibHac.IO.Save WriteEntry(index, ref entry); } + public void Remove(ref SaveEntryKey key) + { + (int index, int previousIndex) = GetIndexFromKey(ref key); + + ReadEntry(previousIndex, out SaveFsEntry prevEntry); + ReadEntry(index, out SaveFsEntry entryToDel); + + prevEntry.Next = entryToDel.Next; + WriteEntry(previousIndex, ref prevEntry); + + Free(index); + } + private int GetListCapacity() { Span buf = stackalloc byte[sizeof(int)]; From 5c84f5c2a41a1bee42a80efdb1d98cfa61b2773c Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Mon, 15 Apr 2019 14:38:18 -0400 Subject: [PATCH 08/14] Add FsTrim for savedata --- src/LibHac/IO/Save/AllocationTable.cs | 36 +++++++++++++ src/LibHac/IO/Save/AllocationTableStorage.cs | 5 +- src/LibHac/IO/Save/SaveDataFileSystem.cs | 5 ++ src/LibHac/IO/Save/SaveDataFileSystemCore.cs | 33 +++++++++--- src/LibHac/IO/StorageExtensions.cs | 53 ++++++++++++++++++++ src/hactoolnet/CliParser.cs | 2 + src/hactoolnet/Options.cs | 1 + src/hactoolnet/ProcessSave.cs | 8 ++- 8 files changed, 134 insertions(+), 9 deletions(-) diff --git a/src/LibHac/IO/Save/AllocationTable.cs b/src/LibHac/IO/Save/AllocationTable.cs index e560b6c7..a8bc04f2 100644 --- a/src/LibHac/IO/Save/AllocationTable.cs +++ b/src/LibHac/IO/Save/AllocationTable.cs @@ -299,6 +299,42 @@ namespace LibHac.IO.Save return totalLength; } + public void FsTrimList(int blockIndex) + { + int index = blockIndex; + + int tableSize = Header.AllocationTableBlockCount; + int nodesIterated = 0; + + while (index != -1) + { + ReadEntry(index, out int next, out int _, out int length); + + if (length > 3) + { + int fillOffset = BlockToEntryIndex(index + 2) * EntrySize; + int fillLength = (length - 3) * EntrySize; + + BaseStorage.Slice(fillOffset, fillLength).Fill(0x00); + } + + nodesIterated++; + + if (nodesIterated > tableSize) + { + return; + } + + index = next; + } + } + + public void FsTrim() + { + int tableSize = BlockToEntryIndex(Header.AllocationTableBlockCount) * EntrySize; + BaseStorage.Slice(tableSize).Fill(0x00); + } + private void ReadEntries(int entryIndex, Span entries) { Debug.Assert(entries.Length >= 2); diff --git a/src/LibHac/IO/Save/AllocationTableStorage.cs b/src/LibHac/IO/Save/AllocationTableStorage.cs index 492749ad..44b4673c 100644 --- a/src/LibHac/IO/Save/AllocationTableStorage.cs +++ b/src/LibHac/IO/Save/AllocationTableStorage.cs @@ -11,13 +11,14 @@ namespace LibHac.IO.Save private long _length; - public AllocationTableStorage(IStorage data, AllocationTable table, int blockSize, int initialBlock, long length) + public AllocationTableStorage(IStorage data, AllocationTable table, int blockSize, int initialBlock) { BaseStorage = data; BlockSize = blockSize; - _length = length; Fat = table; InitialBlock = initialBlock; + + _length = table.GetListLength(initialBlock) * blockSize; } protected override void ReadImpl(Span destination, long offset) diff --git a/src/LibHac/IO/Save/SaveDataFileSystem.cs b/src/LibHac/IO/Save/SaveDataFileSystem.cs index 86312cb2..aa8c2ff1 100644 --- a/src/LibHac/IO/Save/SaveDataFileSystem.cs +++ b/src/LibHac/IO/Save/SaveDataFileSystem.cs @@ -200,6 +200,11 @@ namespace LibHac.IO.Save return true; } + public void FsTrim() + { + SaveDataFileSystemCore.FsTrim(); + } + public Validity Verify(IProgressReport logger = null) { Validity validity = IvfcStorage.Validate(true, logger); diff --git a/src/LibHac/IO/Save/SaveDataFileSystemCore.cs b/src/LibHac/IO/Save/SaveDataFileSystemCore.cs index 12966a8f..d1bd0f27 100644 --- a/src/LibHac/IO/Save/SaveDataFileSystemCore.cs +++ b/src/LibHac/IO/Save/SaveDataFileSystemCore.cs @@ -20,9 +20,8 @@ namespace LibHac.IO.Save Header = new SaveHeader(HeaderStorage); - // todo: Query the FAT for the file size when none is given - AllocationTableStorage dirTableStorage = OpenFatBlock(AllocationTable.Header.DirectoryTableBlock, 1000000); - AllocationTableStorage fileTableStorage = OpenFatBlock(AllocationTable.Header.FileTableBlock, 1000000); + AllocationTableStorage dirTableStorage = OpenFatStorage(AllocationTable.Header.DirectoryTableBlock); + AllocationTableStorage fileTableStorage = OpenFatStorage(AllocationTable.Header.FileTableBlock); FileTable = new HierarchicalSaveFileTable(dirTableStorage, fileTableStorage); } @@ -92,7 +91,7 @@ namespace LibHac.IO.Save return new NullFile(); } - AllocationTableStorage storage = OpenFatBlock(file.StartBlock, file.Length); + AllocationTableStorage storage = OpenFatStorage(file.StartBlock); return new SaveDataFile(storage, 0, file.Length, mode); } @@ -139,9 +138,31 @@ namespace LibHac.IO.Save public IStorage GetBaseStorage() => BaseStorage.AsReadOnly(); public IStorage GetHeaderStorage() => HeaderStorage.AsReadOnly(); - private AllocationTableStorage OpenFatBlock(int blockIndex, long size) + public void FsTrim() { - return new AllocationTableStorage(BaseStorage, AllocationTable, (int)Header.BlockSize, blockIndex, size); + AllocationTable.FsTrim(); + + foreach (DirectoryEntry file in this.EnumerateEntries("*", SearchOptions.RecurseSubdirectories)) + { + if (FileTable.TryOpenFile(file.FullPath, out SaveFileInfo fileInfo) && fileInfo.StartBlock >= 0) + { + AllocationTable.FsTrimList(fileInfo.StartBlock); + + OpenFatStorage(fileInfo.StartBlock).Slice(fileInfo.Length).Fill(0); + } + } + + int freeIndex = AllocationTable.GetFreeListBlockIndex(); + if (freeIndex == 0) return; + + AllocationTable.FsTrimList(freeIndex); + + OpenFatStorage(freeIndex).Fill(0); + } + + private AllocationTableStorage OpenFatStorage(int blockIndex) + { + return new AllocationTableStorage(BaseStorage, AllocationTable, (int)Header.BlockSize, blockIndex); } } diff --git a/src/LibHac/IO/StorageExtensions.cs b/src/LibHac/IO/StorageExtensions.cs index e71c1f4f..9be88414 100644 --- a/src/LibHac/IO/StorageExtensions.cs +++ b/src/LibHac/IO/StorageExtensions.cs @@ -97,6 +97,59 @@ namespace LibHac.IO progress?.SetTotal(0); } + public static void Fill(this IStorage input, byte value, IProgressReport progress = null) + { + const int threshold = 0x400; + + long length = input.GetSize(); + if (length > threshold) + { + input.FillLarge(value, progress); + return; + } + + Span buf = stackalloc byte[(int)length]; + buf.Fill(value); + + input.Write(buf, 0); + } + + private static void FillLarge(this IStorage input, byte value, IProgressReport progress = null) + { + const int bufferSize = 0x4000; + + long remaining = input.GetSize(); + if (remaining < 0) throw new ArgumentException("Storage must have an explicit length"); + progress?.SetTotal(remaining); + + long pos = 0; + + byte[] buffer = ArrayPool.Shared.Rent(bufferSize); + try + { + buffer.AsSpan(0, (int)Math.Min(remaining, bufferSize)).Fill(value); + + while (remaining > 0) + { + int toFill = (int)Math.Min(bufferSize, remaining); + Span buf = buffer.AsSpan(0, toFill); + + input.Write(buf, pos); + + remaining -= toFill; + pos += toFill; + + progress?.ReportAdd(toFill); + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } + + progress?.SetTotal(0); + } + public static void WriteAllBytes(this IStorage input, string filename, IProgressReport progress = null) { using (var outFile = new FileStream(filename, FileMode.Create, FileAccess.Write)) diff --git a/src/hactoolnet/CliParser.cs b/src/hactoolnet/CliParser.cs index df3c1749..52c090cb 100644 --- a/src/hactoolnet/CliParser.cs +++ b/src/hactoolnet/CliParser.cs @@ -52,6 +52,7 @@ namespace hactoolnet new CliOption("listromfs", 0, (o, a) => o.ListRomFs = true), new CliOption("listfiles", 0, (o, a) => o.ListFiles = true), new CliOption("sign", 0, (o, a) => o.SignSave = true), + new CliOption("trim", 0, (o, a) => o.TrimSave = true), new CliOption("readbench", 0, (o, a) => o.ReadBench = true), new CliOption("hashedfs", 0, (o, a) => o.BuildHfs = true), new CliOption("title", 1, (o, a) => o.TitleId = ParseTitleId(a[0])), @@ -232,6 +233,7 @@ namespace hactoolnet sb.AppendLine(" --outdir Specify directory path to save contents to."); sb.AppendLine(" --debugoutdir Specify directory path to save intermediate data to for debugging."); sb.AppendLine(" --sign Sign the save file. (Requires device_key in key file)"); + sb.AppendLine(" --trim Trim garbage data in the save file. (Requires device_key in key file)"); sb.AppendLine(" --listfiles List files in save file."); sb.AppendLine(" --replacefile Replaces a file in the save data"); sb.AppendLine("NDV0 (Delta) options:"); diff --git a/src/hactoolnet/Options.cs b/src/hactoolnet/Options.cs index f4fc1038..c99db2c4 100644 --- a/src/hactoolnet/Options.cs +++ b/src/hactoolnet/Options.cs @@ -45,6 +45,7 @@ namespace hactoolnet public bool ListRomFs; public bool ListFiles; public bool SignSave; + public bool TrimSave; public bool ReadBench; public bool BuildHfs; public ulong TitleId; diff --git a/src/hactoolnet/ProcessSave.cs b/src/hactoolnet/ProcessSave.cs index d8b397a6..f7626a5a 100644 --- a/src/hactoolnet/ProcessSave.cs +++ b/src/hactoolnet/ProcessSave.cs @@ -129,8 +129,14 @@ namespace hactoolnet return; } - if (ctx.Options.SignSave) + if (ctx.Options.SignSave || ctx.Options.TrimSave) { + if (ctx.Options.TrimSave) + { + save.FsTrim(); + ctx.Logger.LogMessage("Trimmed save file"); + } + if (save.Commit(ctx.Keyset)) { ctx.Logger.LogMessage("Successfully signed save file"); From 8914d89b32dc966eb5fc49d9309f690285348ba8 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Sat, 20 Apr 2019 14:15:00 -0400 Subject: [PATCH 09/14] Add SaveDataFile resizing --- src/LibHac/IO/Save/AllocationTableStorage.cs | 43 +++++++++++++++++++- src/LibHac/IO/Save/SaveDataFile.cs | 31 ++++++++++---- src/LibHac/IO/Save/SaveDataFileSystemCore.cs | 7 +--- 3 files changed, 66 insertions(+), 15 deletions(-) diff --git a/src/LibHac/IO/Save/AllocationTableStorage.cs b/src/LibHac/IO/Save/AllocationTableStorage.cs index 44b4673c..9a061088 100644 --- a/src/LibHac/IO/Save/AllocationTableStorage.cs +++ b/src/LibHac/IO/Save/AllocationTableStorage.cs @@ -6,7 +6,7 @@ namespace LibHac.IO.Save { private IStorage BaseStorage { get; } private int BlockSize { get; } - private int InitialBlock { get; } + internal int InitialBlock { get; private set; } private AllocationTable Fat { get; } private long _length; @@ -18,7 +18,7 @@ namespace LibHac.IO.Save Fat = table; InitialBlock = initialBlock; - _length = table.GetListLength(initialBlock) * blockSize; + _length = initialBlock == -1 ? 0 : table.GetListLength(initialBlock) * blockSize; } protected override void ReadImpl(Span destination, long offset) @@ -81,5 +81,44 @@ namespace LibHac.IO.Save } public override long GetSize() => _length; + + public override void SetSize(long size) + { + int oldBlockCount = (int)Util.DivideByRoundUp(_length, BlockSize); + int newBlockCount = (int)Util.DivideByRoundUp(size, BlockSize); + + if (oldBlockCount == newBlockCount) return; + + if (oldBlockCount == 0) + { + InitialBlock = Fat.Allocate(newBlockCount); + _length = newBlockCount * BlockSize; + + return; + } + + if (newBlockCount == 0) + { + Fat.Free(InitialBlock); + + InitialBlock = -1; + _length = 0; + + return; + } + + if (newBlockCount > oldBlockCount) + { + int newBlocks = Fat.Allocate(newBlockCount - oldBlockCount); + Fat.Join(InitialBlock, newBlocks); + } + else + { + int oldBlocks = Fat.Trim(InitialBlock, newBlockCount); + Fat.Free(oldBlocks); + } + + _length = newBlockCount * BlockSize; + } } } diff --git a/src/LibHac/IO/Save/SaveDataFile.cs b/src/LibHac/IO/Save/SaveDataFile.cs index bf852858..24ce1e1a 100644 --- a/src/LibHac/IO/Save/SaveDataFile.cs +++ b/src/LibHac/IO/Save/SaveDataFile.cs @@ -1,18 +1,21 @@ using System; +using System.IO; namespace LibHac.IO.Save { public class SaveDataFile : FileBase { private AllocationTableStorage BaseStorage { get; } - private long Offset { get; } - private long Size { get; } + private string Path { get; } + private HierarchicalSaveFileTable FileTable { get; } + private long Size { get; set; } - public SaveDataFile(AllocationTableStorage baseStorage, long offset, long size, OpenMode mode) + public SaveDataFile(AllocationTableStorage baseStorage, string path, HierarchicalSaveFileTable fileTable, long size, OpenMode mode) { Mode = mode; BaseStorage = baseStorage; - Offset = offset; + Path = path; + FileTable = fileTable; Size = size; } @@ -20,8 +23,7 @@ namespace LibHac.IO.Save { int toRead = ValidateReadParamsAndGetSize(destination, offset); - long storageOffset = Offset + offset; - BaseStorage.Read(destination.Slice(0, toRead), storageOffset); + BaseStorage.Read(destination.Slice(0, toRead), offset); return toRead; } @@ -45,7 +47,22 @@ namespace LibHac.IO.Save public override void SetSize(long size) { - throw new NotImplementedException(); + if (size < 0) throw new ArgumentOutOfRangeException(nameof(size)); + if (Size == size) return; + + BaseStorage.SetSize(size); + + if (!FileTable.TryOpenFile(Path, out SaveFileInfo fileInfo)) + { + throw new FileNotFoundException(); + } + + fileInfo.StartBlock = BaseStorage.InitialBlock; + fileInfo.Length = size; + + FileTable.AddFile(Path, ref fileInfo); + + Size = size; } } } diff --git a/src/LibHac/IO/Save/SaveDataFileSystemCore.cs b/src/LibHac/IO/Save/SaveDataFileSystemCore.cs index d1bd0f27..4345a06a 100644 --- a/src/LibHac/IO/Save/SaveDataFileSystemCore.cs +++ b/src/LibHac/IO/Save/SaveDataFileSystemCore.cs @@ -86,14 +86,9 @@ namespace LibHac.IO.Save throw new FileNotFoundException(); } - if (file.StartBlock < 0) - { - return new NullFile(); - } - AllocationTableStorage storage = OpenFatStorage(file.StartBlock); - return new SaveDataFile(storage, 0, file.Length, mode); + return new SaveDataFile(storage, path, FileTable, file.Length, mode); } public void RenameDirectory(string srcPath, string dstPath) From 5bb46b2b36fde4af9a379ecf8b9988fbe7fc5b48 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Sat, 20 Apr 2019 14:21:09 -0400 Subject: [PATCH 10/14] Forward Savedata FS functions to core FS --- src/LibHac/IO/Save/SaveDataFileSystem.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/LibHac/IO/Save/SaveDataFileSystem.cs b/src/LibHac/IO/Save/SaveDataFileSystem.cs index aa8c2ff1..2ec9616a 100644 --- a/src/LibHac/IO/Save/SaveDataFileSystem.cs +++ b/src/LibHac/IO/Save/SaveDataFileSystem.cs @@ -119,22 +119,22 @@ namespace LibHac.IO.Save public void CreateDirectory(string path) { - throw new System.NotImplementedException(); + SaveDataFileSystemCore.CreateDirectory(path); } public void CreateFile(string path, long size, CreateFileOptions options) { - throw new System.NotImplementedException(); + SaveDataFileSystemCore.CreateFile(path, size, options); } public void DeleteDirectory(string path) { - throw new System.NotImplementedException(); + SaveDataFileSystemCore.DeleteDirectory(path); } public void DeleteFile(string path) { - throw new System.NotImplementedException(); + SaveDataFileSystemCore.DeleteFile(path); } public IDirectory OpenDirectory(string path, OpenDirectoryMode mode) @@ -149,12 +149,12 @@ namespace LibHac.IO.Save public void RenameDirectory(string srcPath, string dstPath) { - throw new System.NotImplementedException(); + SaveDataFileSystemCore.RenameDirectory(srcPath, dstPath); } public void RenameFile(string srcPath, string dstPath) { - throw new System.NotImplementedException(); + SaveDataFileSystemCore.RenameFile(srcPath, dstPath); } public bool DirectoryExists(string path) => SaveDataFileSystemCore.DirectoryExists(path); From 300a4335eacdda1e5d03c14e59597fc29c1748c3 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Mon, 22 Apr 2019 21:32:57 -0500 Subject: [PATCH 11/14] Flush IVFC storage on savedata commit --- src/LibHac/IO/Save/SaveDataFileSystem.cs | 28 +++++++++++++++----- src/LibHac/IO/Save/SaveDataFileSystemCore.cs | 2 +- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/LibHac/IO/Save/SaveDataFileSystem.cs b/src/LibHac/IO/Save/SaveDataFileSystem.cs index 2ec9616a..a10762d3 100644 --- a/src/LibHac/IO/Save/SaveDataFileSystem.cs +++ b/src/LibHac/IO/Save/SaveDataFileSystem.cs @@ -9,7 +9,6 @@ namespace LibHac.IO.Save public IStorage BaseStorage { get; } public bool LeaveOpen { get; } - public HierarchicalIntegrityVerificationStorage IvfcStorage { get; } public SaveDataFileSystemCore SaveDataFileSystemCore { get; } public RemapStorage DataRemapStorage { get; } @@ -18,6 +17,9 @@ namespace LibHac.IO.Save public HierarchicalDuplexStorage DuplexStorage { get; } public JournalStorage JournalStorage { get; } + public HierarchicalIntegrityVerificationStorage JournalIvfcStorage { get; } + public HierarchicalIntegrityVerificationStorage FatIvfcStorage { get; } + private Keyset Keyset { get; } public SaveDataFileSystem(Keyset keyset, IStorage storage, IntegrityCheckLevel integrityCheckLevel, bool leaveOpen) @@ -52,16 +54,17 @@ namespace LibHac.IO.Save JournalStorage = new JournalStorage(journalData, Header.JournalHeader, journalMapInfo, leaveOpen); - IvfcStorage = InitJournalIvfcStorage(integrityCheckLevel); + JournalIvfcStorage = InitJournalIvfcStorage(integrityCheckLevel); IStorage fatStorage = MetaRemapStorage.Slice(layout.FatOffset, layout.FatSize); if (Header.Layout.Version >= 0x50000) { - fatStorage = InitFatIvfcStorage(integrityCheckLevel); + FatIvfcStorage = InitFatIvfcStorage(integrityCheckLevel); + fatStorage = FatIvfcStorage; } - SaveDataFileSystemCore = new SaveDataFileSystemCore(IvfcStorage, fatStorage, Header.SaveHeader); + SaveDataFileSystemCore = new SaveDataFileSystemCore(JournalIvfcStorage, fatStorage, Header.SaveHeader); } private static HierarchicalDuplexStorage InitDuplexStorage(IStorage baseStorage, Header header) @@ -172,6 +175,9 @@ namespace LibHac.IO.Save public bool Commit(Keyset keyset) { + JournalIvfcStorage.Flush(); + FatIvfcStorage.Flush(); + Stream headerStream = BaseStorage.AsStream(); var hashData = new byte[0x3d00]; @@ -207,10 +213,18 @@ namespace LibHac.IO.Save public Validity Verify(IProgressReport logger = null) { - Validity validity = IvfcStorage.Validate(true, logger); - IvfcStorage.SetLevelValidities(Header.Ivfc); + Validity journalValidity = JournalIvfcStorage.Validate(true, logger); + JournalIvfcStorage.SetLevelValidities(Header.Ivfc); - return validity; + if (FatIvfcStorage == null)return journalValidity; + + Validity fatValidity = FatIvfcStorage.Validate(true, logger); + FatIvfcStorage.SetLevelValidities(Header.Ivfc); + + if (journalValidity != Validity.Valid) return journalValidity; + if (fatValidity != Validity.Valid) return fatValidity; + + return journalValidity; } } } diff --git a/src/LibHac/IO/Save/SaveDataFileSystemCore.cs b/src/LibHac/IO/Save/SaveDataFileSystemCore.cs index 4345a06a..d902e08f 100644 --- a/src/LibHac/IO/Save/SaveDataFileSystemCore.cs +++ b/src/LibHac/IO/Save/SaveDataFileSystemCore.cs @@ -127,7 +127,7 @@ namespace LibHac.IO.Save public void Commit() { - throw new System.NotImplementedException(); + } public IStorage GetBaseStorage() => BaseStorage.AsReadOnly(); From fdefd5cf36e5760861c29fedcd30858c363bae90 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Mon, 22 Apr 2019 21:34:32 -0500 Subject: [PATCH 12/14] Create a separate method for savedata debug export --- src/hactoolnet/ProcessSave.cs | 157 +++++++++++++++++++--------------- 1 file changed, 90 insertions(+), 67 deletions(-) diff --git a/src/hactoolnet/ProcessSave.cs b/src/hactoolnet/ProcessSave.cs index f7626a5a..09c6bb4f 100644 --- a/src/hactoolnet/ProcessSave.cs +++ b/src/hactoolnet/ProcessSave.cs @@ -32,68 +32,8 @@ namespace hactoolnet { // todo string dir = ctx.Options.DebugOutDir; - Directory.CreateDirectory(dir); - FsLayout layout = save.Header.Layout; - - string mainRemapDir = Path.Combine(dir, "main_remap"); - Directory.CreateDirectory(mainRemapDir); - - save.DataRemapStorage.GetBaseStorage().WriteAllBytes(Path.Combine(mainRemapDir, "Data")); - save.DataRemapStorage.GetHeaderStorage().WriteAllBytes(Path.Combine(mainRemapDir, "Header")); - save.DataRemapStorage.GetMapEntryStorage().WriteAllBytes(Path.Combine(mainRemapDir, "Map entries")); - - string metadataRemapDir = Path.Combine(dir, "metadata_remap"); - Directory.CreateDirectory(metadataRemapDir); - - save.MetaRemapStorage.GetBaseStorage().WriteAllBytes(Path.Combine(metadataRemapDir, "Data")); - save.MetaRemapStorage.GetHeaderStorage().WriteAllBytes(Path.Combine(metadataRemapDir, "Header")); - save.MetaRemapStorage.GetMapEntryStorage().WriteAllBytes(Path.Combine(metadataRemapDir, "Map entries")); - - string journalDir = Path.Combine(dir, "journal"); - Directory.CreateDirectory(journalDir); - - save.JournalStorage.GetBaseStorage().WriteAllBytes(Path.Combine(journalDir, "Data")); - save.JournalStorage.GetHeaderStorage().WriteAllBytes(Path.Combine(journalDir, "Header")); - save.JournalStorage.Map.GetHeaderStorage().WriteAllBytes(Path.Combine(journalDir, "Map_header")); - save.JournalStorage.Map.GetMapStorage().WriteAllBytes(Path.Combine(journalDir, "Map")); - save.JournalStorage.Map.GetModifiedPhysicalBlocksStorage().WriteAllBytes(Path.Combine(journalDir, "ModifiedPhysicalBlocks")); - save.JournalStorage.Map.GetModifiedVirtualBlocksStorage().WriteAllBytes(Path.Combine(journalDir, "ModifiedVirtualBlocks")); - save.JournalStorage.Map.GetFreeBlocksStorage().WriteAllBytes(Path.Combine(journalDir, "FreeBlocks")); - - string saveDir = Path.Combine(dir, "save"); - Directory.CreateDirectory(saveDir); - - save.SaveDataFileSystemCore.GetHeaderStorage().WriteAllBytes(Path.Combine(saveDir, "Save_Header")); - save.SaveDataFileSystemCore.GetBaseStorage().WriteAllBytes(Path.Combine(saveDir, "Save_Data")); - save.SaveDataFileSystemCore.AllocationTable.GetHeaderStorage().WriteAllBytes(Path.Combine(saveDir, "FAT_header")); - save.SaveDataFileSystemCore.AllocationTable.GetBaseStorage().WriteAllBytes(Path.Combine(saveDir, "FAT_Data")); - - save.Header.DataIvfcMaster.WriteAllBytes(Path.Combine(saveDir, "Save_MasterHash")); - - IStorage saveLayer1Hash = save.MetaRemapStorage.Slice(layout.IvfcL1Offset, layout.IvfcL1Size); - IStorage saveLayer2Hash = save.MetaRemapStorage.Slice(layout.IvfcL2Offset, layout.IvfcL2Size); - IStorage saveLayer3Hash = save.MetaRemapStorage.Slice(layout.IvfcL3Offset, layout.IvfcL3Size); - - saveLayer1Hash.WriteAllBytes(Path.Combine(saveDir, "Save_Layer1Hash"), ctx.Logger); - saveLayer2Hash.WriteAllBytes(Path.Combine(saveDir, "Save_Layer2Hash"), ctx.Logger); - saveLayer3Hash.WriteAllBytes(Path.Combine(saveDir, "Save_Layer3Hash"), ctx.Logger); - - string duplexDir = Path.Combine(dir, "duplex"); - Directory.CreateDirectory(duplexDir); - - save.Header.DuplexMasterBitmapA.WriteAllBytes(Path.Combine(duplexDir, "MasterBitmapA")); - save.Header.DuplexMasterBitmapB.WriteAllBytes(Path.Combine(duplexDir, "MasterBitmapB")); - - IStorage duplexL1A = save.DataRemapStorage.Slice(layout.DuplexL1OffsetA, layout.DuplexL1Size); - IStorage duplexL1B = save.DataRemapStorage.Slice(layout.DuplexL1OffsetB, layout.DuplexL1Size); - IStorage duplexDataA = save.DataRemapStorage.Slice(layout.DuplexDataOffsetA, layout.DuplexDataSize); - IStorage duplexDataB = save.DataRemapStorage.Slice(layout.DuplexDataOffsetB, layout.DuplexDataSize); - - duplexL1A.WriteAllBytes(Path.Combine(duplexDir, "L1BitmapA"), ctx.Logger); - duplexL1B.WriteAllBytes(Path.Combine(duplexDir, "L1BitmapB"), ctx.Logger); - duplexDataA.WriteAllBytes(Path.Combine(duplexDir, "DataA"), ctx.Logger); - duplexDataB.WriteAllBytes(Path.Combine(duplexDir, "DataB"), ctx.Logger); + ExportSaveDebug(ctx, dir, save); } if (ctx.Options.ReplaceFileDest != null && ctx.Options.ReplaceFileSource != null) @@ -162,30 +102,113 @@ namespace hactoolnet } } + internal static void ExportSaveDebug(Context ctx, string dir, SaveDataFileSystem save) + { + Directory.CreateDirectory(dir); + + FsLayout layout = save.Header.Layout; + + string mainRemapDir = Path.Combine(dir, "main_remap"); + Directory.CreateDirectory(mainRemapDir); + + save.DataRemapStorage.GetBaseStorage().WriteAllBytes(Path.Combine(mainRemapDir, "Data")); + save.DataRemapStorage.GetHeaderStorage().WriteAllBytes(Path.Combine(mainRemapDir, "Header")); + save.DataRemapStorage.GetMapEntryStorage().WriteAllBytes(Path.Combine(mainRemapDir, "Map entries")); + + string metadataRemapDir = Path.Combine(dir, "metadata_remap"); + Directory.CreateDirectory(metadataRemapDir); + + save.MetaRemapStorage.GetBaseStorage().WriteAllBytes(Path.Combine(metadataRemapDir, "Data")); + save.MetaRemapStorage.GetHeaderStorage().WriteAllBytes(Path.Combine(metadataRemapDir, "Header")); + save.MetaRemapStorage.GetMapEntryStorage().WriteAllBytes(Path.Combine(metadataRemapDir, "Map entries")); + + string journalDir = Path.Combine(dir, "journal"); + Directory.CreateDirectory(journalDir); + + save.JournalStorage.GetBaseStorage().WriteAllBytes(Path.Combine(journalDir, "Data")); + save.JournalStorage.GetHeaderStorage().WriteAllBytes(Path.Combine(journalDir, "Header")); + save.JournalStorage.Map.GetHeaderStorage().WriteAllBytes(Path.Combine(journalDir, "Map_header")); + save.JournalStorage.Map.GetMapStorage().WriteAllBytes(Path.Combine(journalDir, "Map")); + save.JournalStorage.Map.GetModifiedPhysicalBlocksStorage() + .WriteAllBytes(Path.Combine(journalDir, "ModifiedPhysicalBlocks")); + save.JournalStorage.Map.GetModifiedVirtualBlocksStorage() + .WriteAllBytes(Path.Combine(journalDir, "ModifiedVirtualBlocks")); + save.JournalStorage.Map.GetFreeBlocksStorage().WriteAllBytes(Path.Combine(journalDir, "FreeBlocks")); + + string saveDir = Path.Combine(dir, "save"); + Directory.CreateDirectory(saveDir); + + save.SaveDataFileSystemCore.GetHeaderStorage().WriteAllBytes(Path.Combine(saveDir, "Save_Header")); + save.SaveDataFileSystemCore.GetBaseStorage().WriteAllBytes(Path.Combine(saveDir, "Save_Data")); + save.SaveDataFileSystemCore.AllocationTable.GetHeaderStorage().WriteAllBytes(Path.Combine(saveDir, "FAT_header")); + save.SaveDataFileSystemCore.AllocationTable.GetBaseStorage().WriteAllBytes(Path.Combine(saveDir, "FAT_Data")); + + save.Header.DataIvfcMaster.WriteAllBytes(Path.Combine(saveDir, "Save_MasterHash")); + + IStorage saveLayer1Hash = save.MetaRemapStorage.Slice(layout.IvfcL1Offset, layout.IvfcL1Size); + IStorage saveLayer2Hash = save.MetaRemapStorage.Slice(layout.IvfcL2Offset, layout.IvfcL2Size); + IStorage saveLayer3Hash = save.MetaRemapStorage.Slice(layout.IvfcL3Offset, layout.IvfcL3Size); + + saveLayer1Hash.WriteAllBytes(Path.Combine(saveDir, "Save_Layer1Hash"), ctx.Logger); + saveLayer2Hash.WriteAllBytes(Path.Combine(saveDir, "Save_Layer2Hash"), ctx.Logger); + saveLayer3Hash.WriteAllBytes(Path.Combine(saveDir, "Save_Layer3Hash"), ctx.Logger); + + if (layout.Version >= 0x50000) + { + save.Header.FatIvfcMaster.WriteAllBytes(Path.Combine(saveDir, "Fat_MasterHash")); + + IStorage fatLayer1Hash = save.MetaRemapStorage.Slice(layout.FatIvfcL1Offset, layout.FatIvfcL1Size); + IStorage fatLayer2Hash = save.MetaRemapStorage.Slice(layout.FatIvfcL2Offset, layout.FatIvfcL1Size); + + fatLayer1Hash.WriteAllBytes(Path.Combine(saveDir, "Fat_Layer1Hash"), ctx.Logger); + fatLayer2Hash.WriteAllBytes(Path.Combine(saveDir, "Fat_Layer2Hash"), ctx.Logger); + } + + string duplexDir = Path.Combine(dir, "duplex"); + Directory.CreateDirectory(duplexDir); + + save.Header.DuplexMasterBitmapA.WriteAllBytes(Path.Combine(duplexDir, "MasterBitmapA")); + save.Header.DuplexMasterBitmapB.WriteAllBytes(Path.Combine(duplexDir, "MasterBitmapB")); + + IStorage duplexL1A = save.DataRemapStorage.Slice(layout.DuplexL1OffsetA, layout.DuplexL1Size); + IStorage duplexL1B = save.DataRemapStorage.Slice(layout.DuplexL1OffsetB, layout.DuplexL1Size); + IStorage duplexDataA = save.DataRemapStorage.Slice(layout.DuplexDataOffsetA, layout.DuplexDataSize); + IStorage duplexDataB = save.DataRemapStorage.Slice(layout.DuplexDataOffsetB, layout.DuplexDataSize); + + duplexL1A.WriteAllBytes(Path.Combine(duplexDir, "L1BitmapA"), ctx.Logger); + duplexL1B.WriteAllBytes(Path.Combine(duplexDir, "L1BitmapB"), ctx.Logger); + duplexDataA.WriteAllBytes(Path.Combine(duplexDir, "DataA"), ctx.Logger); + duplexDataB.WriteAllBytes(Path.Combine(duplexDir, "DataB"), ctx.Logger); + } + // ReSharper disable once UnusedMember.Local - private static string PrintFatLayout(this SaveDataFileSystem save) + public static string PrintFatLayout(this SaveDataFileSystemCore save) { var sb = new StringBuilder(); foreach (DirectoryEntry entry in save.EnumerateEntries().Where(x => x.Type == DirectoryEntryType.File)) { - save.SaveDataFileSystemCore.FileTable.TryOpenFile(entry.FullPath, out SaveFileInfo fileInfo); + save.FileTable.TryOpenFile(entry.FullPath, out SaveFileInfo fileInfo); if (fileInfo.StartBlock < 0) continue; - IEnumerable<(int block, int length)> chain = save.SaveDataFileSystemCore.AllocationTable.DumpChain(fileInfo.StartBlock); + IEnumerable<(int block, int length)> chain = save.AllocationTable.DumpChain(fileInfo.StartBlock); sb.AppendLine(entry.FullPath); sb.AppendLine(PrintBlockChain(chain)); + sb.AppendLine(); } sb.AppendLine("Directory Table"); - sb.AppendLine(PrintBlockChain(save.SaveDataFileSystemCore.AllocationTable.DumpChain(0))); + sb.AppendLine(PrintBlockChain(save.AllocationTable.DumpChain(0))); + sb.AppendLine(); sb.AppendLine("File Table"); - sb.AppendLine(PrintBlockChain(save.SaveDataFileSystemCore.AllocationTable.DumpChain(1))); + sb.AppendLine(PrintBlockChain(save.AllocationTable.DumpChain(1))); + sb.AppendLine(); sb.AppendLine("Free blocks"); - sb.AppendLine(PrintBlockChain(save.SaveDataFileSystemCore.AllocationTable.DumpChain(-1))); + sb.AppendLine(PrintBlockChain(save.AllocationTable.DumpChain(-1))); + sb.AppendLine(); return sb.ToString(); } From c383b946320c33bed6577cb183a2876c5432efc6 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Mon, 22 Apr 2019 21:37:01 -0500 Subject: [PATCH 13/14] Allow resizing when replacing file in savedata --- src/hactoolnet/ProcessSave.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hactoolnet/ProcessSave.cs b/src/hactoolnet/ProcessSave.cs index 09c6bb4f..a72d8632 100644 --- a/src/hactoolnet/ProcessSave.cs +++ b/src/hactoolnet/ProcessSave.cs @@ -47,8 +47,7 @@ namespace hactoolnet { if (inFile.GetSize() != outFile.GetSize()) { - ctx.Logger.LogMessage($"Replacement file must be the same size as the original file. ({outFile.GetSize()} bytes)"); - return; + outFile.SetSize(inFile.GetSize()); } inFile.CopyTo(outFile, ctx.Logger); From 0441d6e825de612bd6446187119411a28d1aa2a9 Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Tue, 23 Apr 2019 15:06:48 -0500 Subject: [PATCH 14/14] Small code cleanup --- src/LibHac/IO/Save/AllocationTable.cs | 4 +--- src/LibHac/IO/Save/SaveFsList.cs | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/LibHac/IO/Save/AllocationTable.cs b/src/LibHac/IO/Save/AllocationTable.cs index a8bc04f2..045ae2f4 100644 --- a/src/LibHac/IO/Save/AllocationTable.cs +++ b/src/LibHac/IO/Save/AllocationTable.cs @@ -169,9 +169,6 @@ namespace LibHac.IO.Save ReadEntry(EntryIndexToBlock(currentEntryIndex), out next, out int _, out int segmentLength); - int start = EntryIndexToBlock(currentEntryIndex); - int end = start + segmentLength - 1; - next = BlockToEntryIndex(next); if (segmentLength == blocksRemaining) @@ -369,6 +366,7 @@ namespace LibHac.IO.Save BaseStorage.Write(bytes, offset); } + // ReSharper disable once UnusedMember.Local private int GetListHead(int entryIndex) { int headIndex = entryIndex; diff --git a/src/LibHac/IO/Save/SaveFsList.cs b/src/LibHac/IO/Save/SaveFsList.cs index 8142c49c..726c5ac2 100644 --- a/src/LibHac/IO/Save/SaveFsList.cs +++ b/src/LibHac/IO/Save/SaveFsList.cs @@ -242,6 +242,7 @@ namespace LibHac.IO.Save return MemoryMarshal.Read(buf); } + // ReSharper disable once UnusedMember.Local private void SetListCapacity(int capacity) { Span buf = stackalloc byte[sizeof(int)];