From a551360da0d7ea6d5e7915f627004e0bddfbc21d Mon Sep 17 00:00:00 2001 From: Alex Barney Date: Mon, 3 Aug 2020 18:34:04 -0700 Subject: [PATCH] Use FlatMapKeyValueStore in SaveDataIndexer FS 10.0 tweaks how the SaveDataIndexerManager is accessed. SaveDataFileSystemServiceImpl's configuration now includes an ISaveDataIndexerManager instead of using global state. FS 10.0 uses the new nn::fssrv::storage namespace for abstracting the interface to nn::gc and nn::sdmmc. A tiny bit of the namespace has been added so we can create mock SD card handles for SaveDataIndexerManager. ReferenceCountedDisposable is now being used for returning ISaveDataInfoReader objects from IFileSystemProxy. I've tried coming up with a pattern to cleanly return a ReferenceCountedDisposable, but the lack of features like RAII or move semantics makes this difficult without either lots of allocations or lots of boilerplate code. --- src/LibHac/Diag/Abort.cs | 6 + src/LibHac/Fs/Impl/SdHandleManager.cs | 20 + src/LibHac/Fs/SaveDataStructs.cs | 13 +- src/LibHac/Fs/Shim/SaveDataManagement.cs | 26 +- src/LibHac/Fs/Shim/SystemSaveData.cs | 2 +- src/LibHac/FsService/FileSystemProxy.cs | 243 +++--- src/LibHac/FsService/FileSystemProxyCore.cs | 15 + src/LibHac/FsService/FileSystemServer.cs | 6 +- src/LibHac/FsService/IDeviceHandleManager.cs | 10 + src/LibHac/FsService/IFileSystemProxy.cs | 8 +- src/LibHac/FsService/ISaveDataIndexer.cs | 19 +- .../FsService/ISaveDataIndexerManager.cs | 11 + src/LibHac/FsService/ISaveDataInfoReader.cs | 2 +- src/LibHac/FsService/SaveDataIndexer.cs | 690 +++++++++++------- src/LibHac/FsService/SaveDataIndexerLite.cs | 53 +- .../FsService/SaveDataIndexerManager.cs | 173 +++-- .../FsService/SaveDataInfoFilterReader.cs | 9 +- .../FsService/Storage/SdCardManagement.cs | 20 + .../FsService/Storage/StorageDeviceHandle.cs | 26 + .../FsService/Storage/StorageDevicePortId.cs | 10 + src/LibHac/Kvdb/FlatMapKeyValueStore.cs | 9 +- src/LibHac/ReferenceCountedDisposable.cs | 103 +++ 22 files changed, 986 insertions(+), 488 deletions(-) create mode 100644 src/LibHac/Fs/Impl/SdHandleManager.cs create mode 100644 src/LibHac/FsService/IDeviceHandleManager.cs create mode 100644 src/LibHac/FsService/ISaveDataIndexerManager.cs create mode 100644 src/LibHac/FsService/Storage/SdCardManagement.cs create mode 100644 src/LibHac/FsService/Storage/StorageDeviceHandle.cs create mode 100644 src/LibHac/FsService/Storage/StorageDevicePortId.cs diff --git a/src/LibHac/Diag/Abort.cs b/src/LibHac/Diag/Abort.cs index fae72dc7..1055894c 100644 --- a/src/LibHac/Diag/Abort.cs +++ b/src/LibHac/Diag/Abort.cs @@ -1,4 +1,5 @@ using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; namespace LibHac.Diag { @@ -22,5 +23,10 @@ namespace LibHac.Diag DoAbort(message); } + + public static void UnexpectedDefault([CallerMemberName] string caller = "") + { + throw new LibHacException($"Unexpected value passed to switch statement in {caller}"); + } } } diff --git a/src/LibHac/Fs/Impl/SdHandleManager.cs b/src/LibHac/Fs/Impl/SdHandleManager.cs new file mode 100644 index 00000000..87c5fd84 --- /dev/null +++ b/src/LibHac/Fs/Impl/SdHandleManager.cs @@ -0,0 +1,20 @@ +using LibHac.FsService; +using LibHac.FsService.Storage; + +namespace LibHac.Fs.Impl +{ + internal class SdHandleManager : IDeviceHandleManager + { + public Result GetHandle(out StorageDeviceHandle handle) + { + return SdCardManagement.GetCurrentSdCardHandle(out handle); + } + + public bool IsValid(in StorageDeviceHandle handle) + { + // Note: Nintendo ignores the result here. + SdCardManagement.IsSdCardHandleValid(out bool isValid, in handle).IgnoreResult(); + return isValid; + } + } +} diff --git a/src/LibHac/Fs/SaveDataStructs.cs b/src/LibHac/Fs/SaveDataStructs.cs index 16229e75..8ac01c43 100644 --- a/src/LibHac/Fs/SaveDataStructs.cs +++ b/src/LibHac/Fs/SaveDataStructs.cs @@ -10,7 +10,7 @@ namespace LibHac.Fs { [FieldOffset(0x00)] public ProgramId ProgramId; [FieldOffset(0x08)] public UserId UserId; - [FieldOffset(0x18)] public ulong SaveDataId; + [FieldOffset(0x18)] public ulong StaticSaveDataId; [FieldOffset(0x20)] public SaveDataType Type; [FieldOffset(0x21)] public SaveDataRank Rank; [FieldOffset(0x22)] public short Index; @@ -25,15 +25,18 @@ namespace LibHac.Fs return ProgramId == other.ProgramId && Type == other.Type && UserId.Equals(other.UserId) && - SaveDataId == other.SaveDataId && + StaticSaveDataId == other.StaticSaveDataId && Rank == other.Rank && Index == other.Index; } + public static bool operator ==(SaveDataAttribute left, SaveDataAttribute right) => left.Equals(right); + public static bool operator !=(SaveDataAttribute left, SaveDataAttribute right) => !(left == right); + public override int GetHashCode() { // ReSharper disable NonReadonlyMemberInGetHashCode - return HashCode.Combine(ProgramId, Type, UserId, SaveDataId, Rank, Index); + return HashCode.Combine(ProgramId, Type, UserId, StaticSaveDataId, Rank, Index); // ReSharper restore NonReadonlyMemberInGetHashCode } @@ -45,7 +48,7 @@ namespace LibHac.Fs if (typeComparison != 0) return typeComparison; int userIdComparison = UserId.CompareTo(other.UserId); if (userIdComparison != 0) return userIdComparison; - int saveDataIdComparison = SaveDataId.CompareTo(other.SaveDataId); + int saveDataIdComparison = StaticSaveDataId.CompareTo(other.StaticSaveDataId); if (saveDataIdComparison != 0) return saveDataIdComparison; int rankComparison = Rank.CompareTo(other.Rank); if (rankComparison != 0) return rankComparison; @@ -143,7 +146,7 @@ namespace LibHac.Fs [FieldOffset(0x08)] public SaveDataSpaceId SpaceId; [FieldOffset(0x09)] public SaveDataType Type; [FieldOffset(0x10)] public UserId UserId; - [FieldOffset(0x20)] public ulong SaveDataIdFromKey; + [FieldOffset(0x20)] public ulong StaticSaveDataId; [FieldOffset(0x28)] public ProgramId ProgramId; [FieldOffset(0x30)] public long Size; [FieldOffset(0x38)] public short Index; diff --git a/src/LibHac/Fs/Shim/SaveDataManagement.cs b/src/LibHac/Fs/Shim/SaveDataManagement.cs index 122d7961..59602fbd 100644 --- a/src/LibHac/Fs/Shim/SaveDataManagement.cs +++ b/src/LibHac/Fs/Shim/SaveDataManagement.cs @@ -1,6 +1,7 @@ using System; using System.Runtime.InteropServices; using LibHac.Common; +using LibHac.Diag; using LibHac.FsService; using LibHac.Ncm; @@ -228,7 +229,7 @@ namespace LibHac.Fs.Shim var attribute = new SaveDataAttribute { UserId = userId, - SaveDataId = saveDataId + StaticSaveDataId = saveDataId }; var createInfo = new SaveDataCreationInfo @@ -364,7 +365,8 @@ namespace LibHac.Fs.Shim { IFileSystemProxy fsProxy = fs.GetFileSystemProxyServiceObject(); - Result rc = fsProxy.OpenSaveDataInfoReaderBySaveDataSpaceId(out ISaveDataInfoReader reader, spaceId); + Result rc = fsProxy.OpenSaveDataInfoReaderBySaveDataSpaceId( + out ReferenceCountedDisposable reader, spaceId); if (rc.IsFailure()) return rc; tempIterator = new SaveDataIterator(fs, reader); @@ -388,7 +390,8 @@ namespace LibHac.Fs.Shim { IFileSystemProxy fsProxy = fs.GetFileSystemProxyServiceObject(); - Result rc = fsProxy.OpenSaveDataInfoReaderWithFilter(out ISaveDataInfoReader reader, spaceId, ref tempFilter); + Result rc = fsProxy.OpenSaveDataInfoReaderWithFilter( + out ReferenceCountedDisposable reader, spaceId, ref tempFilter); if (rc.IsFailure()) return rc; tempIterator = new SaveDataIterator(fs, reader); @@ -402,20 +405,25 @@ namespace LibHac.Fs.Shim return result; } - public static Result DisableAutoSaveDataCreation(this FileSystemClient fsClient) + public static void DisableAutoSaveDataCreation(this FileSystemClient fsClient) { IFileSystemProxy fsProxy = fsClient.GetFileSystemProxyServiceObject(); - return fsProxy.DisableAutoSaveDataCreation(); + Result rc = fsProxy.DisableAutoSaveDataCreation(); + + if (rc.IsFailure()) + { + Abort.DoAbort(); + } } } public struct SaveDataIterator : IDisposable { private FileSystemClient FsClient { get; } - private ISaveDataInfoReader Reader { get; } + private ReferenceCountedDisposable Reader { get; } - internal SaveDataIterator(FileSystemClient fsClient, ISaveDataInfoReader reader) + internal SaveDataIterator(FileSystemClient fsClient, ReferenceCountedDisposable reader) { FsClient = fsClient; Reader = reader; @@ -430,14 +438,14 @@ namespace LibHac.Fs.Shim if (FsClient.IsEnabledAccessLog(AccessLogTarget.System)) { TimeSpan startTime = FsClient.Time.GetCurrent(); - rc = Reader.ReadSaveDataInfo(out readCount, byteBuffer); + rc = Reader.Target.Read(out readCount, byteBuffer); TimeSpan endTime = FsClient.Time.GetCurrent(); FsClient.OutputAccessLog(rc, startTime, endTime, $", size: {buffer.Length}"); } else { - rc = Reader.ReadSaveDataInfo(out readCount, byteBuffer); + rc = Reader.Target.Read(out readCount, byteBuffer); } return rc; diff --git a/src/LibHac/Fs/Shim/SystemSaveData.cs b/src/LibHac/Fs/Shim/SystemSaveData.cs index af98ef99..838db618 100644 --- a/src/LibHac/Fs/Shim/SystemSaveData.cs +++ b/src/LibHac/Fs/Shim/SystemSaveData.cs @@ -21,7 +21,7 @@ namespace LibHac.Fs.Shim SaveDataAttribute attribute = default; attribute.UserId = userId; - attribute.SaveDataId = saveDataId; + attribute.StaticSaveDataId = saveDataId; rc = fsProxy.OpenSaveDataFileSystemBySystemSaveDataId(out IFileSystem fileSystem, spaceId, ref attribute); if (rc.IsFailure()) return rc; diff --git a/src/LibHac/FsService/FileSystemProxy.cs b/src/LibHac/FsService/FileSystemProxy.cs index 6358ac05..bec3acfd 100644 --- a/src/LibHac/FsService/FileSystemProxy.cs +++ b/src/LibHac/FsService/FileSystemProxy.cs @@ -1,6 +1,7 @@ using System; using System.Runtime.CompilerServices; using LibHac.Common; +using LibHac.Diag; using LibHac.Fs; using LibHac.Fs.Fsa; using LibHac.FsService.Impl; @@ -129,13 +130,11 @@ namespace LibHac.FsService private Result DeleteSaveDataFileSystemImpl(SaveDataSpaceId spaceId, ulong saveDataId) { - SaveDataIndexerReader reader = default; + Result rc = OpenSaveDataIndexerAccessor(out SaveDataIndexerAccessor accessor, spaceId); + if (rc.IsFailure()) return rc; - try + using (accessor) { - Result rc = FsServer.SaveDataIndexerManager.GetSaveDataIndexer(out reader, spaceId); - if (rc.IsFailure()) return rc; - if (saveDataId == FileSystemServer.SaveIndexerId) { // missing: This save can only be deleted by the FS process itself @@ -144,13 +143,13 @@ namespace LibHac.FsService { if (spaceId != SaveDataSpaceId.ProperSystem && spaceId != SaveDataSpaceId.SafeMode) { - rc = reader.Indexer.GetBySaveDataId(out SaveDataIndexerValue value, saveDataId); + rc = accessor.Indexer.GetValue(out SaveDataIndexerValue value, saveDataId); if (rc.IsFailure()) return rc; spaceId = value.SpaceId; } - rc = reader.Indexer.GetKey(out SaveDataAttribute key, saveDataId); + rc = accessor.Indexer.GetKey(out SaveDataAttribute key, saveDataId); if (rc.IsFailure()) return rc; if (key.Type == SaveDataType.System || key.Type == SaveDataType.SystemBcat) @@ -162,10 +161,10 @@ namespace LibHac.FsService // Check if permissions allow deleting save data } - rc = reader.Indexer.SetState(saveDataId, SaveDataState.Creating); + rc = accessor.Indexer.SetState(saveDataId, SaveDataState.Creating); if (rc.IsFailure()) return rc; - rc = reader.Indexer.Commit(); + rc = accessor.Indexer.Commit(); if (rc.IsFailure()) return rc; } @@ -174,19 +173,15 @@ namespace LibHac.FsService if (saveDataId != FileSystemServer.SaveIndexerId) { - rc = reader.Indexer.Delete(saveDataId); + rc = accessor.Indexer.Delete(saveDataId); if (rc.IsFailure()) return rc; - rc = reader.Indexer.Commit(); + rc = accessor.Indexer.Commit(); if (rc.IsFailure()) return rc; } return Result.Success; } - finally - { - reader.Dispose(); - } } private Result DeleteSaveDataFileSystemImpl2(SaveDataSpaceId spaceId, ulong saveDataId) @@ -215,23 +210,18 @@ namespace LibHac.FsService { if (saveDataId != FileSystemServer.SaveIndexerId) { - SaveDataIndexerReader reader = default; + Result rc = OpenSaveDataIndexerAccessor(out SaveDataIndexerAccessor accessor, spaceId); + if (rc.IsFailure()) return rc; - try + using (accessor) { - Result rc = FsServer.SaveDataIndexerManager.GetSaveDataIndexer(out reader, spaceId); - if (rc.IsFailure()) return rc; - rc = reader.Indexer.GetBySaveDataId(out SaveDataIndexerValue value, saveDataId); + rc = accessor.Indexer.GetValue(out SaveDataIndexerValue value, saveDataId); if (rc.IsFailure()) return rc; if (value.SpaceId != GetSpaceIdForIndexer(spaceId)) return ResultFs.TargetNotFound.Log(); } - finally - { - reader.Dispose(); - } } return DeleteSaveDataFileSystemImpl(spaceId, saveDataId); @@ -239,25 +229,19 @@ namespace LibHac.FsService private Result GetSaveDataInfo(out SaveDataInfo info, SaveDataSpaceId spaceId, ref SaveDataAttribute attribute) { - info = default; + Unsafe.SkipInit(out info); - SaveDataIndexerReader reader = default; + Result rc = OpenSaveDataIndexerAccessor(out SaveDataIndexerAccessor accessor, spaceId); + if (rc.IsFailure()) return rc; - try + using (accessor) { - Result rc = FsServer.SaveDataIndexerManager.GetSaveDataIndexer(out reader, spaceId); + rc = accessor.Indexer.Get(out SaveDataIndexerValue value, ref attribute); if (rc.IsFailure()) return rc; - rc = reader.Indexer.Get(out SaveDataIndexerValue value, ref attribute); - if (rc.IsFailure()) return rc; - - SaveDataIndexer.GetSaveDataInfo(out info, ref attribute, ref value); + SaveDataIndexer.GenerateSaveDataInfo(out info, in attribute, in value); return Result.Success; } - finally - { - reader.Dispose(); - } } public Result DeleteSaveDataFileSystemBySaveDataAttribute(SaveDataSpaceId spaceId, ref SaveDataAttribute attribute) @@ -280,11 +264,11 @@ namespace LibHac.FsService bool isDeleteNeeded = false; Result rc; - SaveDataIndexerReader reader = default; + SaveDataIndexerAccessor accessor = null; try { - if (attribute.SaveDataId == FileSystemServer.SaveIndexerId) + if (attribute.StaticSaveDataId == FileSystemServer.SaveIndexerId) { saveDataId = FileSystemServer.SaveIndexerId; rc = FsProxyCore.DoesSaveDataExist(out bool saveExists, creationInfo.SpaceId, saveDataId); @@ -298,29 +282,29 @@ namespace LibHac.FsService } else { - rc = FsServer.SaveDataIndexerManager.GetSaveDataIndexer(out reader, creationInfo.SpaceId); + rc = OpenSaveDataIndexerAccessor(out accessor, creationInfo.SpaceId); if (rc.IsFailure()) return rc; SaveDataAttribute indexerKey = attribute; - if (attribute.SaveDataId != 0 && attribute.UserId == UserId.Zero) + if (attribute.StaticSaveDataId != 0 && attribute.UserId == UserId.Zero) { - saveDataId = attribute.SaveDataId; + saveDataId = attribute.StaticSaveDataId; - rc = reader.Indexer.AddSystemSaveData(ref indexerKey); + rc = accessor.Indexer.PutStaticSaveDataIdIndex(ref indexerKey); } else { if (attribute.Type != SaveDataType.System && attribute.Type != SaveDataType.SystemBcat) { - if (reader.Indexer.IsFull()) + if (accessor.Indexer.IsRemainedReservedOnly()) { return ResultKvdb.OutOfKeyResource.Log(); } } - rc = reader.Indexer.Add(out saveDataId, ref indexerKey); + rc = accessor.Indexer.Publish(out saveDataId, ref indexerKey); } if (ResultFs.SaveDataPathAlreadyExists.Includes(rc)) @@ -330,21 +314,21 @@ namespace LibHac.FsService isDeleteNeeded = true; - rc = reader.Indexer.SetState(saveDataId, SaveDataState.Creating); + rc = accessor.Indexer.SetState(saveDataId, SaveDataState.Creating); if (rc.IsFailure()) return rc; SaveDataSpaceId indexerSpaceId = GetSpaceIdForIndexer(creationInfo.SpaceId); - rc = reader.Indexer.SetSpaceId(saveDataId, indexerSpaceId); + rc = accessor.Indexer.SetSpaceId(saveDataId, indexerSpaceId); if (rc.IsFailure()) return rc; // todo: calculate size long size = 0; - rc = reader.Indexer.SetSize(saveDataId, size); + rc = accessor.Indexer.SetSize(saveDataId, size); if (rc.IsFailure()) return rc; - rc = reader.Indexer.Commit(); + rc = accessor.Indexer.Commit(); if (rc.IsFailure()) return rc; } @@ -386,17 +370,20 @@ namespace LibHac.FsService } } - if (attribute.SaveDataId == FileSystemServer.SaveIndexerId || something) + if (attribute.StaticSaveDataId == FileSystemServer.SaveIndexerId || something) { isDeleteNeeded = false; return Result.Success; } - rc = reader.Indexer.SetState(saveDataId, SaveDataState.Normal); + // accessor shouldn't ever be null, but checking makes the analyzers happy + Abort.DoAbortUnless(accessor != null); + + rc = accessor.Indexer.SetState(saveDataId, SaveDataState.Normal); if (rc.IsFailure()) return rc; - rc = reader.Indexer.Commit(); + rc = accessor.Indexer.Commit(); if (rc.IsFailure()) return rc; isDeleteNeeded = false; @@ -410,19 +397,19 @@ namespace LibHac.FsService { DeleteSaveDataFileSystemImpl2(creationInfo.SpaceId, saveDataId).IgnoreResult(); - if (reader.IsInitialized && saveDataId != FileSystemServer.SaveIndexerId) + if (accessor != null && saveDataId != FileSystemServer.SaveIndexerId) { - rc = reader.Indexer.GetBySaveDataId(out SaveDataIndexerValue value, saveDataId); + rc = accessor.Indexer.GetValue(out SaveDataIndexerValue value, saveDataId); if (rc.IsSuccess() && value.SpaceId == creationInfo.SpaceId) { - reader.Indexer.Delete(saveDataId).IgnoreResult(); - reader.Indexer.Commit().IgnoreResult(); + accessor.Indexer.Delete(saveDataId).IgnoreResult(); + accessor.Indexer.Commit().IgnoreResult(); } } } - reader.Dispose(); + accessor?.Dispose(); } } @@ -454,7 +441,7 @@ namespace LibHac.FsService public Result CreateSaveDataFileSystemBySystemSaveDataId(ref SaveDataAttribute attribute, ref SaveDataCreationInfo creationInfo) { - if (!IsSystemSaveDataId(attribute.SaveDataId)) + if (!IsSystemSaveDataId(attribute.StaticSaveDataId)) return ResultFs.InvalidArgument.Log(); SaveDataCreationInfo newCreationInfo = creationInfo; @@ -490,21 +477,21 @@ namespace LibHac.FsService fileSystem = default; saveDataId = default; - bool hasFixedId = attribute.SaveDataId != 0 && attribute.UserId == UserId.Zero; + bool hasFixedId = attribute.StaticSaveDataId != 0 && attribute.UserId == UserId.Zero; if (hasFixedId) { - saveDataId = attribute.SaveDataId; + saveDataId = attribute.StaticSaveDataId; } else { SaveDataAttribute indexerKey = attribute; - Result rc = FsServer.SaveDataIndexerManager.GetSaveDataIndexer(out SaveDataIndexerReader tmpReader, spaceId); - using SaveDataIndexerReader reader = tmpReader; + Result rc = OpenSaveDataIndexerAccessor(out SaveDataIndexerAccessor tempAccessor, spaceId); + using SaveDataIndexerAccessor accessor = tempAccessor; if (rc.IsFailure()) return rc; - rc = reader.Indexer.Get(out SaveDataIndexerValue indexerValue, ref indexerKey); + rc = accessor.Indexer.Get(out SaveDataIndexerValue indexerValue, ref indexerKey); if (rc.IsFailure()) return rc; SaveDataSpaceId indexerSpaceId = GetSpaceIdForIndexer(spaceId); @@ -605,7 +592,7 @@ namespace LibHac.FsService // Missing permission check, speed emulation storage type wrapper, and FileSystemInterfaceAdapter fileSystem = default; - if (!IsSystemSaveDataId(attribute.SaveDataId)) return ResultFs.InvalidArgument.Log(); + if (!IsSystemSaveDataId(attribute.StaticSaveDataId)) return ResultFs.InvalidArgument.Log(); Result rc = OpenSaveDataFileSystemImpl(out IFileSystem saveFs, out _, spaceId, ref attribute, false, true); @@ -732,41 +719,34 @@ namespace LibHac.FsService return FsProxyCore.OpenDeviceOperator(out deviceOperator); } - public Result OpenSaveDataInfoReader(out ISaveDataInfoReader infoReader) + public Result OpenSaveDataInfoReader(out ReferenceCountedDisposable infoReader) { infoReader = default; // Missing permission check - SaveDataIndexerReader indexReader = default; + Result rc = OpenSaveDataIndexerAccessor(out SaveDataIndexerAccessor accessor, SaveDataSpaceId.System); + if (rc.IsFailure()) return rc; - try + using (accessor) { - Result rc = FsServer.SaveDataIndexerManager.GetSaveDataIndexer(out indexReader, SaveDataSpaceId.System); - if (rc.IsFailure()) return rc; - - return indexReader.Indexer.OpenSaveDataInfoReader(out infoReader); - } - finally - { - indexReader.Dispose(); + return accessor.Indexer.OpenSaveDataInfoReader(out infoReader); } } - public Result OpenSaveDataInfoReaderBySaveDataSpaceId(out ISaveDataInfoReader infoReader, SaveDataSpaceId spaceId) + public Result OpenSaveDataInfoReaderBySaveDataSpaceId(out ReferenceCountedDisposable infoReader, SaveDataSpaceId spaceId) { infoReader = default; // Missing permission check - SaveDataIndexerReader indexReader = default; + Result rc = OpenSaveDataIndexerAccessor(out SaveDataIndexerAccessor accessor, spaceId); + if (rc.IsFailure()) return rc; - try + using (accessor) { - Result rc = FsServer.SaveDataIndexerManager.GetSaveDataIndexer(out indexReader, spaceId); - if (rc.IsFailure()) return rc; - - rc = indexReader.Indexer.OpenSaveDataInfoReader(out ISaveDataInfoReader baseInfoReader); + rc = accessor.Indexer.OpenSaveDataInfoReader( + out ReferenceCountedDisposable baseInfoReader); if (rc.IsFailure()) return rc; var filter = new SaveDataFilterInternal @@ -775,43 +755,36 @@ namespace LibHac.FsService SpaceId = GetSpaceIdForIndexer(spaceId) }; - infoReader = new SaveDataInfoFilterReader(baseInfoReader, ref filter); + var filterReader = new SaveDataInfoFilterReader(baseInfoReader, ref filter); + infoReader = new ReferenceCountedDisposable(filterReader); return Result.Success; } - finally - { - indexReader.Dispose(); - } } - public Result OpenSaveDataInfoReaderWithFilter(out ISaveDataInfoReader infoReader, SaveDataSpaceId spaceId, + public Result OpenSaveDataInfoReaderWithFilter(out ReferenceCountedDisposable infoReader, SaveDataSpaceId spaceId, ref SaveDataFilter filter) { infoReader = default; // Missing permission check - SaveDataIndexerReader indexReader = default; + Result rc = OpenSaveDataIndexerAccessor(out SaveDataIndexerAccessor accessor, spaceId); + if (rc.IsFailure()) return rc; - try + using (accessor) { - Result rc = FsServer.SaveDataIndexerManager.GetSaveDataIndexer(out indexReader, spaceId); - if (rc.IsFailure()) return rc; - - rc = indexReader.Indexer.OpenSaveDataInfoReader(out ISaveDataInfoReader baseInfoReader); + rc = accessor.Indexer.OpenSaveDataInfoReader( + out ReferenceCountedDisposable baseInfoReader); if (rc.IsFailure()) return rc; var filterInternal = new SaveDataFilterInternal(ref filter, spaceId); - infoReader = new SaveDataInfoFilterReader(baseInfoReader, ref filterInternal); + var filterReader = new SaveDataInfoFilterReader(baseInfoReader, ref filterInternal); + infoReader = new ReferenceCountedDisposable(filterReader); return Result.Success; } - finally - { - indexReader.Dispose(); - } } public Result FindSaveDataWithFilter(out long count, Span saveDataInfoBuffer, SaveDataSpaceId spaceId, @@ -839,23 +812,20 @@ namespace LibHac.FsService count = default; info = default; - SaveDataIndexerReader indexReader = default; - try + Result rc = OpenSaveDataIndexerAccessor(out SaveDataIndexerAccessor accessor, spaceId); + if (rc.IsFailure()) return rc; + + using (accessor) { - Result rc = FsServer.SaveDataIndexerManager.GetSaveDataIndexer(out indexReader, spaceId); + rc = accessor.Indexer.OpenSaveDataInfoReader( + out ReferenceCountedDisposable baseInfoReader); if (rc.IsFailure()) return rc; - rc = indexReader.Indexer.OpenSaveDataInfoReader(out ISaveDataInfoReader baseInfoReader); - if (rc.IsFailure()) return rc; - - var infoReader = new SaveDataInfoFilterReader(baseInfoReader, ref filter); - - return infoReader.ReadSaveDataInfo(out count, SpanHelpers.AsByteSpan(ref info)); - } - finally - { - indexReader.Dispose(); + using (var infoReader = new SaveDataInfoFilterReader(baseInfoReader, ref filter)) + { + return infoReader.Read(out count, SpanHelpers.AsByteSpan(ref info)); + } } } @@ -874,7 +844,7 @@ namespace LibHac.FsService throw new NotImplementedException(); } - public Result OpenSaveDataInfoReaderOnlyCacheStorage(out ISaveDataInfoReader infoReader) + public Result OpenSaveDataInfoReaderOnlyCacheStorage(out ReferenceCountedDisposable infoReader) { throw new NotImplementedException(); } @@ -1053,8 +1023,6 @@ namespace LibHac.FsService Result rc = FsProxyCore.SetSdCardEncryptionSeed(ref seed); if (rc.IsFailure()) return rc; - FsServer.SaveDataIndexerManager.ResetSdCardIndexer(); - return Result.Success; } @@ -1146,21 +1114,9 @@ namespace LibHac.FsService rc = saveDirFs.CleanDirectoryRecursively("/".ToU8Span()); if (rc.IsFailure()) return rc; - SaveDataIndexerReader reader = default; + FsProxyCore.SaveDataIndexerManager.ResetTemporaryStorageIndexer(SaveDataSpaceId.Temporary); - try - { - rc = FsServer.SaveDataIndexerManager.GetSaveDataIndexer(out reader, SaveDataSpaceId.Temporary); - if (rc.IsFailure()) return rc; - - reader.Indexer.Reset().IgnoreResult(); - - return Result.Success; - } - finally - { - reader.Dispose(); - } + return Result.Success; } public Result SetSdCardAccessibility(bool isAccessible) @@ -1189,7 +1145,7 @@ namespace LibHac.FsService SaveDataAttribute attribute = default; attribute.ProgramId = new ProgramId(MultiCommitManager.ProgramId); - attribute.SaveDataId = MultiCommitManager.SaveDataId; + attribute.StaticSaveDataId = MultiCommitManager.SaveDataId; Result rc = OpenSaveDataFileSystemImpl(out IFileSystem saveFs, out _, SaveDataSpaceId.System, ref attribute, false, true); @@ -1199,6 +1155,35 @@ namespace LibHac.FsService return Result.Success; } + // todo: split the FileSystemProxy classes + // nn::fssrv::SaveDataFileSystemService::GetSaveDataIndexerAccessor + private Result OpenSaveDataIndexerAccessor(out SaveDataIndexerAccessor accessor, SaveDataSpaceId spaceId) + { + accessor = default; + + Result rc = FsProxyCore.OpenSaveDataIndexerAccessor(out SaveDataIndexerAccessor accessorTemp, + out bool neededInit, spaceId); + if (rc.IsFailure()) return rc; + + try + { + if (neededInit) + { + // todo: nn::fssrv::SaveDataFileSystemService::CleanUpSaveDataCore + // nn::fssrv::SaveDataFileSystemService::CompleteSaveDataExtensionCore + } + + accessor = accessorTemp; + accessorTemp = null; + + return Result.Success; + } + finally + { + accessorTemp?.Dispose(); + } + } + private static bool IsSystemSaveDataId(ulong id) { return (long)id < 0; diff --git a/src/LibHac/FsService/FileSystemProxyCore.cs b/src/LibHac/FsService/FileSystemProxyCore.cs index 1c9aae64..a5ce4e3d 100644 --- a/src/LibHac/FsService/FileSystemProxyCore.cs +++ b/src/LibHac/FsService/FileSystemProxyCore.cs @@ -27,6 +27,8 @@ namespace LibHac.FsService private GlobalAccessLogMode LogMode { get; set; } public bool IsSdCardAccessible { get; set; } + internal ISaveDataIndexerManager SaveDataIndexerManager { get; private set; } + public FileSystemProxyCore(FileSystemCreators fsCreators, ExternalKeySet externalKeys, IDeviceOperator deviceOperator) { FsCreators = fsCreators; @@ -813,6 +815,9 @@ namespace LibHac.FsService seed.Value.CopyTo(SdEncryptionSeed); // todo: FsCreators.SaveDataFileSystemCreator.SetSdCardEncryptionSeed(seed); + SaveDataIndexerManager.InvalidateSdCardIndexer(SaveDataSpaceId.SdSystem); + SaveDataIndexerManager.InvalidateSdCardIndexer(SaveDataSpaceId.SdCache); + return Result.Success; } @@ -1058,6 +1063,16 @@ namespace LibHac.FsService return Result.Success; } + internal void SetSaveDataIndexerManager(ISaveDataIndexerManager manager) + { + SaveDataIndexerManager = manager; + } + + internal Result OpenSaveDataIndexerAccessor(out SaveDataIndexerAccessor accessor, out bool neededInit, SaveDataSpaceId spaceId) + { + return SaveDataIndexerManager.OpenAccessor(out accessor, out neededInit, spaceId); + } + private string GetSaveDataIdPath(ulong saveDataId) { return $"/{saveDataId:x16}"; diff --git a/src/LibHac/FsService/FileSystemServer.cs b/src/LibHac/FsService/FileSystemServer.cs index 84c16f05..9d3535bd 100644 --- a/src/LibHac/FsService/FileSystemServer.cs +++ b/src/LibHac/FsService/FileSystemServer.cs @@ -1,5 +1,6 @@ using System; using LibHac.Fs; +using LibHac.Fs.Impl; using LibHac.Fs.Shim; using LibHac.FsService.Creators; @@ -15,8 +16,6 @@ namespace LibHac.FsService public FileSystemClient FsClient { get; } private ITimeSpanGenerator Timer { get; } - internal SaveDataIndexerManager SaveDataIndexerManager { get; } - /// /// Creates a new . /// @@ -40,7 +39,8 @@ namespace LibHac.FsService if (FsClient.IsSdCardInserted()) FsClient.SetSdCardAccessibility(true); - SaveDataIndexerManager = new SaveDataIndexerManager(FsClient, SaveIndexerId); + FsProxyCore.SetSaveDataIndexerManager(new SaveDataIndexerManager(FsClient, SaveIndexerId, + new ArrayPoolMemoryResource(), new SdHandleManager(), false)); fsProxy.CleanUpTemporaryStorage().IgnoreResult(); } diff --git a/src/LibHac/FsService/IDeviceHandleManager.cs b/src/LibHac/FsService/IDeviceHandleManager.cs new file mode 100644 index 00000000..58f59837 --- /dev/null +++ b/src/LibHac/FsService/IDeviceHandleManager.cs @@ -0,0 +1,10 @@ +using LibHac.FsService.Storage; + +namespace LibHac.FsService +{ + public interface IDeviceHandleManager + { + Result GetHandle(out StorageDeviceHandle handle); + bool IsValid(in StorageDeviceHandle handle); + } +} diff --git a/src/LibHac/FsService/IFileSystemProxy.cs b/src/LibHac/FsService/IFileSystemProxy.cs index 77a9250c..2ad86552 100644 --- a/src/LibHac/FsService/IFileSystemProxy.cs +++ b/src/LibHac/FsService/IFileSystemProxy.cs @@ -42,14 +42,14 @@ namespace LibHac.FsService Result ReadSaveDataFileSystemExtraDataBySaveDataSpaceId(Span extraDataBuffer, SaveDataSpaceId spaceId, ulong saveDataId); Result ReadSaveDataFileSystemExtraData(Span extraDataBuffer, ulong saveDataId); Result WriteSaveDataFileSystemExtraData(ulong saveDataId, SaveDataSpaceId spaceId, ReadOnlySpan extraDataBuffer); - Result OpenSaveDataInfoReader(out ISaveDataInfoReader infoReader); - Result OpenSaveDataInfoReaderBySaveDataSpaceId(out ISaveDataInfoReader infoReader, SaveDataSpaceId spaceId); - Result OpenSaveDataInfoReaderOnlyCacheStorage(out ISaveDataInfoReader infoReader); + Result OpenSaveDataInfoReader(out ReferenceCountedDisposable infoReader); + Result OpenSaveDataInfoReaderBySaveDataSpaceId(out ReferenceCountedDisposable infoReader, SaveDataSpaceId spaceId); + Result OpenSaveDataInfoReaderOnlyCacheStorage(out ReferenceCountedDisposable infoReader); Result OpenSaveDataInternalStorageFileSystem(out IFileSystem fileSystem, SaveDataSpaceId spaceId, ulong saveDataId); Result UpdateSaveDataMacForDebug(SaveDataSpaceId spaceId, ulong saveDataId); Result WriteSaveDataFileSystemExtraDataWithMask(ulong saveDataId, SaveDataSpaceId spaceId, ReadOnlySpan extraDataBuffer, ReadOnlySpan maskBuffer); Result FindSaveDataWithFilter(out long count, Span saveDataInfoBuffer, SaveDataSpaceId spaceId, ref SaveDataFilter filter); - Result OpenSaveDataInfoReaderWithFilter(out ISaveDataInfoReader infoReader, SaveDataSpaceId spaceId, ref SaveDataFilter filter); + Result OpenSaveDataInfoReaderWithFilter(out ReferenceCountedDisposable infoReader, SaveDataSpaceId spaceId, ref SaveDataFilter filter); Result ReadSaveDataFileSystemExtraDataBySaveDataAttribute(Span extraDataBuffer, SaveDataSpaceId spaceId, ref SaveDataAttribute attribute); Result WriteSaveDataFileSystemExtraDataBySaveDataAttribute(ref SaveDataAttribute attribute, SaveDataSpaceId spaceId, ReadOnlySpan extraDataBuffer, ReadOnlySpan maskBuffer); Result OpenSaveDataMetaFile(out IFile file, SaveDataSpaceId spaceId, ref SaveDataAttribute attribute, SaveDataMetaType type); diff --git a/src/LibHac/FsService/ISaveDataIndexer.cs b/src/LibHac/FsService/ISaveDataIndexer.cs index 3cac0f4b..ea618a87 100644 --- a/src/LibHac/FsService/ISaveDataIndexer.cs +++ b/src/LibHac/FsService/ISaveDataIndexer.cs @@ -1,22 +1,25 @@ -using LibHac.Fs; +using System; +using LibHac.Fs; namespace LibHac.FsService { - public interface ISaveDataIndexer + public interface ISaveDataIndexer : IDisposable { Result Commit(); + Result Rollback(); Result Reset(); - Result Add(out ulong saveDataId, ref SaveDataAttribute key); + Result Publish(out ulong saveDataId, ref SaveDataAttribute key); Result Get(out SaveDataIndexerValue value, ref SaveDataAttribute key); - Result AddSystemSaveData(ref SaveDataAttribute key); - bool IsFull(); + Result PutStaticSaveDataIdIndex(ref SaveDataAttribute key); + bool IsRemainedReservedOnly(); Result Delete(ulong saveDataId); Result SetSpaceId(ulong saveDataId, SaveDataSpaceId spaceId); Result SetSize(ulong saveDataId, long size); Result SetState(ulong saveDataId, SaveDataState state); Result GetKey(out SaveDataAttribute key, ulong saveDataId); - Result GetBySaveDataId(out SaveDataIndexerValue value, ulong saveDataId); - int GetCount(); - Result OpenSaveDataInfoReader(out ISaveDataInfoReader infoReader); + Result GetValue(out SaveDataIndexerValue value, ulong saveDataId); + Result SetValue(ref SaveDataAttribute key, ref SaveDataIndexerValue value); + int GetIndexCount(); + Result OpenSaveDataInfoReader(out ReferenceCountedDisposable infoReader); } } \ No newline at end of file diff --git a/src/LibHac/FsService/ISaveDataIndexerManager.cs b/src/LibHac/FsService/ISaveDataIndexerManager.cs new file mode 100644 index 00000000..7f1ab6d5 --- /dev/null +++ b/src/LibHac/FsService/ISaveDataIndexerManager.cs @@ -0,0 +1,11 @@ +using LibHac.Fs; + +namespace LibHac.FsService +{ + public interface ISaveDataIndexerManager + { + Result OpenAccessor(out SaveDataIndexerAccessor accessor, out bool neededInit, SaveDataSpaceId spaceId); + void ResetTemporaryStorageIndexer(SaveDataSpaceId spaceId); + void InvalidateSdCardIndexer(SaveDataSpaceId spaceId); + } +} \ No newline at end of file diff --git a/src/LibHac/FsService/ISaveDataInfoReader.cs b/src/LibHac/FsService/ISaveDataInfoReader.cs index 2dc3abb5..4f6fd3de 100644 --- a/src/LibHac/FsService/ISaveDataInfoReader.cs +++ b/src/LibHac/FsService/ISaveDataInfoReader.cs @@ -4,6 +4,6 @@ namespace LibHac.FsService { public interface ISaveDataInfoReader : IDisposable { - Result ReadSaveDataInfo(out long readCount, Span saveDataInfoBuffer); + Result Read(out long readCount, Span saveDataInfoBuffer); } } \ No newline at end of file diff --git a/src/LibHac/FsService/SaveDataIndexer.cs b/src/LibHac/FsService/SaveDataIndexer.cs index 57fe4213..b1dd3913 100644 --- a/src/LibHac/FsService/SaveDataIndexer.cs +++ b/src/LibHac/FsService/SaveDataIndexer.cs @@ -1,10 +1,11 @@ using System; +using System.Buffers.Binary; using System.Collections.Generic; using System.Diagnostics; -using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using LibHac.Common; +using LibHac.Diag; using LibHac.Fs; using LibHac.Fs.Shim; using LibHac.Kvdb; @@ -13,31 +14,85 @@ namespace LibHac.FsService { public class SaveDataIndexer : ISaveDataIndexer { - private const string LastIdFileName = "lastPublishedId"; - private const long LastIdFileSize = 8; + private const int KvDatabaseCapacity = 0x1080; + private const int KvDatabaseReservedEntryCount = 0x80; + + private const int SaveDataAvailableSize = 0xC0000; + private const int SaveDataJournalSize = 0xC0000; + + private const long LastPublishedIdFileSize = sizeof(long); + private const int MaxPathLength = 0x30; + + private static ReadOnlySpan LastPublishedIdFileName => // lastPublishedId + new[] + { + (byte) 'l', (byte) 'a', (byte) 's', (byte) 't', (byte) 'P', (byte) 'u', (byte) 'b', (byte) 'l', + (byte) 'i', (byte) 's', (byte) 'h', (byte) 'e', (byte) 'd', (byte) 'I', (byte) 'd' + }; + + private static ReadOnlySpan MountDelimiter => // :/ + new[] { (byte)':', (byte)'/' }; + + private delegate void SaveDataValueTransform(ref SaveDataIndexerValue value, ReadOnlySpan updateData); private FileSystemClient FsClient { get; } private U8String MountName { get; } private ulong SaveDataId { get; } private SaveDataSpaceId SpaceId { get; } - private KeyValueDatabase KvDatabase { get; set; } + private MemoryResource MemoryResource { get; } + private MemoryResource BufferMemoryResource { get; } + + private FlatMapKeyValueStore KvDatabase { get; } + private object Locker { get; } = new object(); private bool IsInitialized { get; set; } private bool IsKvdbLoaded { get; set; } - private ulong LastPublishedId { get; set; } - private int Version { get; set; } - private List OpenedReaders { get; } = new List(); + private ulong _lastPublishedId; + private int Handle { get; set; } - public SaveDataIndexer(FileSystemClient fsClient, U8Span mountName, SaveDataSpaceId spaceId, ulong saveDataId) + private List OpenReaders { get; } = new List(); + + public SaveDataIndexer(FileSystemClient fsClient, U8Span mountName, SaveDataSpaceId spaceId, ulong saveDataId, MemoryResource memoryResource) { FsClient = fsClient; - MountName = mountName.ToU8String(); SaveDataId = saveDataId; SpaceId = spaceId; - Version = 1; + MemoryResource = memoryResource; + + // note: FS uses a separate PooledBufferMemoryResource here + BufferMemoryResource = memoryResource; + + KvDatabase = new FlatMapKeyValueStore(); + + IsInitialized = false; + IsKvdbLoaded = false; + Handle = 1; + + MountName = mountName.ToU8String(); } - public static void GetSaveDataInfo(out SaveDataInfo info, ref SaveDataAttribute key, ref SaveDataIndexerValue value) + private static void MakeLastPublishedIdSaveFilePath(Span buffer, ReadOnlySpan mountName) + { + // returns "%s:/%s", mountName, "lastPublishedId" + var sb = new U8StringBuilder(buffer); + sb.Append(mountName); + sb.Append(MountDelimiter); + sb.Append(LastPublishedIdFileName); + + Debug.Assert(!sb.Overflowed); + } + + private static void MakeRootPath(Span buffer, ReadOnlySpan mountName) + { + // returns "%s:/", mountName + var sb = new U8StringBuilder(buffer); + sb.Append(mountName); + sb.Append(MountDelimiter); + + Debug.Assert(!sb.Overflowed); + } + + public static void GenerateSaveDataInfo(out SaveDataInfo info, in SaveDataAttribute key, in SaveDataIndexerValue value) { info = new SaveDataInfo { @@ -45,7 +100,7 @@ namespace LibHac.FsService SpaceId = value.SpaceId, Type = key.Type, UserId = key.UserId, - SaveDataIdFromKey = key.SaveDataId, + StaticSaveDataId = key.StaticSaveDataId, ProgramId = key.ProgramId, Size = value.Size, Index = key.Index, @@ -58,47 +113,46 @@ namespace LibHac.FsService { lock (Locker) { - Result rc = Initialize(); + Result rc = TryInitializeDatabase(); if (rc.IsFailure()) return rc; - rc = EnsureKvDatabaseLoaded(false); + rc = TryLoadDatabase(false); if (rc.IsFailure()) return rc; var mount = new Mounter(); try { - rc = mount.Initialize(FsClient, MountName, SpaceId, SaveDataId); + rc = mount.Mount(FsClient, MountName, SpaceId, SaveDataId); if (rc.IsFailure()) return rc; - rc = KvDatabase.WriteDatabaseToFile(); + rc = KvDatabase.Save(); if (rc.IsFailure()) return rc; - string idFilePath = $"{MountName}:/{LastIdFileName}"; + Span lastPublishedIdPath = stackalloc byte[MaxPathLength]; + MakeLastPublishedIdSaveFilePath(lastPublishedIdPath, MountName); - rc = FsClient.OpenFile(out FileHandle handle, idFilePath.ToU8Span(), OpenMode.Write); + rc = FsClient.OpenFile(out FileHandle handle, new U8Span(lastPublishedIdPath), OpenMode.Write); if (rc.IsFailure()) return rc; - bool fileAlreadyClosed = false; + bool isFileClosed = false; try { - ulong lastId = LastPublishedId; - - rc = FsClient.WriteFile(handle, 0, SpanHelpers.AsByteSpan(ref lastId), WriteOption.None); + rc = FsClient.WriteFile(handle, 0, SpanHelpers.AsByteSpan(ref _lastPublishedId), WriteOption.None); if (rc.IsFailure()) return rc; rc = FsClient.FlushFile(handle); if (rc.IsFailure()) return rc; FsClient.CloseFile(handle); - fileAlreadyClosed = true; + isFileClosed = true; return FsClient.Commit(MountName); } finally { - if (!fileAlreadyClosed) + if (!isFileClosed) { FsClient.CloseFile(handle); } @@ -111,6 +165,21 @@ namespace LibHac.FsService } } + public Result Rollback() + { + lock (Locker) + { + Result rc = TryInitializeDatabase(); + if (rc.IsFailure()) return rc; + + rc = TryLoadDatabase(forceLoad: true); + if (rc.IsFailure()) return rc; + + UpdateHandle(); + return Result.Success; + } + } + public Result Reset() { lock (Locker) @@ -121,36 +190,37 @@ namespace LibHac.FsService if (rc.IsSuccess() || ResultFs.TargetNotFound.Includes(rc)) { - Version++; + UpdateHandle(); } return rc; } } - public Result Add(out ulong saveDataId, ref SaveDataAttribute key) + public Result Publish(out ulong saveDataId, ref SaveDataAttribute key) { saveDataId = default; lock (Locker) { - Result rc = Initialize(); + Result rc = TryInitializeDatabase(); if (rc.IsFailure()) return rc; - rc = EnsureKvDatabaseLoaded(false); + rc = TryLoadDatabase(false); if (rc.IsFailure()) return rc; - SaveDataIndexerValue value = default; + Unsafe.SkipInit(out SaveDataIndexerValue value); - rc = KvDatabase.Get(ref key, SpanHelpers.AsByteSpan(ref value)); + // Make sure the key isn't in the database already. + rc = KvDatabase.Get(out _, ref key, SpanHelpers.AsByteSpan(ref value)); if (rc.IsSuccess()) { return ResultFs.SaveDataPathAlreadyExists.Log(); } - LastPublishedId++; - ulong newSaveDataId = LastPublishedId; + _lastPublishedId++; + ulong newSaveDataId = _lastPublishedId; value = new SaveDataIndexerValue { SaveDataId = newSaveDataId }; @@ -158,11 +228,11 @@ namespace LibHac.FsService if (rc.IsFailure()) { - LastPublishedId--; + _lastPublishedId--; return rc; } - rc = AdjustOpenedInfoReaders(ref key); + rc = FixReader(ref key); if (rc.IsFailure()) return rc; saveDataId = newSaveDataId; @@ -172,17 +242,17 @@ namespace LibHac.FsService public Result Get(out SaveDataIndexerValue value, ref SaveDataAttribute key) { - value = default; + Unsafe.SkipInit(out value); lock (Locker) { - Result rc = Initialize(); + Result rc = TryInitializeDatabase(); if (rc.IsFailure()) return rc; - rc = EnsureKvDatabaseLoaded(false); + rc = TryLoadDatabase(false); if (rc.IsFailure()) return rc; - rc = KvDatabase.Get(ref key, SpanHelpers.AsByteSpan(ref value)); + rc = KvDatabase.Get(out _, ref key, SpanHelpers.AsByteSpan(ref value)); if (rc.IsFailure()) { @@ -193,174 +263,224 @@ namespace LibHac.FsService } } - public Result AddSystemSaveData(ref SaveDataAttribute key) + public Result PutStaticSaveDataIdIndex(ref SaveDataAttribute key) { lock (Locker) { - Result rc = Initialize(); + Result rc = TryInitializeDatabase(); if (rc.IsFailure()) return rc; - rc = EnsureKvDatabaseLoaded(false); + rc = TryLoadDatabase(false); if (rc.IsFailure()) return rc; - foreach (KeyValuePair kvp in KvDatabase) + // Iterate through all existing values to check if the save ID is already in use. + FlatMapKeyValueStore.Iterator iterator = KvDatabase.GetBeginIterator(); + while (!iterator.IsEnd()) { - ref SaveDataIndexerValue value = ref Unsafe.As(ref kvp.Value[0]); - - if (key.SaveDataId == value.SaveDataId) + if (iterator.GetValue().SaveDataId == key.StaticSaveDataId) { return ResultFs.SaveDataPathAlreadyExists.Log(); } + + iterator.Next(); } var newValue = new SaveDataIndexerValue { - SaveDataId = key.SaveDataId + SaveDataId = key.StaticSaveDataId }; - rc = KvDatabase.Set(ref key, SpanHelpers.AsByteSpan(ref newValue)); + rc = KvDatabase.Set(ref key, SpanHelpers.AsReadOnlyByteSpan(in newValue)); if (rc.IsFailure()) return rc; - rc = AdjustOpenedInfoReaders(ref key); + rc = FixReader(ref key); if (rc.IsFailure()) return rc; - return rc; + return Result.Success; } } - public bool IsFull() + public bool IsRemainedReservedOnly() { - return false; + return KvDatabase.Count >= KvDatabaseCapacity - KvDatabaseReservedEntryCount; } public Result Delete(ulong saveDataId) { lock (Locker) { - Result rc = Initialize(); + Result rc = TryInitializeDatabase(); if (rc.IsFailure()) return rc; - rc = EnsureKvDatabaseLoaded(false); + rc = TryLoadDatabase(false); if (rc.IsFailure()) return rc; - if (!TryGetBySaveDataIdInternal(out SaveDataAttribute key, out _, saveDataId)) + FlatMapKeyValueStore.Iterator iterator = KvDatabase.GetBeginIterator(); + + while (true) { - return ResultFs.TargetNotFound.Log(); + if (iterator.IsEnd()) + return ResultFs.TargetNotFound.Log(); + + if (iterator.GetValue().SaveDataId == saveDataId) + break; + + iterator.Next(); } + SaveDataAttribute key = iterator.Get().Key; + rc = KvDatabase.Delete(ref key); if (rc.IsFailure()) return rc; - return AdjustOpenedInfoReaders(ref key); + rc = FixReader(ref key); + if (rc.IsFailure()) return rc; + + return Result.Success; + } + } + + private Result UpdateValueBySaveDataId(ulong saveDataId, SaveDataValueTransform func, ReadOnlySpan data) + { + lock (Locker) + { + Result rc = TryInitializeDatabase(); + if (rc.IsFailure()) return rc; + + rc = TryLoadDatabase(false); + if (rc.IsFailure()) return rc; + + SaveDataIndexerValue value; + FlatMapKeyValueStore.Iterator iterator = KvDatabase.GetBeginIterator(); + + while (true) + { + if (iterator.IsEnd()) + return ResultFs.TargetNotFound.Log(); + + ref SaveDataIndexerValue val = ref iterator.GetValue(); + + if (val.SaveDataId == saveDataId) + { + value = val; + break; + } + + iterator.Next(); + } + + func(ref value, data); + + rc = KvDatabase.Set(ref iterator.Get().Key, SpanHelpers.AsReadOnlyByteSpan(in value)); + if (rc.IsFailure()) return rc; + + return Result.Success; } } public Result SetSpaceId(ulong saveDataId, SaveDataSpaceId spaceId) { - lock (Locker) + return UpdateValueBySaveDataId(saveDataId, SetSpaceIdImpl, SpanHelpers.AsReadOnlyByteSpan(in spaceId)); + + static void SetSpaceIdImpl(ref SaveDataIndexerValue value, ReadOnlySpan updateData) { - Result rc = Initialize(); - if (rc.IsFailure()) return rc; - - rc = EnsureKvDatabaseLoaded(false); - if (rc.IsFailure()) return rc; - - if (!TryGetBySaveDataIdInternal(out SaveDataAttribute key, out SaveDataIndexerValue value, saveDataId)) - { - return ResultFs.TargetNotFound.Log(); - } - - value.SpaceId = spaceId; - - return KvDatabase.Set(ref key, SpanHelpers.AsByteSpan(ref value)); + value.SpaceId = (SaveDataSpaceId)updateData[0]; } } public Result SetSize(ulong saveDataId, long size) { - lock (Locker) + return UpdateValueBySaveDataId(saveDataId, SetSizeImpl, SpanHelpers.AsReadOnlyByteSpan(in size)); + + static void SetSizeImpl(ref SaveDataIndexerValue value, ReadOnlySpan updateData) { - Result rc = Initialize(); - if (rc.IsFailure()) return rc; - - rc = EnsureKvDatabaseLoaded(false); - if (rc.IsFailure()) return rc; - - if (!TryGetBySaveDataIdInternal(out SaveDataAttribute key, out SaveDataIndexerValue value, saveDataId)) - { - return ResultFs.TargetNotFound.Log(); - } - - value.Size = size; - - return KvDatabase.Set(ref key, SpanHelpers.AsByteSpan(ref value)); + value.Size = BinaryPrimitives.ReadInt64LittleEndian(updateData); } } public Result SetState(ulong saveDataId, SaveDataState state) { - lock (Locker) + return UpdateValueBySaveDataId(saveDataId, SetSizeImpl, SpanHelpers.AsReadOnlyByteSpan(in state)); + + static void SetSizeImpl(ref SaveDataIndexerValue value, ReadOnlySpan updateData) { - Result rc = Initialize(); - if (rc.IsFailure()) return rc; - - rc = EnsureKvDatabaseLoaded(false); - if (rc.IsFailure()) return rc; - - if (!TryGetBySaveDataIdInternal(out SaveDataAttribute key, out SaveDataIndexerValue value, saveDataId)) - { - return ResultFs.TargetNotFound.Log(); - } - - value.State = state; - - return KvDatabase.Set(ref key, SpanHelpers.AsByteSpan(ref value)); + value.State = (SaveDataState)updateData[0]; } } public Result GetKey(out SaveDataAttribute key, ulong saveDataId) { - key = default; + Unsafe.SkipInit(out key); - lock (Locker) + Result rc = TryInitializeDatabase(); + if (rc.IsFailure()) return rc; + + rc = TryLoadDatabase(false); + if (rc.IsFailure()) return rc; + + FlatMapKeyValueStore.Iterator iterator = KvDatabase.GetBeginIterator(); + + while (!iterator.IsEnd()) { - Result rc = Initialize(); - if (rc.IsFailure()) return rc; - - rc = EnsureKvDatabaseLoaded(false); - if (rc.IsFailure()) return rc; - - if (TryGetBySaveDataIdInternal(out key, out _, saveDataId)) + if (iterator.GetValue().SaveDataId == saveDataId) { + key = iterator.Get().Key; return Result.Success; } - return ResultFs.TargetNotFound.Log(); + iterator.Next(); } + + return ResultFs.TargetNotFound.Log(); } - public Result GetBySaveDataId(out SaveDataIndexerValue value, ulong saveDataId) + public Result GetValue(out SaveDataIndexerValue value, ulong saveDataId) { - value = default; + Unsafe.SkipInit(out value); - lock (Locker) + Result rc = TryInitializeDatabase(); + if (rc.IsFailure()) return rc; + + rc = TryLoadDatabase(false); + if (rc.IsFailure()) return rc; + + FlatMapKeyValueStore.Iterator iterator = KvDatabase.GetBeginIterator(); + + while (!iterator.IsEnd()) { - Result rc = Initialize(); - if (rc.IsFailure()) return rc; + ref SaveDataIndexerValue val = ref iterator.GetValue(); - rc = EnsureKvDatabaseLoaded(false); - if (rc.IsFailure()) return rc; - - if (TryGetBySaveDataIdInternal(out _, out value, saveDataId)) + if (val.SaveDataId == saveDataId) { + value = val; return Result.Success; } - return ResultFs.TargetNotFound.Log(); + iterator.Next(); } + + return ResultFs.TargetNotFound.Log(); } - public int GetCount() + public Result SetValue(ref SaveDataAttribute key, ref SaveDataIndexerValue value) + { + Result rc = TryInitializeDatabase(); + if (rc.IsFailure()) return rc; + + rc = TryLoadDatabase(false); + if (rc.IsFailure()) return rc; + + FlatMapKeyValueStore.Iterator iterator = KvDatabase.GetLowerBoundIterator(ref key); + + // Key was not found + if (iterator.IsEnd()) + return ResultFs.TargetNotFound.Log(); + + iterator.GetValue() = value; + return Result.Success; + } + + public int GetIndexCount() { lock (Locker) { @@ -368,48 +488,49 @@ namespace LibHac.FsService } } - public Result OpenSaveDataInfoReader(out ISaveDataInfoReader infoReader) + public Result OpenSaveDataInfoReader(out ReferenceCountedDisposable infoReader) { infoReader = default; lock (Locker) { - Result rc = Initialize(); + Result rc = TryInitializeDatabase(); if (rc.IsFailure()) return rc; - rc = EnsureKvDatabaseLoaded(false); + rc = TryLoadDatabase(false); if (rc.IsFailure()) return rc; - var reader = new SaveDataInfoReader(this); - - OpenedReaders.Add(reader); - - infoReader = reader; - - return Result.Success; - } - } - - private bool TryGetBySaveDataIdInternal(out SaveDataAttribute key, out SaveDataIndexerValue value, ulong saveDataId) - { - foreach (KeyValuePair kvp in KvDatabase) - { - ref SaveDataIndexerValue currentValue = ref Unsafe.As(ref kvp.Value[0]); - - if (currentValue.SaveDataId == saveDataId) + // Create the reader and register it in the opened-reader list + using (var reader = new ReferenceCountedDisposable(new Reader(this))) { - key = kvp.Key; - value = currentValue; - return true; + rc = RegisterReader(reader); + if (rc.IsFailure()) return rc; + + infoReader = reader.AddReference(); + return Result.Success; } } - - key = default; - value = default; - return false; } - private Result Initialize() + private FlatMapKeyValueStore.Iterator GetBeginIterator() + { + Assert.AssertTrue(IsKvdbLoaded); + + return KvDatabase.GetBeginIterator(); + } + + private void FixIterator(ref FlatMapKeyValueStore.Iterator iterator, + ref SaveDataAttribute key) + { + KvDatabase.FixIterator(ref iterator, ref key); + } + + /// + /// Initializes and ensures that the indexer's save data is created. + /// Does nothing if this has already been initialized. + /// + /// The of the operation. + private Result TryInitializeDatabase() { if (IsInitialized) return Result.Success; @@ -417,21 +538,15 @@ namespace LibHac.FsService try { - Result rc = mount.Initialize(FsClient, MountName, SpaceId, SaveDataId); + Result rc = mount.Mount(FsClient, MountName, SpaceId, SaveDataId); if (rc.IsFailure()) return rc; - string dbDirectory = $"{MountName}:/"; + Span rootPath = stackalloc byte[MaxPathLength]; + MakeRootPath(rootPath, MountName); - rc = FsClient.GetEntryType(out DirectoryEntryType entryType, dbDirectory.ToU8Span()); + rc = KvDatabase.Initialize(FsClient, new U8Span(rootPath), KvDatabaseCapacity, MemoryResource, BufferMemoryResource); if (rc.IsFailure()) return rc; - if (entryType == DirectoryEntryType.File) - return ResultFs.PathNotFound.Log(); - - string dbArchiveFile = $"{dbDirectory}imkvdb.arc"; - - KvDatabase = new KeyValueDatabase(FsClient, dbArchiveFile.ToU8Span()); - IsInitialized = true; return Result.Success; } @@ -441,10 +556,15 @@ namespace LibHac.FsService } } - private Result EnsureKvDatabaseLoaded(bool forceLoad) + /// + /// Ensures that the database file exists and loads any existing entries. + /// Does nothing if the database has already been loaded and is . + /// + /// If , forces the database to be reloaded, + /// even it it was already loaded previously. + /// The of the operation. + private Result TryLoadDatabase(bool forceLoad) { - Debug.Assert(KvDatabase != null); - if (forceLoad) { IsKvdbLoaded = false; @@ -458,55 +578,63 @@ namespace LibHac.FsService try { - Result rc = mount.Initialize(FsClient, MountName, SpaceId, SaveDataId); + Result rc = mount.Mount(FsClient, MountName, SpaceId, SaveDataId); if (rc.IsFailure()) return rc; - rc = KvDatabase.ReadDatabaseFromFile(); + rc = KvDatabase.Load(); if (rc.IsFailure()) return rc; bool createdNewFile = false; - var idFilePath = $"{MountName}:/{LastIdFileName}".ToU8String(); - rc = FsClient.OpenFile(out FileHandle handle, idFilePath, OpenMode.Read); - - if (rc.IsFailure()) - { - if (!ResultFs.PathNotFound.Includes(rc)) return rc; - - rc = FsClient.CreateFile(idFilePath, LastIdFileSize); - if (rc.IsFailure()) return rc; - - rc = FsClient.OpenFile(out handle, idFilePath, OpenMode.Read); - if (rc.IsFailure()) return rc; - - createdNewFile = true; - - LastPublishedId = 0; - IsKvdbLoaded = true; - } + Span lastPublishedIdPath = stackalloc byte[MaxPathLength]; + MakeLastPublishedIdSaveFilePath(lastPublishedIdPath, MountName); try { - if (!createdNewFile) - { - ulong lastId = default; + rc = FsClient.OpenFile(out FileHandle handle, new U8Span(lastPublishedIdPath), OpenMode.Read); - rc = FsClient.ReadFile(handle, 0, SpanHelpers.AsByteSpan(ref lastId)); + // Create the last published ID file if it doesn't exist. + if (rc.IsFailure()) + { + if (!ResultFs.PathNotFound.Includes(rc)) return rc; + + rc = FsClient.CreateFile(new U8Span(lastPublishedIdPath), LastPublishedIdFileSize); if (rc.IsFailure()) return rc; - LastPublishedId = lastId; + rc = FsClient.OpenFile(out handle, new U8Span(lastPublishedIdPath), OpenMode.Read); + if (rc.IsFailure()) return rc; + + createdNewFile = true; + + _lastPublishedId = 0; IsKvdbLoaded = true; } - return Result.Success; + try + { + // If we had to create the file earlier, we don't need to load the value again. + if (!createdNewFile) + { + rc = FsClient.ReadFile(handle, 0, SpanHelpers.AsByteSpan(ref _lastPublishedId)); + if (rc.IsFailure()) return rc; + + IsKvdbLoaded = true; + } + + return Result.Success; + } + finally + { + FsClient.CloseFile(handle); + } } finally { - FsClient.CloseFile(handle); - + // The save data needs to be committed if we created the last published ID file. if (createdNewFile) { - FsClient.Commit(MountName); + // Note: Nintendo does not check this return value, probably because it's in a scope-exit block. + FsClient.Commit(MountName).IgnoreResult(); } } } @@ -516,56 +644,61 @@ namespace LibHac.FsService } } - private Result AdjustOpenedInfoReaders(ref SaveDataAttribute key) + private void UpdateHandle() => Handle++; + + /// + /// Adds a to the list of registered readers. + /// + /// The reader to add. + /// The of the operation. + private Result RegisterReader(ReferenceCountedDisposable reader) { - // If a new key is added or removed during iteration of the list, - // make sure the current item of the iterator remains the same - - // Todo: A more efficient way of doing this - List list = KvDatabase.ToList().Select(x => x.key).ToList(); - - int index = list.BinarySearch(key); - - bool keyWasAdded = index >= 0; - - if (!keyWasAdded) - { - // If the item was not found, List.BinarySearch returns a negative number that - // is the bitwise complement of the index of the next element that is larger than the item - index = ~index; - } - - foreach (SaveDataInfoReader reader in OpenedReaders) - { - if (keyWasAdded) - { - // New key was inserted before the iterator's position - // increment the position to compensate - if (reader.Position >= index) - { - reader.Position++; - } - } - else - { - // The position should be decremented if the iterator's position is - // after the key that came directly after the deleted key - if (reader.Position > index) - { - reader.Position--; - } - } - } + OpenReaders.Add(new ReaderAccessor(reader)); return Result.Success; } - private void CloseReader(SaveDataInfoReader reader) + /// + /// Removes any s that are no longer in use from the registered readers. + /// + private void UnregisterReader() { - // ReSharper disable once RedundantAssignment - bool wasRemoved = OpenedReaders.Remove(reader); + int i = 0; + List readers = OpenReaders; - Debug.Assert(wasRemoved); + while (i < readers.Count) + { + // Remove the reader if there are no references to it. There is no need to increment + // i in this case because the next reader in the list will be shifted to index i + if (readers[i].IsExpired()) + { + readers.RemoveAt(i); + } + else + { + i++; + } + } + } + + /// + /// Adjusts the position of any opened s so that they still point to the + /// same element after the addition or removal of another element. If the reader was on the + /// element that was removed, it will now point to the element that was next in the list. + /// + /// The key of the element that was removed or added. + /// The of the operation. + private Result FixReader(ref SaveDataAttribute key) + { + foreach (ReaderAccessor accessor in OpenReaders) + { + using (ReferenceCountedDisposable reader = accessor.Lock()) + { + reader?.Target.Fix(ref key); + } + } + + return Result.Success; } private ref struct Mounter @@ -574,7 +707,7 @@ namespace LibHac.FsService private U8String MountName { get; set; } private bool IsMounted { get; set; } - public Result Initialize(FileSystemClient fsClient, U8String mountName, SaveDataSpaceId spaceId, + public Result Mount(FileSystemClient fsClient, U8String mountName, SaveDataSpaceId spaceId, ulong saveDataId) { FsClient = fsClient; @@ -588,7 +721,7 @@ namespace LibHac.FsService { if (ResultFs.TargetNotFound.Includes(rc)) { - rc = FsClient.CreateSystemSaveData(spaceId, saveDataId, 0, 0xC0000, 0xC0000, 0); + rc = FsClient.CreateSystemSaveData(spaceId, saveDataId, 0, SaveDataAvailableSize, SaveDataJournalSize, 0); if (rc.IsFailure()) return rc; rc = FsClient.MountSystemSaveData(MountName, spaceId, saveDataId); @@ -626,61 +759,90 @@ namespace LibHac.FsService } } - private class SaveDataInfoReader : ISaveDataInfoReader + private class ReaderAccessor { - private SaveDataIndexer Indexer { get; } - private int Version { get; } - public int Position { get; set; } + private ReferenceCountedDisposable.WeakReference _reader; - public SaveDataInfoReader(SaveDataIndexer indexer) + public ReaderAccessor(ReferenceCountedDisposable reader) { - Indexer = indexer; - Version = indexer.Version; + _reader = new ReferenceCountedDisposable.WeakReference(reader); } - public Result ReadSaveDataInfo(out long readCount, Span saveDataInfoBuffer) + public ReferenceCountedDisposable Lock() { - readCount = default; + return _reader.TryAddReference(); + } - lock (Indexer.Locker) + public bool IsExpired() + { + using (ReferenceCountedDisposable reference = _reader.TryAddReference()) + { + return reference == null; + } + } + } + + private class Reader : ISaveDataInfoReader + { + private readonly SaveDataIndexer _indexer; + private FlatMapKeyValueStore.Iterator _iterator; + private readonly int _handle; + + public Reader(SaveDataIndexer indexer) + { + _indexer = indexer; + _handle = indexer.Handle; + + _iterator = indexer.GetBeginIterator(); + } + + public Result Read(out long readCount, Span saveDataInfoBuffer) + { + Unsafe.SkipInit(out readCount); + + lock (_indexer.Locker) { // Indexer has been reloaded since this info reader was created - if (Version != Indexer.Version) + if (_handle != _indexer.Handle) { return ResultFs.InvalidSaveDataInfoReader.Log(); } - // No more to iterate - if (Position == Indexer.KvDatabase.Count) - { - readCount = 0; - return Result.Success; - } - Span outInfo = MemoryMarshal.Cast(saveDataInfoBuffer); - // Todo: A more efficient way of doing this - List<(SaveDataAttribute key, byte[] value)> list = Indexer.KvDatabase.ToList(); - int i; - for (i = 0; i < outInfo.Length && Position < list.Count; i++, Position++) + for (i = 0; !_iterator.IsEnd() && i < outInfo.Length; i++) { - SaveDataAttribute key = list[Position].key; - ref SaveDataIndexerValue value = ref Unsafe.As(ref list[Position].value[0]); + ref SaveDataAttribute key = ref _iterator.Get().Key; + ref SaveDataIndexerValue value = ref _iterator.GetValue(); - GetSaveDataInfo(out outInfo[i], ref key, ref value); + GenerateSaveDataInfo(out outInfo[i], in key, in value); + + _iterator.Next(); } readCount = i; - return Result.Success; } } + public void Fix(ref SaveDataAttribute attribute) + { + _indexer.FixIterator(ref _iterator, ref attribute); + } + public void Dispose() { - Indexer?.CloseReader(this); + lock (_indexer.Locker) + { + _indexer.UnregisterReader(); + } } } + + public void Dispose() + { + KvDatabase?.Dispose(); + } } } diff --git a/src/LibHac/FsService/SaveDataIndexerLite.cs b/src/LibHac/FsService/SaveDataIndexerLite.cs index 65f8034e..040ccedb 100644 --- a/src/LibHac/FsService/SaveDataIndexerLite.cs +++ b/src/LibHac/FsService/SaveDataIndexerLite.cs @@ -18,6 +18,11 @@ namespace LibHac.FsService return Result.Success; } + public Result Rollback() + { + return Result.Success; + } + public Result Reset() { lock (Locker) @@ -27,11 +32,11 @@ namespace LibHac.FsService } } - public Result Add(out ulong saveDataId, ref SaveDataAttribute key) + public Result Publish(out ulong saveDataId, ref SaveDataAttribute key) { lock (Locker) { - if (IsKeyValueSet && _key.Equals(key)) + if (IsKeyValueSet && _key == key) { saveDataId = default; return ResultFs.SaveDataPathAlreadyExists.Log(); @@ -52,7 +57,7 @@ namespace LibHac.FsService { lock (Locker) { - if (IsKeyValueSet && _key.Equals(key)) + if (IsKeyValueSet && _key == key) { value = _value; return Result.Success; @@ -63,11 +68,11 @@ namespace LibHac.FsService } } - public Result AddSystemSaveData(ref SaveDataAttribute key) + public Result PutStaticSaveDataIdIndex(ref SaveDataAttribute key) { lock (Locker) { - if (IsKeyValueSet && _key.Equals(key)) + if (IsKeyValueSet && _key == key) { return ResultFs.SaveDataPathAlreadyExists.Log(); } @@ -81,7 +86,7 @@ namespace LibHac.FsService } } - public bool IsFull() + public bool IsRemainedReservedOnly() { return false; } @@ -159,7 +164,7 @@ namespace LibHac.FsService } } - public Result GetBySaveDataId(out SaveDataIndexerValue value, ulong saveDataId) + public Result GetValue(out SaveDataIndexerValue value, ulong saveDataId) { lock (Locker) { @@ -174,22 +179,40 @@ namespace LibHac.FsService } } - public int GetCount() + public Result SetValue(ref SaveDataAttribute key, ref SaveDataIndexerValue value) + { + lock (Locker) + { + if (IsKeyValueSet && _key == key) + { + _value = value; + return Result.Success; + } + + return ResultFs.TargetNotFound.Log(); + } + } + + public int GetIndexCount() { return 1; } - public Result OpenSaveDataInfoReader(out ISaveDataInfoReader infoReader) + public Result OpenSaveDataInfoReader(out ReferenceCountedDisposable infoReader) { + SaveDataIndexerLiteInfoReader reader; + if (IsKeyValueSet) { - infoReader = new SaveDataIndexerLiteInfoReader(ref _key, ref _value); + reader = new SaveDataIndexerLiteInfoReader(ref _key, ref _value); } else { - infoReader = new SaveDataIndexerLiteInfoReader(); + reader = new SaveDataIndexerLiteInfoReader(); } + infoReader = new ReferenceCountedDisposable(reader); + return Result.Success; } @@ -205,14 +228,14 @@ namespace LibHac.FsService public SaveDataIndexerLiteInfoReader(ref SaveDataAttribute key, ref SaveDataIndexerValue value) { - SaveDataIndexer.GetSaveDataInfo(out _info, ref key, ref value); + SaveDataIndexer.GenerateSaveDataInfo(out _info, in key, in value); } - public Result ReadSaveDataInfo(out long readCount, Span saveDataInfoBuffer) + public Result Read(out long readCount, Span saveDataInfoBuffer) { Span outInfo = MemoryMarshal.Cast(saveDataInfoBuffer); - // Nintendo doesn't check if the buffer is too small here + // Note: Nintendo doesn't check if the buffer is too small here if (_finishedIterating || outInfo.IsEmpty) { readCount = 0; @@ -229,5 +252,7 @@ namespace LibHac.FsService public void Dispose() { } } + + public void Dispose() { } } } diff --git a/src/LibHac/FsService/SaveDataIndexerManager.cs b/src/LibHac/FsService/SaveDataIndexerManager.cs index ee6722da..d3d1f8e5 100644 --- a/src/LibHac/FsService/SaveDataIndexerManager.cs +++ b/src/LibHac/FsService/SaveDataIndexerManager.cs @@ -1,28 +1,51 @@ -using System.Threading; +using System; +using System.Threading; using LibHac.Common; +using LibHac.Diag; using LibHac.Fs; +using LibHac.FsService.Storage; namespace LibHac.FsService { - internal class SaveDataIndexerManager + internal class SaveDataIndexerManager : ISaveDataIndexerManager { private FileSystemClient FsClient { get; } + private MemoryResource MemoryResource { get; } private ulong SaveDataId { get; } private IndexerHolder _bisIndexer = new IndexerHolder(new object()); - private IndexerHolder _sdCardIndexer = new IndexerHolder(new object()); private IndexerHolder _tempIndexer = new IndexerHolder(new object()); + + private IndexerHolder _sdCardIndexer = new IndexerHolder(new object()); + private StorageDeviceHandle _sdCardHandle; + private IDeviceHandleManager _sdCardHandleManager; + private IndexerHolder _safeIndexer = new IndexerHolder(new object()); private IndexerHolder _properSystemIndexer = new IndexerHolder(new object()); - public SaveDataIndexerManager(FileSystemClient fsClient, ulong saveDataId) + private bool IsBisUserRedirectionEnabled { get; } + + public SaveDataIndexerManager(FileSystemClient fsClient, ulong saveDataId, MemoryResource memoryResource, + IDeviceHandleManager sdCardHandleManager, bool isBisUserRedirectionEnabled) { FsClient = fsClient; SaveDataId = saveDataId; + MemoryResource = memoryResource; + _sdCardHandleManager = sdCardHandleManager; + IsBisUserRedirectionEnabled = isBisUserRedirectionEnabled; + + _tempIndexer.Indexer = new SaveDataIndexerLite(); } - public Result GetSaveDataIndexer(out SaveDataIndexerReader reader, SaveDataSpaceId spaceId) + public Result OpenAccessor(out SaveDataIndexerAccessor accessor, out bool neededInit, SaveDataSpaceId spaceId) { + neededInit = false; + + if (IsBisUserRedirectionEnabled && spaceId == SaveDataSpaceId.User) + { + spaceId = SaveDataSpaceId.ProperSystem; + } + switch (spaceId) { case SaveDataSpaceId.System: @@ -31,70 +54,106 @@ namespace LibHac.FsService if (!_bisIndexer.IsInitialized) { - _bisIndexer.Indexer = new SaveDataIndexer(FsClient, "saveDataIxrDb".ToU8Span(), SaveDataSpaceId.System, SaveDataId); + _bisIndexer.Indexer = new SaveDataIndexer(FsClient, new U8Span(SystemIndexerMountName), + SaveDataSpaceId.System, SaveDataId, MemoryResource); + + neededInit = true; } - reader = new SaveDataIndexerReader(_bisIndexer.Indexer, _bisIndexer.Locker); + accessor = new SaveDataIndexerAccessor(_bisIndexer.Indexer, _bisIndexer.Locker); return Result.Success; case SaveDataSpaceId.SdSystem: case SaveDataSpaceId.SdCache: Monitor.Enter(_sdCardIndexer.Locker); - // todo: Missing reinitialize if SD handle is old + // We need to reinitialize the indexer if the SD card has changed + if (!_sdCardHandleManager.IsValid(in _sdCardHandle) && _sdCardIndexer.IsInitialized) + { + _sdCardIndexer.Indexer.Dispose(); + _sdCardIndexer.Indexer = null; + } if (!_sdCardIndexer.IsInitialized) { - _sdCardIndexer.Indexer = new SaveDataIndexer(FsClient, "saveDataIxrDbSd".ToU8Span(), SaveDataSpaceId.SdSystem, SaveDataId); + _sdCardIndexer.Indexer = new SaveDataIndexer(FsClient, new U8Span(SdCardIndexerMountName), + SaveDataSpaceId.SdSystem, SaveDataId, MemoryResource); + + _sdCardHandleManager.GetHandle(out _sdCardHandle).IgnoreResult(); + + neededInit = true; } - reader = new SaveDataIndexerReader(_sdCardIndexer.Indexer, _sdCardIndexer.Locker); + accessor = new SaveDataIndexerAccessor(_sdCardIndexer.Indexer, _sdCardIndexer.Locker); return Result.Success; case SaveDataSpaceId.Temporary: Monitor.Enter(_tempIndexer.Locker); - if (!_tempIndexer.IsInitialized) - { - _tempIndexer.Indexer = new SaveDataIndexerLite(); - } - - reader = new SaveDataIndexerReader(_tempIndexer.Indexer, _tempIndexer.Locker); + accessor = new SaveDataIndexerAccessor(_tempIndexer.Indexer, _tempIndexer.Locker); return Result.Success; case SaveDataSpaceId.ProperSystem: - Monitor.Enter(_safeIndexer.Locker); - - if (!_safeIndexer.IsInitialized) - { - _safeIndexer.Indexer = new SaveDataIndexer(FsClient, "saveDataIxrDbPr".ToU8Span(), SaveDataSpaceId.ProperSystem, SaveDataId); - } - - reader = new SaveDataIndexerReader(_safeIndexer.Indexer, _safeIndexer.Locker); - return Result.Success; - - case SaveDataSpaceId.SafeMode: Monitor.Enter(_properSystemIndexer.Locker); if (!_properSystemIndexer.IsInitialized) { - _properSystemIndexer.Indexer = new SaveDataIndexer(FsClient, "saveDataIxrDbSf".ToU8Span(), SaveDataSpaceId.SafeMode, SaveDataId); + _properSystemIndexer.Indexer = new SaveDataIndexer(FsClient, new U8Span(ProperSystemIndexerMountName), + SaveDataSpaceId.ProperSystem, SaveDataId, MemoryResource); + + neededInit = true; } - reader = new SaveDataIndexerReader(_properSystemIndexer.Indexer, _properSystemIndexer.Locker); + accessor = new SaveDataIndexerAccessor(_properSystemIndexer.Indexer, _properSystemIndexer.Locker); + return Result.Success; + + case SaveDataSpaceId.SafeMode: + Monitor.Enter(_safeIndexer.Locker); + + if (!_safeIndexer.IsInitialized) + { + _safeIndexer.Indexer = new SaveDataIndexer(FsClient, new U8Span(SafeModeIndexerMountName), + SaveDataSpaceId.SafeMode, SaveDataId, MemoryResource); + + neededInit = true; + } + + accessor = new SaveDataIndexerAccessor(_safeIndexer.Indexer, _safeIndexer.Locker); return Result.Success; default: - reader = default; + accessor = default; return ResultFs.InvalidArgument.Log(); } } - internal void ResetSdCardIndexer() + public void ResetTemporaryStorageIndexer(SaveDataSpaceId spaceId) { + if (spaceId != SaveDataSpaceId.Temporary) + { + Abort.UnexpectedDefault(); + } + + // ReSharper disable once RedundantAssignment + Result rc = _tempIndexer.Indexer.Reset(); + Assert.AssertTrue(rc.IsSuccess()); + } + + public void InvalidateSdCardIndexer(SaveDataSpaceId spaceId) + { + // Note: Nintendo doesn't lock when doing this operation lock (_sdCardIndexer.Locker) { - _sdCardIndexer.Indexer = null; + if (spaceId != SaveDataSpaceId.SdCache && spaceId != SaveDataSpaceId.SdSystem) + { + Abort.UnexpectedDefault(); + } + + if (_sdCardIndexer.IsInitialized) + { + _sdCardIndexer.Indexer.Dispose(); + _sdCardIndexer.Indexer = null; + } } } @@ -111,30 +170,56 @@ namespace LibHac.FsService public bool IsInitialized => Indexer != null; } + + private static ReadOnlySpan SystemIndexerMountName => // saveDataIxrDb + new[] + { + (byte) 's', (byte) 'a', (byte) 'v', (byte) 'e', (byte) 'D', (byte) 'a', (byte) 't', (byte) 'a', + (byte) 'I', (byte) 'x', (byte) 'r', (byte) 'D', (byte) 'b' + }; + + private static ReadOnlySpan SdCardIndexerMountName => // saveDataIxrDbSd + new[] + { + (byte) 's', (byte) 'a', (byte) 'v', (byte) 'e', (byte) 'D', (byte) 'a', (byte) 't', (byte) 'a', + (byte) 'I', (byte) 'x', (byte) 'r', (byte) 'D', (byte) 'b', (byte) 'S', (byte) 'd' + }; + + private static ReadOnlySpan ProperSystemIndexerMountName => // saveDataIxrDbPr + new[] + { + (byte) 's', (byte) 'a', (byte) 'v', (byte) 'e', (byte) 'D', (byte) 'a', (byte) 't', (byte) 'a', + (byte) 'I', (byte) 'x', (byte) 'r', (byte) 'D', (byte) 'b', (byte) 'P', (byte) 'r' + }; + + private static ReadOnlySpan SafeModeIndexerMountName => // saveDataIxrDbSf + new[] + { + (byte) 's', (byte) 'a', (byte) 'v', (byte) 'e', (byte) 'D', (byte) 'a', (byte) 't', (byte) 'a', + (byte) 'I', (byte) 'x', (byte) 'r', (byte) 'D', (byte) 'b', (byte) 'S', (byte) 'f' + }; } - public ref struct SaveDataIndexerReader + public class SaveDataIndexerAccessor : IDisposable { - private bool _isInitialized; - private object _locker; + public ISaveDataIndexer Indexer { get; } + private object Locker { get; } + private bool HasLock { get; set; } - public ISaveDataIndexer Indexer; - public bool IsInitialized => _isInitialized; - - internal SaveDataIndexerReader(ISaveDataIndexer indexer, object locker) + public SaveDataIndexerAccessor(ISaveDataIndexer indexer, object locker) { - _isInitialized = true; - _locker = locker; Indexer = indexer; + Locker = locker; + HasLock = true; } public void Dispose() { - if (_isInitialized) + if (HasLock) { - Monitor.Exit(_locker); + Monitor.Exit(Locker); - _isInitialized = false; + HasLock = false; } } } diff --git a/src/LibHac/FsService/SaveDataInfoFilterReader.cs b/src/LibHac/FsService/SaveDataInfoFilterReader.cs index c2345af5..6f639309 100644 --- a/src/LibHac/FsService/SaveDataInfoFilterReader.cs +++ b/src/LibHac/FsService/SaveDataInfoFilterReader.cs @@ -8,16 +8,16 @@ namespace LibHac.FsService { internal class SaveDataInfoFilterReader : ISaveDataInfoReader { - private ISaveDataInfoReader Reader { get; } + private ReferenceCountedDisposable Reader { get; } private SaveDataFilterInternal Filter { get; } - public SaveDataInfoFilterReader(ISaveDataInfoReader reader, ref SaveDataFilterInternal filter) + public SaveDataInfoFilterReader(ReferenceCountedDisposable reader, ref SaveDataFilterInternal filter) { Reader = reader; Filter = filter; } - public Result ReadSaveDataInfo(out long readCount, Span saveDataInfoBuffer) + public Result Read(out long readCount, Span saveDataInfoBuffer) { readCount = default; @@ -26,11 +26,12 @@ namespace LibHac.FsService SaveDataInfo tempInfo = default; Span tempInfoBytes = SpanHelpers.AsByteSpan(ref tempInfo); + ISaveDataInfoReader reader = Reader.Target; int count = 0; while (count < outInfo.Length) { - Result rc = Reader.ReadSaveDataInfo(out long baseReadCount, tempInfoBytes); + Result rc = reader.Read(out long baseReadCount, tempInfoBytes); if (rc.IsFailure()) return rc; if (baseReadCount == 0) break; diff --git a/src/LibHac/FsService/Storage/SdCardManagement.cs b/src/LibHac/FsService/Storage/SdCardManagement.cs new file mode 100644 index 00000000..d84f2574 --- /dev/null +++ b/src/LibHac/FsService/Storage/SdCardManagement.cs @@ -0,0 +1,20 @@ +namespace LibHac.FsService.Storage +{ + public static class SdCardManagement + { + public static Result GetCurrentSdCardHandle(out StorageDeviceHandle handle) + { + // todo: StorageDevice interfaces + handle = new StorageDeviceHandle(1, StorageDevicePortId.SdCard); + return Result.Success; + } + + public static Result IsSdCardHandleValid(out bool isValid, in StorageDeviceHandle handle) + { + // todo: StorageDevice interfaces + isValid = handle.PortId == StorageDevicePortId.SdCard; + + return Result.Success; + } + } +} diff --git a/src/LibHac/FsService/Storage/StorageDeviceHandle.cs b/src/LibHac/FsService/Storage/StorageDeviceHandle.cs new file mode 100644 index 00000000..149050ce --- /dev/null +++ b/src/LibHac/FsService/Storage/StorageDeviceHandle.cs @@ -0,0 +1,26 @@ +using System; +using System.Runtime.InteropServices; + +namespace LibHac.FsService.Storage +{ + [StructLayout(LayoutKind.Sequential, Size = 0x10)] + public readonly struct StorageDeviceHandle : IEquatable + { + public readonly uint Value; + public readonly StorageDevicePortId PortId; + + public StorageDeviceHandle(uint value, StorageDevicePortId portId) + { + Value = value; + PortId = portId; + } + + public override bool Equals(object obj) => obj is StorageDeviceHandle other && Equals(other); + public bool Equals(StorageDeviceHandle other) => Value == other.Value && PortId == other.PortId; + + public static bool operator ==(StorageDeviceHandle left, StorageDeviceHandle right) => left.Equals(right); + public static bool operator !=(StorageDeviceHandle left, StorageDeviceHandle right) => !(left == right); + + public override int GetHashCode() => HashCode.Combine(Value, (int) PortId); + } +} diff --git a/src/LibHac/FsService/Storage/StorageDevicePortId.cs b/src/LibHac/FsService/Storage/StorageDevicePortId.cs new file mode 100644 index 00000000..22a8349f --- /dev/null +++ b/src/LibHac/FsService/Storage/StorageDevicePortId.cs @@ -0,0 +1,10 @@ +namespace LibHac.FsService.Storage +{ + public enum StorageDevicePortId : byte + { + Invalid = 0, + Mmc = 1, + SdCard = 2, + GameCard = 3 + } +} diff --git a/src/LibHac/Kvdb/FlatMapKeyValueStore.cs b/src/LibHac/Kvdb/FlatMapKeyValueStore.cs index efe6dfb9..dbeb2211 100644 --- a/src/LibHac/Kvdb/FlatMapKeyValueStore.cs +++ b/src/LibHac/Kvdb/FlatMapKeyValueStore.cs @@ -25,8 +25,8 @@ namespace LibHac.Kvdb private static ReadOnlySpan ArchiveFileName => // /imkvdb.arc new[] { - (byte) '/', (byte) 'i', (byte) 'm', (byte) 'k', (byte) 'v', (byte) 'd', (byte) 'b', (byte) '.', - (byte) 'a', (byte) 'r', (byte) 'c' + (byte) '/', (byte) 'i', (byte) 'm', (byte) 'k', (byte) 'v', (byte) 'd', (byte) 'b', (byte) '.', + (byte) 'a', (byte) 'r', (byte) 'c' }; public int Count => _index.Count; @@ -646,6 +646,11 @@ namespace LibHac.Kvdb public ref KeyValue Get() => ref _entries[_index]; public Span GetValue() => _entries[_index].Value.Get(); + public ref T GetValue() where T : unmanaged + { + return ref SpanHelpers.AsStruct(_entries[_index].Value.Get()); + } + public void Next() => _index++; public bool IsEnd() => _index == _length; diff --git a/src/LibHac/ReferenceCountedDisposable.cs b/src/LibHac/ReferenceCountedDisposable.cs index 5943417a..ca2b74ed 100644 --- a/src/LibHac/ReferenceCountedDisposable.cs +++ b/src/LibHac/ReferenceCountedDisposable.cs @@ -134,6 +134,54 @@ namespace LibHac => TryAddReferenceImpl(_instance, _boxedReferenceCount) ?? throw new ObjectDisposedException(nameof(ReferenceCountedDisposable)); + /// + /// Increments the reference count for the disposable object, and returns a new disposable reference to it + /// of type . The type of the disposable object must be compatible with type + /// . + /// + /// + /// The returned object is an independent reference to the same underlying object. Disposing of the + /// returned value multiple times will only cause the reference count to be decreased once. + /// + /// The type of the new reference to the disposable object. + /// A new pointing to the same underlying object, if it + /// has not yet been disposed; otherwise, is thrown if this reference + /// to the underlying object has already been disposed, and is thrown if + /// is not compatible with . + public ReferenceCountedDisposable AddReference() where TTo : class, IDisposable + { + ReferenceCountedDisposable? newReference = + TryAddReferenceImpl(_instance, _boxedReferenceCount, out CreateResult result); + + if (newReference != null) + { + return newReference; + } + + throw result switch + { + CreateResult.Disposed => new ObjectDisposedException(nameof(ReferenceCountedDisposable)), + CreateResult.NotCastable => new InvalidCastException(), + _ => new NotSupportedException("This exception should never be thrown.") + }; + } + + /// + /// Increments the reference count for the disposable object, and returns a new disposable reference to it + /// of type . The type of the disposable object must be compatible with type + /// . + /// + /// + /// The returned object is an independent reference to the same underlying object. Disposing of the + /// returned value multiple times will only cause the reference count to be decreased once. + /// + /// A new pointing to the same underlying object, if it + /// has not yet been disposed, and is compatible with ; + /// otherwise, if this reference to the underlying object has already been disposed. + /// + public ReferenceCountedDisposable? TryAddReference() where TTo : class, IDisposable + => TryAddReferenceImpl(_instance, _boxedReferenceCount, out _); + /// /// Increments the reference count for the disposable object, and returns a new disposable reference to it. /// @@ -179,6 +227,51 @@ namespace LibHac } } + /// + /// Provides the implementation for and + /// when casting the underlying object to another type. + /// + private static ReferenceCountedDisposable? TryAddReferenceImpl(TFrom? target, StrongBox referenceCount, + out CreateResult result) + where TFrom : class, IDisposable + where TTo : class, IDisposable + { + lock (referenceCount) + { + if (referenceCount.Value == 0) + { + // The target is already disposed, and cannot be reused + result = CreateResult.Disposed; + return null; + } + + if (target == null) + { + // The current reference has been disposed, so even though it isn't disposed yet we don't have a + // reference to the target + result = CreateResult.Disposed; + return null; + } + + if (!(target is TTo castedTarget)) + { + // The target cannot be casted to type TTo + result = CreateResult.NotCastable; + return null; + } + + checked + { + referenceCount.Value++; + } + + // Must return a new instance, in order for the Dispose operation on each individual instance to + // be idempotent. + result = CreateResult.Success; + return new ReferenceCountedDisposable(castedTarget, referenceCount); + } + } + /// /// Releases the current reference, causing the underlying object to be disposed if this was the last /// reference. @@ -212,6 +305,16 @@ namespace LibHac instanceToDispose?.Dispose(); } + /// + /// Represents the result of the TryAddReferenceImpl method. + /// + private enum CreateResult + { + Success, + Disposed, + NotCastable + } + /// /// Represents a weak reference to a which is capable of /// obtaining a new counted reference up until the point when the object is no longer accessible.