Add BlockCacheManager and skeleton CompressedStorage

This commit is contained in:
Alex Barney 2021-12-15 01:17:56 -07:00
parent 527d81a3b2
commit 634ab59742
5 changed files with 775 additions and 1 deletions

View File

@ -0,0 +1,76 @@
using System;
using LibHac.Common;
using LibHac.Diag;
using LibHac.Util;
namespace LibHac.FsSystem;
public interface IAsynchronousAccessSplitter : IDisposable
{
private static readonly DefaultAsynchronousAccessSplitter DefaultAccessSplitter = new();
public static IAsynchronousAccessSplitter GetDefaultAsynchronousAccessSplitter()
{
return DefaultAccessSplitter;
}
Result QueryNextOffset(out long nextOffset, long startOffset, long endOffset, long accessSize, long alignmentSize)
{
UnsafeHelpers.SkipParamInit(out nextOffset);
Assert.SdkRequiresLess(0, accessSize);
Assert.SdkRequiresLess(0, alignmentSize);
if (endOffset - startOffset <= accessSize)
{
nextOffset = endOffset;
return Result.Success;
}
Result rc = QueryAppropriateOffset(out long offsetAppropriate, startOffset, accessSize, alignmentSize);
if (rc.IsFailure()) return rc.Miss();
Assert.SdkNotEqual(startOffset, offsetAppropriate);
nextOffset = Math.Min(startOffset, offsetAppropriate);
return Result.Success;
}
Result QueryInvocationCount(out long count, long startOffset, long endOffset, long accessSize, long alignmentSize)
{
UnsafeHelpers.SkipParamInit(out count);
long invocationCount = 0;
long currentOffset = startOffset;
while (currentOffset < endOffset)
{
Result rc = QueryNextOffset(out currentOffset, currentOffset, endOffset, accessSize, alignmentSize);
if (rc.IsFailure()) return rc.Miss();
invocationCount++;
}
count = invocationCount;
return Result.Success;
}
Result QueryAppropriateOffset(out long offsetAppropriate, long startOffset, long accessSize, long alignmentSize);
}
public class DefaultAsynchronousAccessSplitter : IAsynchronousAccessSplitter
{
public void Dispose() { }
public Result QueryAppropriateOffset(out long offsetAppropriate, long startOffset, long accessSize, long alignmentSize)
{
offsetAppropriate = Alignment.AlignDownPow2(startOffset + accessSize, alignmentSize);
return Result.Success;
}
public Result QueryInvocationCount(out long count, long startOffset, long endOffset, long accessSize, long alignmentSize)
{
long alignedStartOffset = Alignment.AlignDownPow2(startOffset, alignmentSize);
count = BitUtil.DivideUp(endOffset - alignedStartOffset, accessSize);
return Result.Success;
}
}

View File

@ -0,0 +1,389 @@
using System;
using System.Runtime.CompilerServices;
using LibHac.Diag;
using LibHac.Fs;
using LibHac.FsSystem.Impl;
using LibHac.Os;
using Buffer = LibHac.Mem.Buffer;
namespace LibHac.FsSystem;
public class CompressedStorage : IStorage, IAsynchronousAccessSplitter
{
public delegate Result DecompressorFunction(Span<byte> destination, ReadOnlySpan<byte> source);
public delegate DecompressorFunction GetDecompressorFunction(CompressionType type);
public class CompressedStorageCore : IDisposable
{
private long _blockSizeMax;
private long _continuousReadingSizeMax;
private readonly BucketTree _bucketTree;
private ValueSubStorage _dataStorage;
private GetDecompressorFunction _getDecompressorFunction;
public CompressedStorageCore()
{
_bucketTree = new BucketTree();
_dataStorage = new ValueSubStorage();
}
public void Dispose()
{
FinalizeObject();
_dataStorage.Dispose();
_bucketTree.Dispose();
}
private bool IsInitialized()
{
return _bucketTree.IsInitialized();
}
public Result GetSize(out long size)
{
Assert.SdkRequiresNotNullOut(out size);
Result rc = _bucketTree.GetOffsets(out BucketTree.Offsets offsets);
if (rc.IsFailure()) return rc.Miss();
size = offsets.EndOffset;
return Result.Success;
}
public delegate Result OperatePerEntryFunc(out bool isContinuous, in Entry entry, long virtualDataSize,
long offsetInEntry, long readSize);
public Result OperatePerEntry(long offset, long size, OperatePerEntryFunc func)
{
throw new NotImplementedException();
}
public delegate Result OperateEntryFunc(long offset, long size);
public Result OperateEntry(long offset, long size, OperatePerEntryFunc func)
{
throw new NotImplementedException();
}
public Result Initialize(MemoryResource allocatorForBucketTree, in ValueSubStorage dataStorage,
in ValueSubStorage nodeStorage, in ValueSubStorage entryStorage, int bucketTreeEntryCount,
long blockSizeMax, long continuousReadingSizeMax, GetDecompressorFunction getDecompressorFunc)
{
Assert.SdkRequiresNotNull(allocatorForBucketTree);
Assert.SdkRequiresLess(0, blockSizeMax);
Assert.SdkRequiresLessEqual(blockSizeMax, continuousReadingSizeMax);
Assert.SdkRequiresNotNull(getDecompressorFunc);
Result rc = _bucketTree.Initialize(allocatorForBucketTree, in nodeStorage, in entryStorage, NodeSize,
Unsafe.SizeOf<Entry>(), bucketTreeEntryCount);
if (rc.IsFailure()) return rc.Miss();
_blockSizeMax = blockSizeMax;
_continuousReadingSizeMax = continuousReadingSizeMax;
_dataStorage.Set(in dataStorage);
_getDecompressorFunction = getDecompressorFunc;
return Result.Success;
}
public void FinalizeObject()
{
if (IsInitialized())
{
_bucketTree.FinalizeObject();
using var temp = new ValueSubStorage();
_dataStorage.Set(in temp);
}
}
public delegate Result ReadImplFunc(Span<byte> buffer);
public delegate Result ReadFunc(long sizeBufferRequired, ReadImplFunc readImplFunc);
public Result Read(long offset, long size, ReadFunc func)
{
throw new NotImplementedException();
}
public Result Invalidate()
{
throw new NotImplementedException();
}
public Result QueryRange(Span<byte> buffer, long offset, long size)
{
throw new NotImplementedException();
}
public Result QueryAppropriateOffsetForAsynchronousAccess(out long offsetAppropriate, long offset,
long accessSize, long alignmentSize)
{
throw new NotImplementedException();
}
private DecompressorFunction GetDecompressor(CompressionType type)
{
if (CompressionTypeUtility.IsUnknownType(type))
return null;
return _getDecompressorFunction(type);
}
}
public class CacheManager : IDisposable
{
public struct CacheEntry : IBlockCacheManagerEntry<Range>
{
public Range Range { get; set; }
public long Handle { get; set; }
public Buffer Buffer { get; set; }
public bool IsValid { get; set; }
public bool IsCached { get; set; }
public short Age { get; set; }
public void Invalidate() { /* empty */ }
public readonly bool IsAllocated() => IsValid && Handle != 0;
public bool IsWriteBack
{
get => false;
set { }
}
public bool IsFlushing
{
set { }
}
}
public struct Range : IBlockCacheManagerRange
{
public long Offset { get; set; }
public uint Size { get; set; }
public long GetEndOffset() => Offset + Size;
public bool IsIncluded(long offset) => Offset <= offset && offset < GetEndOffset();
}
public struct AccessRange
{
public long VirtualOffset;
public long VirtualSize;
public uint PhysicalSize;
public bool IsBlockAlignmentRequired;
public readonly long GetEndVirtualOffset() => VirtualOffset + VirtualSize;
}
private long _cacheSize0;
private long _cacheSize1;
private SdkMutexType _mutex;
private BlockCacheManager<CacheEntry, Range> _cacheManager;
private long _storageSize;
public CacheManager()
{
_mutex = new SdkMutexType();
_cacheManager = new BlockCacheManager<CacheEntry, Range>();
_storageSize = 0;
}
public void Dispose()
{
throw new NotImplementedException();
}
public Result Initialize(IBufferManager allocator, long storageSize, long cacheSize0, long cacheSize1,
int maxCacheEntries)
{
Result rc = _cacheManager.Initialize(allocator, maxCacheEntries);
if (rc.IsFailure()) return rc.Miss();
_storageSize = storageSize;
_cacheSize0 = cacheSize0;
_cacheSize1 = cacheSize1;
return Result.Success;
}
public void FinalizeObject()
{
throw new NotImplementedException();
}
public void Invalidate()
{
throw new NotImplementedException();
}
public Result Read(CompressedStorageCore core, long offset, Span<byte> buffer)
{
throw new NotImplementedException();
}
private Result FindBufferImpl(out Buffer buffer, out CacheEntry entry, long offset)
{
throw new NotImplementedException();
}
private Result FindBuffer(out Buffer buffer, out CacheEntry entry, long offset)
{
throw new NotImplementedException();
}
private Result FindOrAllocateBuffer(out Buffer buffer, out CacheEntry entry, long offset, ulong size)
{
throw new NotImplementedException();
}
private void StoreAssociateBuffer(Buffer buffer, in CacheEntry entry)
{
throw new NotImplementedException();
}
private Result ReadHeadCache(CompressedStorageCore core, ref long offset, ref Span<byte> buffer,
ref AccessRange headRange, in AccessRange endRange)
{
throw new NotImplementedException();
}
private Result ReadTailCache(CompressedStorageCore core, long offset, Span<byte> buffer,
in AccessRange headRange, ref AccessRange endRange)
{
throw new NotImplementedException();
}
}
public struct Entry
{
public long VirtualOffset;
public long PhysicalOffset;
public CompressionType CompressionType;
public uint PhysicalSize;
public readonly long GetPhysicalSize() => PhysicalSize;
}
public static readonly int NodeSize = 0x4000;
private CompressedStorageCore _core;
private CacheManager _cacheManager;
public static long QueryEntryStorageSize(int entryCount)
{
return BucketTree.QueryEntryStorageSize(NodeSize, Unsafe.SizeOf<Entry>(), entryCount);
}
public static long QueryNodeStorageSize(int entryCount)
{
return BucketTree.QueryNodeStorageSize(NodeSize, Unsafe.SizeOf<Entry>(), entryCount);
}
public CompressedStorage()
{
_core = new CompressedStorageCore();
_cacheManager = new CacheManager();
}
public override void Dispose()
{
FinalizeObject();
_cacheManager.Dispose();
_core.Dispose();
}
public Result Initialize(MemoryResource allocatorForBucketTree, IBufferManager allocatorForCacheManager,
in ValueSubStorage dataStorage, in ValueSubStorage nodeStorage, in ValueSubStorage entryStorage,
int bucketTreeEntryCount, long blockSizeMax, long continuousReadingSizeMax,
GetDecompressorFunction getDecompressorFunc, long cacheSize0, long cacheSize1, int maxCacheEntries)
{
Result rc = _core.Initialize(allocatorForBucketTree, in dataStorage, in nodeStorage, in entryStorage,
bucketTreeEntryCount, blockSizeMax, continuousReadingSizeMax, getDecompressorFunc);
if (rc.IsFailure()) return rc.Miss();
rc = _core.GetSize(out long size);
if (rc.IsFailure()) return rc.Miss();
rc = _cacheManager.Initialize(allocatorForCacheManager, size, cacheSize0, cacheSize1, maxCacheEntries);
if (rc.IsFailure()) return rc.Miss();
return Result.Success;
}
public void FinalizeObject()
{
_cacheManager.FinalizeObject();
_core.FinalizeObject();
}
protected override Result DoRead(long offset, Span<byte> destination)
{
Result rc = _cacheManager.Read(_core, offset, destination);
if (rc.IsFailure()) return rc.Miss();
return Result.Success;
}
protected override Result DoWrite(long offset, ReadOnlySpan<byte> source)
{
return ResultFs.UnsupportedWriteForCompressedStorage.Log();
}
protected override Result DoFlush()
{
return Result.Success;
}
protected override Result DoSetSize(long size)
{
return ResultFs.UnsupportedSetSizeForIndirectStorage.Log();
}
protected override Result DoGetSize(out long size)
{
Result rc = _core.GetSize(out size);
if (rc.IsFailure()) return rc.Miss();
return Result.Success;
}
protected override Result DoOperateRange(Span<byte> outBuffer, OperationId operationId, long offset, long size,
ReadOnlySpan<byte> inBuffer)
{
Assert.SdkRequiresLessEqual(0, offset);
Assert.SdkRequiresLessEqual(0, size);
switch (operationId)
{
case OperationId.InvalidateCache:
{
_cacheManager.Invalidate();
Result rc = _core.Invalidate();
if (rc.IsFailure()) return rc.Miss();
break;
}
case OperationId.QueryRange:
{
Result rc = _core.QueryRange(outBuffer, offset, size);
if (rc.IsFailure()) return rc.Miss();
break;
}
default:
return ResultFs.UnsupportedOperateRangeForCompressedStorage.Log();
}
return Result.Success;
}
public Result QueryAppropriateOffset(out long offsetAppropriate, long startOffset, long accessSize,
long alignmentSize)
{
Result rc = _core.QueryAppropriateOffsetForAsynchronousAccess(out offsetAppropriate, startOffset, accessSize,
alignmentSize);
if (rc.IsFailure()) return rc.Miss();
return Result.Success;
}
}

View File

@ -0,0 +1,32 @@
namespace LibHac.FsSystem;
public enum CompressionType
{
None = 0,
Zeroed = 1,
Lz4 = 3,
Unknown = 4
}
public static class CompressionTypeUtility
{
public static bool IsBlockAlignmentRequired(CompressionType type)
{
return type != CompressionType.None && type != CompressionType.Zeroed;
}
public static bool IsDataStorageAccessRequired(CompressionType type)
{
return type != CompressionType.Zeroed;
}
public static bool IsRandomAccessible(CompressionType type)
{
return type == CompressionType.None;
}
public static bool IsUnknownType(CompressionType type)
{
return type >= CompressionType.Unknown;
}
}

View File

@ -0,0 +1,276 @@
using System;
using LibHac.Diag;
using LibHac.Fs;
using Buffer = LibHac.Mem.Buffer;
namespace LibHac.FsSystem.Impl;
public interface IBlockCacheManagerEntry<TRange> where TRange : struct, IBlockCacheManagerRange
{
TRange Range { get; }
bool IsValid { get; set; }
bool IsWriteBack { get; set; }
bool IsCached { get; set; }
bool IsFlushing { set; }
long Handle { get; set; }
Buffer Buffer { get; set; }
short Age { get; set; }
void Invalidate();
bool IsAllocated();
}
public interface IBlockCacheManagerRange
{
long Offset { get; }
long GetEndOffset();
}
public class BlockCacheManager<TEntry, TRange> : IDisposable
where TEntry : struct, IBlockCacheManagerEntry<TRange>
where TRange : struct, IBlockCacheManagerRange
{
private IBufferManager _allocator;
private TEntry[] _cacheEntries;
private int _cacheEntriesCount;
public void Dispose()
{
}
public Result Initialize(IBufferManager allocator, int maxCacheEntries)
{
Assert.SdkRequiresNull(_allocator);
Assert.SdkRequiresNull(_cacheEntries);
Assert.SdkRequiresNotNull(allocator);
if (maxCacheEntries > 0)
{
_cacheEntries = new TEntry[maxCacheEntries];
}
_allocator = allocator;
_cacheEntriesCount = maxCacheEntries;
return Result.Success;
}
public void FinalizeObject()
{
_allocator = null;
_cacheEntries = null;
_cacheEntriesCount = 0;
}
public ref readonly TEntry this[int index]
{
get
{
Assert.SdkRequires(IsInitialized());
Assert.SdkRequiresInRange(index, 0, _cacheEntriesCount);
return ref _cacheEntries[index];
}
}
public bool IsInitialized() => _allocator is not null;
public IBufferManager GetAllocator() => _allocator;
public int GetCount() => _cacheEntriesCount;
public void GetEmptyCacheIndex(out int emptyIndex, out int leastRecentlyUsedIndex)
{
int empty = -1;
int leastRecentlyUsed = -1;
for (int i = 0; i < GetCount(); i++)
{
if (!_cacheEntries[i].IsValid)
{
// Get the first empty cache index
if (empty < 0)
empty = i;
}
else
{
// Protect against overflow
if (_cacheEntries[i].Age != short.MaxValue)
{
_cacheEntries[i].Age++;
}
// Get the cache index that was least recently used
if (leastRecentlyUsed < 0 || _cacheEntries[leastRecentlyUsed].Age < _cacheEntries[i].Age)
{
leastRecentlyUsed = i;
}
}
}
emptyIndex = empty;
leastRecentlyUsedIndex = leastRecentlyUsed;
}
public void InvalidateCacheEntry(int index)
{
Assert.SdkRequires(IsInitialized());
Assert.SdkRequiresLess(index, GetCount());
ref TEntry entry = ref _cacheEntries[index];
Assert.SdkAssert(entry.IsValid);
if (entry.IsWriteBack)
{
Assert.SdkAssert(!entry.Buffer.IsNull && entry.Handle == 0);
_allocator.DeallocateBuffer(entry.Buffer);
entry.Buffer = new Buffer();
}
else
{
Assert.SdkAssert(entry.Buffer.IsNull && entry.Handle != 0);
Buffer buffer = _allocator.AcquireCache(entry.Handle);
if (!buffer.IsNull)
_allocator.DeallocateBuffer(buffer);
}
entry.IsValid = false;
entry.Invalidate();
}
public void Invalidate()
{
Assert.SdkRequires(IsInitialized());
int count = _cacheEntriesCount;
for (int i = 0; i < count; i++)
{
if (_cacheEntries[i].IsValid)
InvalidateCacheEntry(i);
}
}
public void ReleaseCacheEntry(int index, Buffer buffer)
{
ReleaseCacheEntry(ref _cacheEntries[index], buffer);
}
public void ReleaseCacheEntry(ref TEntry entry, Buffer buffer)
{
Assert.SdkRequires(IsInitialized());
_allocator.DeallocateBuffer(buffer);
entry.IsValid = false;
entry.IsCached = false;
}
public void RegisterCacheEntry(int index, Buffer buffer, IBufferManager.BufferAttribute attribute)
{
Assert.SdkRequires(IsInitialized());
ref TEntry entry = ref _cacheEntries[index];
if (entry.IsWriteBack)
{
entry.Handle = 0;
entry.Buffer = buffer;
}
else
{
entry.Handle = _allocator.RegisterCache(buffer, attribute);
entry.Buffer = new Buffer();
}
}
public void AcquireCacheEntry(out TEntry outEntry, out Buffer outBuffer, int index)
{
Assert.SdkRequires(IsInitialized());
Assert.SdkRequiresLess(index, GetCount());
ref TEntry entry = ref _cacheEntries[index];
if (entry.IsWriteBack)
{
outBuffer = entry.Buffer;
}
else
{
outBuffer = _allocator.AcquireCache(entry.Handle);
}
outEntry = entry;
Assert.SdkAssert(outEntry.IsValid);
Assert.SdkAssert(outEntry.IsCached);
entry.IsValid = false;
entry.Handle = 0;
entry.Buffer = new Buffer();
entry.Age = 0;
outEntry.IsValid = true;
outEntry.Handle = 0;
outEntry.Buffer = new Buffer();
outEntry.Age = 0;
}
private bool ExistsRedundantCacheEntry(in TEntry entry)
{
Assert.SdkRequires(IsInitialized());
for (int i = 0; i < GetCount(); i++)
{
ref TEntry currentEntry = ref _cacheEntries[i];
if (currentEntry.IsAllocated() &&
currentEntry.Range.Offset < entry.Range.GetEndOffset() &&
entry.Range.Offset < currentEntry.Range.GetEndOffset())
{
return true;
}
}
return false;
}
public bool SetCacheEntry(int index, in TEntry entry, Buffer buffer)
{
return SetCacheEntry(index, in entry, buffer, new IBufferManager.BufferAttribute());
}
public bool SetCacheEntry(int index, in TEntry entry, Buffer buffer, IBufferManager.BufferAttribute attribute)
{
Assert.SdkRequires(IsInitialized());
Assert.SdkRequiresInRange(index, 0, _cacheEntriesCount);
_cacheEntries[index] = entry;
Assert.SdkAssert(entry.IsValid);
Assert.SdkAssert(entry.IsCached);
Assert.SdkAssert(entry.Handle == 0);
Assert.SdkAssert(entry.Buffer.IsNull);
// Get rid of the input entry if it overlaps with anything currently in the cache
if (ExistsRedundantCacheEntry(in entry))
{
ReleaseCacheEntry(index, buffer);
return false;
}
RegisterCacheEntry(index, buffer, attribute);
return true;
}
public void SetFlushing(int index, bool isFlushing)
{
_cacheEntries[index].IsFlushing = isFlushing;
}
public void SetWriteBack(int index, bool isWriteBack)
{
_cacheEntries[index].IsWriteBack = isWriteBack;
}
}

View File

@ -47,6 +47,7 @@ public static class Alignment
public static long AlignUpPow2(long value, uint alignment) => (long)AlignUpPow2((ulong)value, alignment);
public static int AlignDownPow2(int value, uint alignment) => (int)AlignDownPow2((ulong)value, alignment);
public static long AlignDownPow2(long value, uint alignment) => (long)AlignDownPow2((ulong)value, alignment);
public static long AlignDownPow2(long value, long alignment) => (long)AlignDownPow2((ulong)value, (uint)alignment);
public static bool IsAlignedPow2(int value, uint alignment) => IsAlignedPow2((ulong)value, alignment);
public static bool IsAlignedPow2(long value, uint alignment) => IsAlignedPow2((ulong)value, alignment);
@ -60,4 +61,4 @@ public static class Alignment
public static long AlignDown(long value, uint alignment) => (long)AlignDown((ulong)value, alignment);
public static bool IsAligned(int value, uint alignment) => IsAligned((ulong)value, alignment);
public static bool IsAligned(long value, uint alignment) => IsAligned((ulong)value, alignment);
}
}