update cache system

优化了缓存系统初始化逻辑,支持分帧获取所有缓存文件。
This commit is contained in:
hevinci
2023-02-28 18:25:27 +08:00
parent 7c4dbb1f38
commit ebcb05cc55
8 changed files with 257 additions and 67 deletions
@@ -0,0 +1,168 @@
using System;
using System.IO;
using System.Collections;
using System.Collections.Generic;
namespace YooAsset
{
internal class FindCacheFilesOperation : AsyncOperationBase
{
private enum ESteps
{
None,
FindPrepare,
FindBundleFiles,
FindRawFiles,
Done,
}
private readonly string _packageName;
private float _verifyStartTime;
private IEnumerator<DirectoryInfo> _bundleFilesEnumerator = null;
private IEnumerator<DirectoryInfo> _rawFilesEnumerator = null;
private ESteps _steps = ESteps.None;
/// <summary>
/// 需要验证的元素
/// </summary>
public readonly List<VerifyElement> VerifyElements = new List<VerifyElement>(5000);
public FindCacheFilesOperation(string packageName)
{
_packageName = packageName;
}
internal override void Start()
{
_steps = ESteps.FindPrepare;
_verifyStartTime = UnityEngine.Time.realtimeSinceStartup;
}
internal override void Update()
{
if (_steps == ESteps.None || _steps == ESteps.Done)
return;
if (_steps == ESteps.FindPrepare)
{
// BundleFiles
{
string rootPath = PersistentHelper.GetCachedBundleFileFolderPath(_packageName);
DirectoryInfo rootDirectory = new DirectoryInfo(rootPath);
if (rootDirectory.Exists)
{
var directorieInfos = rootDirectory.EnumerateDirectories();
_bundleFilesEnumerator = directorieInfos.GetEnumerator();
}
}
// RawFiles
{
string rootPath = PersistentHelper.GetCachedRawFileFolderPath(_packageName);
DirectoryInfo rootDirectory = new DirectoryInfo(rootPath);
if (rootDirectory.Exists)
{
var directorieInfos = rootDirectory.EnumerateDirectories();
_rawFilesEnumerator = directorieInfos.GetEnumerator();
}
}
_steps = ESteps.FindBundleFiles;
}
if (_steps == ESteps.FindBundleFiles)
{
if (UpdateFindBundleFiles())
return;
_steps = ESteps.FindRawFiles;
}
if (_steps == ESteps.FindRawFiles)
{
if (UpdateFindRawFiles())
return;
// 注意:总是返回成功
_steps = ESteps.Done;
Status = EOperationStatus.Succeed;
float costTime = UnityEngine.Time.realtimeSinceStartup - _verifyStartTime;
YooLogger.Log($"Find cache files elapsed time {costTime:f1} seconds");
}
}
private bool UpdateFindBundleFiles()
{
if (_bundleFilesEnumerator == null)
return false;
bool isFindItem;
while (true)
{
isFindItem = _bundleFilesEnumerator.MoveNext();
if (isFindItem == false)
break;
var fileFoder = _bundleFilesEnumerator.Current;
string cacheGUID = fileFoder.Name;
if (CacheSystem.IsCached(_packageName, cacheGUID))
continue;
// 创建验证元素类
string fileRootPath = fileFoder.FullName;
string dataFilePath = $"{fileRootPath}/{ YooAssetSettings.CacheBundleDataFileName}";
string infoFilePath = $"{fileRootPath}/{ YooAssetSettings.CacheBundleInfoFileName}";
VerifyElement element = new VerifyElement(_packageName, cacheGUID, fileRootPath, dataFilePath, infoFilePath);
VerifyElements.Add(element);
if (OperationSystem.IsBusy)
break;
}
return isFindItem;
}
private bool UpdateFindRawFiles()
{
if (_rawFilesEnumerator == null)
return false;
bool isFindItem;
while (true)
{
isFindItem = _rawFilesEnumerator.MoveNext();
if (isFindItem == false)
break;
var fileFoder = _rawFilesEnumerator.Current;
string cacheGUID = fileFoder.Name;
if (CacheSystem.IsCached(_packageName, cacheGUID))
continue;
// 获取数据文件的后缀名
string dataFileExtension = string.Empty;
var fileInfos = fileFoder.GetFiles();
foreach (var fileInfo in fileInfos)
{
if (fileInfo.Extension == ".temp")
continue;
if (fileInfo.Name.StartsWith(YooAssetSettings.CacheBundleDataFileName))
{
dataFileExtension = fileInfo.Extension;
break;
}
}
// 创建验证元素类
string fileRootPath = fileFoder.FullName;
string dataFilePath = $"{fileRootPath}/{ YooAssetSettings.CacheBundleDataFileName}{dataFileExtension}";
string infoFilePath = $"{fileRootPath}/{ YooAssetSettings.CacheBundleInfoFileName}";
VerifyElement element = new VerifyElement(_packageName, cacheGUID, fileRootPath, dataFilePath, infoFilePath);
VerifyElements.Add(element);
if (OperationSystem.IsBusy)
break;
}
return isFindItem;
}
}
}
@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 00ec004354d75ac499606d6959192f9b
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:
@@ -0,0 +1,250 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Threading;
namespace YooAsset
{
internal abstract class VerifyCacheFilesOperation : AsyncOperationBase
{
public static VerifyCacheFilesOperation CreateOperation(List<VerifyElement> elements)
{
#if UNITY_WEBGL
var operation = new VerifyCacheFilesWithoutThreadOperation(elements);
#else
var operation = new VerifyCacheFilesWithThreadOperation(elements);
#endif
return operation;
}
}
/// <summary>
/// 本地缓存文件验证(线程版)
/// </summary>
internal class VerifyCacheFilesWithThreadOperation : VerifyCacheFilesOperation
{
private enum ESteps
{
None,
InitVerify,
UpdateVerify,
Done,
}
private readonly ThreadSyncContext _syncContext = new ThreadSyncContext();
private List<VerifyElement> _waitingList;
private List<VerifyElement> _verifyingList;
private int _verifyMaxNum;
private int _verifyTotalCount;
private float _verifyStartTime;
private int _succeedCount;
private int _failedCount;
private ESteps _steps = ESteps.None;
public VerifyCacheFilesWithThreadOperation(List<VerifyElement> elements)
{
_waitingList = elements;
}
internal override void Start()
{
_steps = ESteps.InitVerify;
_verifyStartTime = UnityEngine.Time.realtimeSinceStartup;
}
internal override void Update()
{
if (_steps == ESteps.None || _steps == ESteps.Done)
return;
if (_steps == ESteps.InitVerify)
{
int fileCount = _waitingList.Count;
// 设置同时验证的最大数
ThreadPool.GetMaxThreads(out int workerThreads, out int ioThreads);
YooLogger.Log($"Work threads : {workerThreads}, IO threads : {ioThreads}");
_verifyMaxNum = Math.Min(workerThreads, ioThreads);
_verifyTotalCount = fileCount;
if (_verifyMaxNum < 1)
_verifyMaxNum = 1;
_verifyingList = new List<VerifyElement>(_verifyMaxNum);
_steps = ESteps.UpdateVerify;
}
if (_steps == ESteps.UpdateVerify)
{
_syncContext.Update();
Progress = GetProgress();
if (_waitingList.Count == 0 && _verifyingList.Count == 0)
{
_steps = ESteps.Done;
Status = EOperationStatus.Succeed;
float costTime = UnityEngine.Time.realtimeSinceStartup - _verifyStartTime;
YooLogger.Log($"Verify cache files elapsed time {costTime:f1} seconds");
}
for (int i = _waitingList.Count - 1; i >= 0; i--)
{
if (OperationSystem.IsBusy)
break;
if (_verifyingList.Count >= _verifyMaxNum)
break;
var element = _waitingList[i];
if (BeginVerifyFileWithThread(element))
{
_waitingList.RemoveAt(i);
_verifyingList.Add(element);
}
else
{
YooLogger.Warning("The thread pool is failed queued.");
break;
}
}
}
}
private float GetProgress()
{
if (_verifyTotalCount == 0)
return 1f;
return (float)(_succeedCount + _failedCount) / _verifyTotalCount;
}
private bool BeginVerifyFileWithThread(VerifyElement element)
{
return ThreadPool.QueueUserWorkItem(new WaitCallback(VerifyInThread), element);
}
private void VerifyInThread(object obj)
{
VerifyElement element = (VerifyElement)obj;
element.Result = CacheSystem.VerifyingCacheFile(element);
_syncContext.Post(VerifyCallback, element);
}
private void VerifyCallback(object obj)
{
VerifyElement element = (VerifyElement)obj;
_verifyingList.Remove(element);
if (element.Result == EVerifyResult.Succeed)
{
_succeedCount++;
var wrapper = new PackageCache.RecordWrapper(element.InfoFilePath, element.DataFilePath, element.DataFileCRC, element.DataFileSize);
CacheSystem.RecordFile(element.PackageName, element.CacheGUID, wrapper);
}
else
{
_failedCount++;
YooLogger.Warning($"Failed verify file and delete files : {element.FileRootPath}");
element.DeleteFiles();
}
}
}
/// <summary>
/// 本地缓存文件验证(非线程版)
/// </summary>
internal class VerifyCacheFilesWithoutThreadOperation : VerifyCacheFilesOperation
{
private enum ESteps
{
None,
InitVerify,
UpdateVerify,
Done,
}
private List<VerifyElement> _waitingList;
private List<VerifyElement> _verifyingList;
private int _verifyMaxNum;
private int _verifyTotalCount;
private float _verifyStartTime;
private int _succeedCount;
private int _failedCount;
private ESteps _steps = ESteps.None;
public VerifyCacheFilesWithoutThreadOperation(List<VerifyElement> elements)
{
_waitingList = elements;
}
internal override void Start()
{
_steps = ESteps.InitVerify;
_verifyStartTime = UnityEngine.Time.realtimeSinceStartup;
}
internal override void Update()
{
if (_steps == ESteps.None || _steps == ESteps.Done)
return;
if (_steps == ESteps.InitVerify)
{
int fileCount = _waitingList.Count;
// 设置同时验证的最大数
_verifyMaxNum = fileCount;
_verifyTotalCount = fileCount;
_verifyingList = new List<VerifyElement>(_verifyMaxNum);
_steps = ESteps.UpdateVerify;
}
if (_steps == ESteps.UpdateVerify)
{
Progress = GetProgress();
if (_waitingList.Count == 0 && _verifyingList.Count == 0)
{
_steps = ESteps.Done;
Status = EOperationStatus.Succeed;
float costTime = UnityEngine.Time.realtimeSinceStartup - _verifyStartTime;
YooLogger.Log($"Package verify elapsed time {costTime:f1} seconds");
}
for (int i = _waitingList.Count - 1; i >= 0; i--)
{
if (OperationSystem.IsBusy)
break;
if (_verifyingList.Count >= _verifyMaxNum)
break;
var element = _waitingList[i];
BeginVerifyFileWithoutThread(element);
_waitingList.RemoveAt(i);
_verifyingList.Add(element);
}
// 主线程内验证,可以清空列表
_verifyingList.Clear();
}
}
private float GetProgress()
{
if (_verifyTotalCount == 0)
return 1f;
return (float)(_succeedCount + _failedCount) / _verifyTotalCount;
}
private void BeginVerifyFileWithoutThread(VerifyElement element)
{
element.Result = CacheSystem.VerifyingCacheFile(element);
if (element.Result == EVerifyResult.Succeed)
{
_succeedCount++;
var wrapper = new PackageCache.RecordWrapper(element.InfoFilePath, element.DataFilePath, element.DataFileCRC, element.DataFileSize);
CacheSystem.RecordFile(element.PackageName, element.CacheGUID, wrapper);
}
else
{
_failedCount++;
YooLogger.Warning($"Failed verify file and delete files : {element.FileRootPath}");
element.DeleteFiles();
}
}
}
}
@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 868188d19436b674b8c403d2d90afeaf
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant: