Update CacheSystem

This commit is contained in:
hevinci
2022-08-06 16:23:38 +08:00
parent dcd4475617
commit a1ad7acb3d
16 changed files with 202 additions and 131 deletions
@@ -14,18 +14,15 @@ namespace YooAsset
{
private static readonly Dictionary<string, DownloaderBase> _downloaderDic = new Dictionary<string, DownloaderBase>();
private static readonly List<string> _removeList = new List<string>(100);
private static readonly Dictionary<string, string> _cachedHashList = new Dictionary<string, string>(1000);
private static int _breakpointResumeFileSize = int.MaxValue;
private static EVerifyLevel _verifyLevel = EVerifyLevel.High;
/// <summary>
/// 初始化
/// </summary>
public static void Initialize(int breakpointResumeFileSize, EVerifyLevel verifyLevel)
public static void Initialize(int breakpointResumeFileSize)
{
_breakpointResumeFileSize = breakpointResumeFileSize;
_verifyLevel = verifyLevel;
}
/// <summary>
@@ -62,7 +59,6 @@ namespace YooAsset
}
_downloaderDic.Clear();
_removeList.Clear();
_cachedHashList.Clear();
_breakpointResumeFileSize = int.MaxValue;
}
@@ -80,7 +76,7 @@ namespace YooAsset
}
// 如果资源已经缓存
if (ContainsVerifyFile(bundleInfo.FileHash))
if (CacheSystem.ContainsVerifyFile(bundleInfo.FileHash))
{
var tempDownloader = new TempDownloader(bundleInfo);
return tempDownloader;
@@ -108,84 +104,5 @@ namespace YooAsset
{
return _downloaderDic.Count;
}
/// <summary>
/// 查询是否为验证文件
/// 注意:被收录的文件完整性是绝对有效的
/// </summary>
public static bool ContainsVerifyFile(string fileHash)
{
if (_cachedHashList.ContainsKey(fileHash))
{
string fileName = _cachedHashList[fileHash];
string filePath = SandboxHelper.MakeCacheFilePath(fileName);
if (File.Exists(filePath))
{
return true;
}
else
{
_cachedHashList.Remove(fileHash);
YooLogger.Error($"Cache file is missing : {fileName}");
return false;
}
}
else
{
return false;
}
}
/// <summary>
/// 缓存验证过的文件
/// </summary>
public static void CacheVerifyFile(string fileHash, string fileName)
{
if (_cachedHashList.ContainsKey(fileHash) == false)
{
YooLogger.Log($"Cache verify file : {fileName}");
_cachedHashList.Add(fileHash, fileName);
}
}
/// <summary>
/// 验证文件完整性
/// </summary>
public static bool CheckContentIntegrity(string filePath, long fileSize, string fileCRC)
{
return CheckContentIntegrity(_verifyLevel, filePath, fileSize, fileCRC);
}
/// <summary>
/// 验证文件完整性
/// </summary>
public static bool CheckContentIntegrity(EVerifyLevel verifyLevel, string filePath, long fileSize, string fileCRC)
{
try
{
if (File.Exists(filePath) == false)
return false;
// 先验证文件大小
long size = FileUtility.GetFileSize(filePath);
if (size != fileSize)
return false;
// 再验证文件CRC
if (verifyLevel == EVerifyLevel.High)
{
string crc = HashUtility.FileCRC32(filePath);
return crc == fileCRC;
}
else
{
return true;
}
}
catch (Exception)
{
return false;
}
}
}
}
@@ -81,7 +81,7 @@ namespace YooAsset
if (hasError == false)
{
// 注意:如果文件验证失败需要删除文件
if (DownloadSystem.CheckContentIntegrity(_bundleInfo.GetCacheLoadPath(), _bundleInfo.FileSize, _bundleInfo.FileCRC) == false)
if (CacheSystem.CheckContentIntegrity(_bundleInfo.GetCacheLoadPath(), _bundleInfo.FileSize, _bundleInfo.FileCRC) == false)
{
hasError = true;
_lastError = $"Verification failed";
@@ -91,7 +91,7 @@ namespace YooAsset
if (hasError == false)
{
_steps = ESteps.Succeed;
DownloadSystem.CacheVerifyFile(_bundleInfo.FileHash, _bundleInfo.FileName);
CacheSystem.CacheVerifyFile(_bundleInfo.FileHash, _bundleInfo.FileName);
}
else
{
@@ -156,7 +156,7 @@ namespace YooAsset
// 验证下载文件完整性
if (DownloadedBytes == (ulong)_fileSize)
{
bool verfiyResult = DownloadSystem.CheckContentIntegrity(_savePath, _fileSize, _fileCRC);
bool verfiyResult = CacheSystem.CheckContentIntegrity(_savePath, _fileSize, _fileCRC);
if (verfiyResult == false)
{
Error = $"Verify download content failed : {_fileName}";
@@ -226,7 +226,7 @@ namespace YooAsset
}
else
{
DownloadSystem.CacheVerifyFile(_bundleInfo.FileHash, _bundleInfo.FileName);
CacheSystem.CacheVerifyFile(_bundleInfo.FileHash, _bundleInfo.FileName);
_steps = ESteps.Succeed;
}
}
@@ -1,19 +0,0 @@
namespace YooAsset
{
/// <summary>
/// 下载文件校验等级
/// </summary>
public enum EVerifyLevel
{
/// <summary>
/// 验证文件大小
/// </summary>
Low,
/// <summary>
/// 验证文件大小和CRC
/// </summary>
High,
}
}
@@ -1,11 +0,0 @@
fileFormatVersion: 2
guid: 02ad351eb8539da47a0c789e2f8c468f
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:
@@ -1,257 +0,0 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Threading;
namespace YooAsset
{
/// <summary>
/// 本地缓存文件验证器
/// </summary>
internal abstract class PatchCacheVerifier
{
public abstract bool InitVerifier(PatchManifest appPatchManifest, PatchManifest localPatchManifest, bool weaklyUpdate);
public abstract bool UpdateVerifier();
public abstract float GetVerifierProgress();
public int VerifySuccessCount { protected set; get; } = 0;
public int VerifyFailCount { protected set; get; } = 0;
}
/// <summary>
/// 本地缓存文件验证器(线程版)
/// </summary>
internal class PatchCacheVerifierWithThread : PatchCacheVerifier
{
private class ThreadInfo
{
public bool Result = false;
public string FilePath { private set; get; }
public PatchBundle Bundle { private set; get; }
public ThreadInfo(string filePath, PatchBundle bundle)
{
FilePath = filePath;
Bundle = bundle;
}
}
private readonly ThreadSyncContext _syncContext = new ThreadSyncContext();
private readonly List<PatchBundle> _waitingList = new List<PatchBundle>(1000);
private readonly List<PatchBundle> _verifyingList = new List<PatchBundle>(100);
private int _verifyMaxNum;
private int _verifyTotalCount;
public override bool InitVerifier(PatchManifest appPatchManifest, PatchManifest localPatchManifest, bool weaklyUpdate)
{
// 遍历所有文件然后验证并缓存合法文件
foreach (var patchBundle in localPatchManifest.BundleList)
{
// 忽略缓存文件
if (DownloadSystem.ContainsVerifyFile(patchBundle.FileHash))
continue;
// 忽略APP资源
// 注意:如果是APP资源并且哈希值相同,则不需要下载
if (appPatchManifest.TryGetPatchBundle(patchBundle.BundleName, out PatchBundle appPatchBundle))
{
if (appPatchBundle.IsBuildin && appPatchBundle.Equals(patchBundle))
continue;
}
// 注意:在弱联网模式下,我们需要验证指定资源版本的所有资源完整性
if (weaklyUpdate)
{
string filePath = SandboxHelper.MakeCacheFilePath(patchBundle.FileName);
if (File.Exists(filePath))
_waitingList.Add(patchBundle);
else
return false;
}
else
{
string filePath = SandboxHelper.MakeCacheFilePath(patchBundle.FileName);
if (File.Exists(filePath))
_waitingList.Add(patchBundle);
}
}
// 设置同时验证的最大数
ThreadPool.GetMaxThreads(out int workerThreads, out int ioThreads);
YooLogger.Log($"Work threads : {workerThreads}, IO threads : {ioThreads}");
_verifyMaxNum = Math.Min(workerThreads, ioThreads);
_verifyTotalCount = _waitingList.Count;
if (_verifyMaxNum < 1)
_verifyMaxNum = 1;
return true;
}
public override bool UpdateVerifier()
{
_syncContext.Update();
if (_waitingList.Count == 0 && _verifyingList.Count == 0)
return true;
if (_verifyingList.Count >= _verifyMaxNum)
return false;
for (int i = _waitingList.Count - 1; i >= 0; i--)
{
if (_verifyingList.Count >= _verifyMaxNum)
break;
var patchBundle = _waitingList[i];
if (VerifyFile(patchBundle))
{
_waitingList.RemoveAt(i);
_verifyingList.Add(patchBundle);
}
else
{
YooLogger.Warning("The thread pool is failed queued.");
break;
}
}
return false;
}
public override float GetVerifierProgress()
{
if (_verifyTotalCount == 0)
return 1f;
return (float)(VerifySuccessCount + VerifyFailCount) / _verifyTotalCount;
}
private bool VerifyFile(PatchBundle patchBundle)
{
string filePath = SandboxHelper.MakeCacheFilePath(patchBundle.FileName);
ThreadInfo info = new ThreadInfo(filePath, patchBundle);
return ThreadPool.QueueUserWorkItem(new WaitCallback(VerifyInThread), info);
}
private void VerifyInThread(object infoObj)
{
ThreadInfo info = (ThreadInfo)infoObj;
info.Result = DownloadSystem.CheckContentIntegrity(info.FilePath, info.Bundle.FileSize, info.Bundle.FileCRC);
_syncContext.Post(VerifyCallback, info);
}
private void VerifyCallback(object obj)
{
ThreadInfo info = (ThreadInfo)obj;
if (info.Result)
{
VerifySuccessCount++;
DownloadSystem.CacheVerifyFile(info.Bundle.FileHash, info.Bundle.FileName);
}
else
{
VerifyFailCount++;
// NOTE:不期望删除断点续传的资源文件
/*
YooLogger.Warning($"Failed to verify file : {info.FilePath}");
if (File.Exists(info.FilePath))
File.Delete(info.FilePath);
*/
}
_verifyingList.Remove(info.Bundle);
}
}
/// <summary>
/// 本地缓存文件验证器(非线程版)
/// </summary>
internal class PatchCacheVerifierWithoutThread : PatchCacheVerifier
{
private readonly List<PatchBundle> _waitingList = new List<PatchBundle>(1000);
private readonly List<PatchBundle> _verifyingList = new List<PatchBundle>(100);
private int _verifyMaxNum;
private int _verifyTotalCount;
public override bool InitVerifier(PatchManifest appPatchManifest, PatchManifest localPatchManifest, bool weaklyUpdate)
{
// 遍历所有文件然后验证并缓存合法文件
foreach (var patchBundle in localPatchManifest.BundleList)
{
// 忽略缓存文件
if (DownloadSystem.ContainsVerifyFile(patchBundle.FileHash))
continue;
// 忽略APP资源
// 注意:如果是APP资源并且哈希值相同,则不需要下载
if (appPatchManifest.TryGetPatchBundle(patchBundle.BundleName, out PatchBundle appPatchBundle))
{
if (appPatchBundle.IsBuildin && appPatchBundle.Equals(patchBundle))
continue;
}
// 注意:在弱联网模式下,我们需要验证指定资源版本的所有资源完整性
if (weaklyUpdate)
{
string filePath = SandboxHelper.MakeCacheFilePath(patchBundle.FileName);
if (File.Exists(filePath))
_waitingList.Add(patchBundle);
else
return false;
}
else
{
string filePath = SandboxHelper.MakeCacheFilePath(patchBundle.FileName);
if (File.Exists(filePath))
_waitingList.Add(patchBundle);
}
}
// 设置同时验证的最大数
_verifyMaxNum = 32;
_verifyTotalCount = _waitingList.Count;
return true;
}
public override bool UpdateVerifier()
{
if (_waitingList.Count == 0 && _verifyingList.Count == 0)
return true;
for (int i = _waitingList.Count - 1; i >= 0; i--)
{
if (_verifyingList.Count >= _verifyMaxNum)
break;
var patchBundle = _waitingList[i];
VerifyFile(patchBundle);
_waitingList.RemoveAt(i);
_verifyingList.Add(patchBundle);
}
_verifyingList.Clear();
return false;
}
public override float GetVerifierProgress()
{
if (_verifyTotalCount == 0)
return 1f;
return (float)(VerifySuccessCount + VerifyFailCount) / _verifyTotalCount;
}
private void VerifyFile(PatchBundle patchBundle)
{
string filePath = SandboxHelper.MakeCacheFilePath(patchBundle.FileName);
bool result = DownloadSystem.CheckContentIntegrity(filePath, patchBundle.FileSize, patchBundle.FileCRC);
if (result)
{
VerifySuccessCount++;
DownloadSystem.CacheVerifyFile(patchBundle.FileHash, patchBundle.FileName);
}
else
{
VerifyFailCount++;
// NOTE:不期望删除断点续传的资源文件
/*
YooLogger.Warning($"Failed to verify file : {info.FilePath}");
if (File.Exists(info.FilePath))
File.Delete(info.FilePath);
*/
}
}
}
}
@@ -1,11 +0,0 @@
fileFormatVersion: 2
guid: 223f217ed81460541b5ea7eb7a7d82a4
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant: