refactor : cache file system

2024-12-12 15:00:08 +08:00
parent bf1e3da298
commit b1338a9ffd
23 changed files with 336 additions and 221 deletions
@@ -1,67 +0,0 @@
-using System.Collections;
-using System.Collections.Generic;
-
-namespace YooAsset
-{
-    internal sealed class DCFSClearAllBundleFilesOperation : FSClearAllBundleFilesOperation
-    {
-        private enum ESteps
-        {
-            None,
-            GetAllCacheFiles,
-            ClearAllCacheFiles,
-            Done,
-        }
-
-        private readonly DefaultCacheFileSystem _fileSystem;
-        private List<string> _allBundleGUIDs;
-        private int _fileTotalCount = 0;
-        private ESteps _steps = ESteps.None;
-
-
-        internal DCFSClearAllBundleFilesOperation(DefaultCacheFileSystem fileSystem)
-        {
-            _fileSystem = fileSystem;
-        }
-        internal override void InternalOnStart()
-        {
-            _steps = ESteps.GetAllCacheFiles;
-        }
-        internal override void InternalOnUpdate()
-        {
-            if (_steps == ESteps.None || _steps == ESteps.Done)
-                return;
-
-            if (_steps == ESteps.GetAllCacheFiles)
-            {
-                _allBundleGUIDs = _fileSystem.GetAllCachedBundleGUIDs();
-                _fileTotalCount = _allBundleGUIDs.Count;
-                _steps = ESteps.ClearAllCacheFiles;
-                YooLogger.Log($"Found all cache files count : {_fileTotalCount}");
-            }
-
-            if (_steps == ESteps.ClearAllCacheFiles)
-            {
-                for (int i = _allBundleGUIDs.Count - 1; i >= 0; i--)
-                {
-                    string bundleGUID = _allBundleGUIDs[i];
-                    _fileSystem.DeleteCacheFile(bundleGUID);
-                    _allBundleGUIDs.RemoveAt(i);
-                    if (OperationSystem.IsBusy)
-                        break;
-                }
-
-                if (_fileTotalCount == 0)
-                    Progress = 1.0f;
-                else
-                    Progress = 1.0f - (_allBundleGUIDs.Count / _fileTotalCount);
-
-                if (_allBundleGUIDs.Count == 0)
-                {
-                    _steps = ESteps.Done;
-                    Status = EOperationStatus.Succeed;
-                }
-            }
-        }
-    }
-}
@@ -1,11 +0,0 @@
-fileFormatVersion: 2
-guid: daed187502431d347bf11bddcf8d77de
-MonoImporter:
-  externalObjects: {}
-  serializedVersion: 2
-  defaultReferences: []
-  executionOrder: 0
-  icon: {instanceID: 0}
-  userData: 
-  assetBundleName: 
-  assetBundleVariant: 
@@ -1,83 +0,0 @@
-using System.Collections;
-using System.Collections.Generic;
-
-namespace YooAsset
-{
-    internal sealed class DCFSClearUnusedBundleFilesOperation : FSClearUnusedBundleFilesOperation
-    {
-        private enum ESteps
-        {
-            None,
-            GetUnusedCacheFiles,
-            ClearUnusedCacheFiles,
-            Done,
-        }
-
-        private readonly DefaultCacheFileSystem _fileSystem;
-        private readonly PackageManifest _manifest;
-        private List<string> _unusedBundleGUIDs;
-        private int _unusedFileTotalCount = 0;
-        private ESteps _steps = ESteps.None;
-
-
-        internal DCFSClearUnusedBundleFilesOperation(DefaultCacheFileSystem fileSystem, PackageManifest manifest)
-        {
-            _fileSystem = fileSystem;
-            _manifest = manifest;
-        }
-        internal override void InternalOnStart()
-        {
-            _steps = ESteps.GetUnusedCacheFiles;
-        }
-        internal override void InternalOnUpdate()
-        {
-            if (_steps == ESteps.None || _steps == ESteps.Done)
-                return;
-
-            if (_steps == ESteps.GetUnusedCacheFiles)
-            {    
-                _unusedBundleGUIDs = GetUnusedBundleGUIDs();
-                _unusedFileTotalCount = _unusedBundleGUIDs.Count;
-                _steps = ESteps.ClearUnusedCacheFiles;
-                YooLogger.Log($"Found unused cache files count : {_unusedFileTotalCount}");
-            }
-
-            if (_steps == ESteps.ClearUnusedCacheFiles)
-            {
-                for (int i = _unusedBundleGUIDs.Count - 1; i >= 0; i--)
-                {
-                    string bundleGUID = _unusedBundleGUIDs[i];
-                    _fileSystem.DeleteCacheFile(bundleGUID);
-                    _unusedBundleGUIDs.RemoveAt(i);
-                    if (OperationSystem.IsBusy)
-                        break;
-                }
-
-                if (_unusedFileTotalCount == 0)
-                    Progress = 1.0f;
-                else
-                    Progress = 1.0f - (_unusedBundleGUIDs.Count / _unusedFileTotalCount);
-
-                if (_unusedBundleGUIDs.Count == 0)
-                {
-                    _steps = ESteps.Done;
-                    Status = EOperationStatus.Succeed;
-                }
-            }
-        }
-
-        private List<string> GetUnusedBundleGUIDs()
-        {
-            var allBundleGUIDs = _fileSystem.GetAllCachedBundleGUIDs();
-            List<string> result = new List<string>(allBundleGUIDs.Count);
-            foreach (var bundleGUID in allBundleGUIDs)
-            {
-                if (_manifest.IsIncludeBundleFile(bundleGUID) == false)
-                {
-                    result.Add(bundleGUID);
-                }
-            }
-            return result;
-        }
-    }
-}
@@ -1,11 +0,0 @@
-fileFormatVersion: 2
-guid: be7b242f10abf524fa59e9ca0d12b052
-MonoImporter:
-  externalObjects: {}
-  serializedVersion: 2
-  defaultReferences: []
-  executionOrder: 0
-  icon: {instanceID: 0}
-  userData: 
-  assetBundleName: 
-  assetBundleVariant: 
@@ -57,7 +57,7 @@ namespace YooAsset
            {
                if (_searchCacheFilesOp == null)
                {
-                    _searchCacheFilesOp = new SearchCacheFilesOperation(_fileSytem);
+                    _searchCacheFilesOp = new SearchCacheFilesOperation(_fileSytem, _fileSytem.PackageName, _fileSytem.AppendFileExtension);
                    OperationSystem.StartOperation(_fileSytem.PackageName, _searchCacheFilesOp);
                }

@@ -72,7 +72,7 @@ namespace YooAsset
            {
                if (_verifyCacheFilesOp == null)
                {
-                    _verifyCacheFilesOp = new VerifyCacheFilesOperation(_fileSytem, _searchCacheFilesOp.Result);
+                    _verifyCacheFilesOp = new VerifyCacheFilesOperation(_fileSytem, _fileSytem.FileVerifyLevel, _searchCacheFilesOp.Result);
                    OperationSystem.StartOperation(_fileSytem.PackageName, _verifyCacheFilesOp);
                }

@@ -1,124 +0,0 @@
-using System;
-using System.IO;
-using System.Collections;
-using System.Collections.Generic;
-
-namespace YooAsset
-{
-    internal sealed class SearchCacheFilesOperation : AsyncOperationBase
-    {
-        private enum ESteps
-        {
-            None,
-            Prepare,
-            SearchFiles,
-            Done,
-        }
-
-        private readonly DefaultCacheFileSystem _fileSystem;
-        private IEnumerator<DirectoryInfo> _filesEnumerator = null;
-        private float _verifyStartTime;
-        private ESteps _steps = ESteps.None;
-
-        /// <summary>
-        /// 需要验证的元素
-        /// </summary>
-        public readonly List<CacheFileElement> Result = new List<CacheFileElement>(5000);
-
-
-        internal SearchCacheFilesOperation(DefaultCacheFileSystem fileSystem)
-        {
-            _fileSystem = fileSystem;
-        }
-        internal override void InternalOnStart()
-        {
-            _steps = ESteps.Prepare;
-            _verifyStartTime = UnityEngine.Time.realtimeSinceStartup;
-        }
-        internal override void InternalOnUpdate()
-        {
-            if (_steps == ESteps.None || _steps == ESteps.Done)
-                return;
-
-            if (_steps == ESteps.Prepare)
-            {
-                DirectoryInfo rootDirectory = new DirectoryInfo(_fileSystem.GetCacheFilesRoot());
-                if (rootDirectory.Exists)
-                {
-                    var directorieInfos = rootDirectory.EnumerateDirectories();
-                    _filesEnumerator = directorieInfos.GetEnumerator();
-                }
-                _steps = ESteps.SearchFiles;
-            }
-
-            if (_steps == ESteps.SearchFiles)
-            {
-                if (SearchFiles())
-                    return;
-
-                _steps = ESteps.Done;
-                Status = EOperationStatus.Succeed;
-                float costTime = UnityEngine.Time.realtimeSinceStartup - _verifyStartTime;
-                YooLogger.Log($"Search cache files elapsed time {costTime:f1} seconds");
-            }
-        }
-
-        private bool SearchFiles()
-        {
-            if (_filesEnumerator == null)
-                return false;
-
-            bool isFindItem;
-            while (true)
-            {
-                isFindItem = _filesEnumerator.MoveNext();
-                if (isFindItem == false)
-                    break;
-
-                var rootFoder = _filesEnumerator.Current;
-                var childDirectories = rootFoder.GetDirectories();
-                foreach (var chidDirectory in childDirectories)
-                {
-                    string bundleGUID = chidDirectory.Name;
-                    if (_fileSystem.IsRecordFile(bundleGUID))
-                        continue;
-
-                    // 创建验证元素类
-                    string fileRootPath = chidDirectory.FullName;
-                    string dataFilePath = $"{fileRootPath}/{ DefaultCacheFileSystemDefine.SaveBundleDataFileName}";
-                    string infoFilePath = $"{fileRootPath}/{ DefaultCacheFileSystemDefine.SaveBundleInfoFileName}";
-
-                    // 存储的数据文件追加文件格式
-                    if (_fileSystem.AppendFileExtension)
-                    {
-                        string dataFileExtension = FindDataFileExtension(chidDirectory);
-                        if (string.IsNullOrEmpty(dataFileExtension) == false)
-                            dataFilePath += dataFileExtension;
-                    }
-
-                    var element = new CacheFileElement(_fileSystem.PackageName, bundleGUID, fileRootPath, dataFilePath, infoFilePath);
-                    Result.Add(element);
-                }
-
-                if (OperationSystem.IsBusy)
-                    break;
-            }
-
-            return isFindItem;
-        }
-        private string FindDataFileExtension(DirectoryInfo directoryInfo)
-        {
-            string dataFileExtension = string.Empty;
-            var fileInfos = directoryInfo.GetFiles();
-            foreach (var fileInfo in fileInfos)
-            {
-                if (fileInfo.Name.StartsWith(DefaultCacheFileSystemDefine.SaveBundleDataFileName))
-                {
-                    dataFileExtension = fileInfo.Extension;
-                    break;
-                }
-            }
-            return dataFileExtension;
-        }
-    }
-}
@@ -1,11 +0,0 @@
-fileFormatVersion: 2
-guid: 8adb017a806563548b1d4c47f7b343e2
-MonoImporter:
-  externalObjects: {}
-  serializedVersion: 2
-  defaultReferences: []
-  executionOrder: 0
-  icon: {instanceID: 0}
-  userData: 
-  assetBundleName: 
-  assetBundleVariant: 
@@ -1,205 +0,0 @@
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.IO;
-using System.Threading;
-
-namespace YooAsset
-{
-    internal class CacheFileElement
-    {
-        public string PackageName { private set; get; }
-        public string BundleGUID { private set; get; }
-        public string FileRootPath { private set; get; }
-        public string DataFilePath { private set; get; }
-        public string InfoFilePath { private set; get; }
-
-        public EFileVerifyResult Result;
-        public string DataFileCRC;
-        public long DataFileSize;
-
-        public CacheFileElement(string packageName, string bundleGUID, string fileRootPath, string dataFilePath, string infoFilePath)
-        {
-            PackageName = packageName;
-            BundleGUID = bundleGUID;
-            FileRootPath = fileRootPath;
-            DataFilePath = dataFilePath;
-            InfoFilePath = infoFilePath;
-        }
-
-        public void DeleteFiles()
-        {
-            try
-            {
-                Directory.Delete(FileRootPath, true);
-            }
-            catch (System.Exception e)
-            {
-                YooLogger.Warning($"Failed to delete cache bundle folder : {e}");
-            }
-        }
-    }
-
-    /// <summary>
-    /// 缓存文件验证（线程版）
-    /// </summary>
-    internal class VerifyCacheFilesOperation : AsyncOperationBase
-    {
-        private enum ESteps
-        {
-            None,
-            InitVerify,
-            UpdateVerify,
-            Done,
-        }
-
-        private readonly ThreadSyncContext _syncContext = new ThreadSyncContext();
-        private readonly DefaultCacheFileSystem _fileSystem;
-        private List<CacheFileElement> _waitingList;
-        private List<CacheFileElement> _verifyingList;
-        private EFileVerifyLevel _verifyLevel = EFileVerifyLevel.Middle;
-        private int _verifyMaxNum;
-        private int _verifyTotalCount;
-        private float _verifyStartTime;
-        private int _succeedCount;
-        private int _failedCount;
-        private ESteps _steps = ESteps.None;
-
-
-        internal VerifyCacheFilesOperation(DefaultCacheFileSystem fileSystem, List<CacheFileElement> elements)
-        {
-            _fileSystem = fileSystem;
-            _waitingList = elements;
-            _verifyLevel = _fileSystem.FileVerifyLevel;
-        }
-        internal override void InternalOnStart()
-        {
-            _steps = ESteps.InitVerify;
-            _verifyStartTime = UnityEngine.Time.realtimeSinceStartup;
-        }
-        internal override void InternalOnUpdate()
-        {
-            if (_steps == ESteps.None || _steps == ESteps.Done)
-                return;
-
-            if (_steps == ESteps.InitVerify)
-            {
-                int fileCount = _waitingList.Count;
-
-                // 设置同时验证的最大数
-                ThreadPool.GetMaxThreads(out int workerThreads, out int ioThreads);
-                YooLogger.Log($"Work threads : {workerThreads}, IO threads : {ioThreads}");
-                _verifyMaxNum = Math.Min(workerThreads, ioThreads);
-                _verifyTotalCount = fileCount;
-                if (_verifyMaxNum < 1)
-                    _verifyMaxNum = 1;
-
-                _verifyingList = new List<CacheFileElement>(_verifyMaxNum);
-                _steps = ESteps.UpdateVerify;
-            }
-
-            if (_steps == ESteps.UpdateVerify)
-            {
-                _syncContext.Update();
-
-                Progress = GetProgress();
-                if (_waitingList.Count == 0 && _verifyingList.Count == 0)
-                {
-                    _steps = ESteps.Done;
-                    Status = EOperationStatus.Succeed;
-                    float costTime = UnityEngine.Time.realtimeSinceStartup - _verifyStartTime;
-                    YooLogger.Log($"Verify cache files elapsed time {costTime:f1} seconds");
-                }
-
-                for (int i = _waitingList.Count - 1; i >= 0; i--)
-                {
-                    if (OperationSystem.IsBusy)
-                        break;
-
-                    if (_verifyingList.Count >= _verifyMaxNum)
-                        break;
-
-                    var element = _waitingList[i];
-                    if (BeginVerifyFileWithThread(element))
-                    {
-                        _waitingList.RemoveAt(i);
-                        _verifyingList.Add(element);
-                    }
-                    else
-                    {
-                        YooLogger.Warning("The thread pool is failed queued.");
-                        break;
-                    }
-                }
-            }
-        }
-
-        private float GetProgress()
-        {
-            if (_verifyTotalCount == 0)
-                return 1f;
-            return (float)(_succeedCount + _failedCount) / _verifyTotalCount;
-        }
-        private bool BeginVerifyFileWithThread(CacheFileElement element)
-        {
-            return ThreadPool.QueueUserWorkItem(new WaitCallback(VerifyInThread), element);
-        }
-        private void VerifyInThread(object obj)
-        {
-            CacheFileElement element = (CacheFileElement)obj;
-            element.Result = VerifyingCacheFile(element, _verifyLevel);
-            _syncContext.Post(VerifyCallback, element);
-        }
-        private void VerifyCallback(object obj)
-        {
-            CacheFileElement element = (CacheFileElement)obj;
-            _verifyingList.Remove(element);
-
-            if (element.Result == EFileVerifyResult.Succeed)
-            {
-                _succeedCount++;
-                var fileWrapper = new DefaultCacheFileSystem.FileWrapper(element.InfoFilePath, element.DataFilePath, element.DataFileCRC, element.DataFileSize);
-                _fileSystem.RecordFile(element.BundleGUID, fileWrapper);
-            }
-            else
-            {
-                _failedCount++;
-
-                YooLogger.Warning($"Failed to verify file {element.Result} and delete files : {element.FileRootPath}");
-                element.DeleteFiles();
-            }
-        }
-
-        /// <summary>
-        /// 验证缓存文件（子线程内操作）
-        /// </summary>
-        private EFileVerifyResult VerifyingCacheFile(CacheFileElement element, EFileVerifyLevel verifyLevel)
-        {
-            try
-            {
-                if (verifyLevel == EFileVerifyLevel.Low)
-                {
-                    if (File.Exists(element.InfoFilePath) == false)
-                        return EFileVerifyResult.InfoFileNotExisted;
-                    if (File.Exists(element.DataFilePath) == false)
-                        return EFileVerifyResult.DataFileNotExisted;
-                    return EFileVerifyResult.Succeed;
-                }
-                else
-                {
-                    if (File.Exists(element.InfoFilePath) == false)
-                        return EFileVerifyResult.InfoFileNotExisted;
-
-                    // 解析信息文件获取验证数据
-                    _fileSystem.ReadInfoFile(element.InfoFilePath, out element.DataFileCRC, out element.DataFileSize);
-                }
-            }
-            catch (Exception)
-            {
-                return EFileVerifyResult.Exception;
-            }
-
-            return FileSystemHelper.FileVerify(element.DataFilePath, element.DataFileSize, element.DataFileCRC, verifyLevel);
-        }
-    }
-}
@@ -1,11 +0,0 @@
-fileFormatVersion: 2
-guid: 5e9325baa24ae234ea0208d70dd3347e
-MonoImporter:
-  externalObjects: {}
-  serializedVersion: 2
-  defaultReferences: []
-  executionOrder: 0
-  icon: {instanceID: 0}
-  userData: 
-  assetBundleName: 
-  assetBundleVariant: 
@@ -1,110 +0,0 @@
-using System;
-using System.Threading;
-
-namespace YooAsset
-{
-    internal class TempFileElement
-    {
-        public string TempFilePath { private set; get; }
-        public string TempFileCRC { private set; get; }
-        public long TempFileSize { private set; get; }
-
-        /// <summary>
-        /// 注意：原子操作对象
-        /// </summary>
-        public int Result = 0;
-
-        public TempFileElement(string filePath, string fileCRC, long fileSize)
-        {
-            TempFilePath = filePath;
-            TempFileCRC = fileCRC;
-            TempFileSize = fileSize;
-        }
-    }
-
-    /// <summary>
-    /// 下载文件验证（线程版）
-    /// </summary>
-    internal class VerifyTempFileOperation : AsyncOperationBase
-    {
-        private enum ESteps
-        {
-            None,
-            VerifyFile,
-            Waiting,
-            Done,
-        }
-
-        private readonly TempFileElement _element;
-        private ESteps _steps = ESteps.None;
-
-        /// <summary>
-        /// 验证结果
-        /// </summary>
-        public EFileVerifyResult VerifyResult { protected set; get; }
-
-
-        internal VerifyTempFileOperation(TempFileElement element)
-        {
-            _element = element;
-        }
-        internal override void InternalOnStart()
-        {
-            _steps = ESteps.VerifyFile;
-        }
-        internal override void InternalOnUpdate()
-        {
-            if (_steps == ESteps.None || _steps == ESteps.Done)
-                return;
-
-            if (_steps == ESteps.VerifyFile)
-            {
-                if (BeginVerifyFileWithThread(_element))
-                {
-                    _steps = ESteps.Waiting;
-                }
-            }
-
-            if (_steps == ESteps.Waiting)
-            {
-                int result = _element.Result;
-                if (result == 0)
-                    return;
-
-                VerifyResult = (EFileVerifyResult)result;
-                if (VerifyResult == EFileVerifyResult.Succeed)
-                {
-                    _steps = ESteps.Done;
-                    Status = EOperationStatus.Succeed;
-                }
-                else
-                {
-                    _steps = ESteps.Done;
-                    Status = EOperationStatus.Failed;
-                    Error = $"Failed to verify file : {_element.TempFilePath} ! ErrorCode : {VerifyResult}";
-                }
-            }
-        }
-        internal override void InternalWaitForAsyncComplete()
-        {
-            while (true)
-            {
-                // 注意：等待子线程验证文件完毕
-                InternalOnUpdate();
-                if (IsDone)
-                    break;
-            }
-        }
-
-        private bool BeginVerifyFileWithThread(TempFileElement element)
-        {
-            return ThreadPool.QueueUserWorkItem(new WaitCallback(VerifyInThread), element);
-        }
-        private void VerifyInThread(object obj)
-        {
-            TempFileElement element = (TempFileElement)obj;
-            int result = (int)FileSystemHelper.FileVerify(element.TempFilePath, element.TempFileSize, element.TempFileCRC, EFileVerifyLevel.High);
-            element.Result = result;
-        }
-    }
-}
@@ -1,11 +0,0 @@
-fileFormatVersion: 2
-guid: 764ac4f47afd3be4a8d583578f51126d
-MonoImporter:
-  externalObjects: {}
-  serializedVersion: 2
-  defaultReferences: []
-  executionOrder: 0
-  icon: {instanceID: 0}
-  userData: 
-  assetBundleName: 
-  assetBundleVariant: