前言 前段时间隔壁项目组主程为了能更快速的打包项目里的AssetBundle ,提了个增加补丁包构建的需求,于是费了几番功夫后便在CatAsset 中加入了此功能,这篇文章便用来记录此功能的实现思路
何为补丁包 首先需要明确几个AssetBundle 构建相关的概念:
全量构建 :将所有需要打包的资源提交给Unity的AssetBundle 构建管线,然后Unity对所有AssetBundle 全部进行重新构建(此构建方式很少被使用)
增量构建 :将所有需要打包的资源提交给Unity的AssetBundle 构建管线,然后Unity仅对发生了变化的AssetBundle 进行重新构建(这是最常见的标准构建方式)
补丁构建 :仅将发生了变化的资源及其相关资源提交给Unity的AssetBundle 构建管线,然后由Unity进行AssetBundle 构建
所谓补丁包 ,即是补丁构建的产物
举例来说,若有资源a、b、c、d被打包为资源包A ,e、f、g、h被打包为资源包B (这些资源之间没有依赖关系),且其中资源b发生了变化
那么在进行增量构建 时,会将这8个资源全部提交给Unity底层的构建管线,然后Unity会将a、b、c、d所属的资源包A进行重新构建,资源包B则从缓存中复制
而如果进行的是补丁构建 ,那么只会将发生了变化的资源b提交给Unity单独打包为资源包A_patch,与资源包A、B共存,且后续在加载资源b时只会从A_patch中加载
补丁构建的优缺点 补丁构建的优点主要在于:
资源数量庞大的时候可以很好的加快打包速度,节省时间,减少因为版本日打包导致的加班风险 ,
在进行资源更新时可以有效降低玩家需要更新的资源大小
而其缺点则主要是:
因为补丁包与正式包是共存的,所以会产生一定程度的资源冗余
补丁资源的计算基于与上一次完整打包(非补丁构建)产生的缓存信息的对比,所以随着变化的资源增多,会提交给Unity的资源数也会增多,构建速度也会逐渐趋近于完整打包的速度,这时需要通过重新进行一次新的完整打包来更新缓存信息
构建补丁包时要注意的地方 想要进行补丁包构建,首先需要记录上次完整打包的缓存信息,在CatAsset中是通过记录文件最后写入时间 实现,但不仅仅是资源文件的最后写入时间,还需要包括资源文件对应的Meta文件的最后写入时间 才行,否则会导致补丁资源计算出错,因为有些对资源的修改是被反应到Meta文件里的
另外在计算补丁资源时,不仅是需要判断资源自身是否有变化,还需要考虑它所依赖的资源是否为补丁资源,若它所依赖(直接或间接)的任意一个资源为补丁资源,则此资源也必须视为补丁资源处理(这就意味着补丁资源具有下游传染性 ,会传染给依赖链下游的所有资源),因为只有在同一批AssetBundle打包的资源之间才能正确进行互相的依赖引用 ,否则就会产生依赖补丁资源的资源,其依赖加载的是旧资源而非新的补丁资源 的问题
对于补丁资源所依赖的资源,则采用隐式依赖自动包含的机制,不对其进行显式构建,且从补丁资源的依赖列表中移除,以此故意冗余一份相同的依赖资源到补丁资源所在的补丁包中,加载时让Unity自动加载,以保证正式包和补丁包的依赖到相同资源时都能被正确的加载到
举例来说,假设有依赖链为D -> C -> B -> A 和D -> E ,且C为变化的资源,那么最终会将C以及依赖C的B和A 作为补丁资源,D作为C的隐式依赖 包含进C的补丁包里,运行时E依赖的D和C依赖的D会分别在不同的包里 保证E的依赖不丢失(因为在补丁构建后会将资源清单中的旧资源信息删除,保留新的补丁资源信息,以保证能加载到最新的补丁资源)
构建补丁包的具体步骤 CatAsset使用SBP进行AssetBundle构建,并自定义了一些构建任务来满足需求
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 public static ReturnCode BuildBundles (BuildTarget targetPlatform,bool isBuildPatch ) { BundleBuildConfigSO config = BundleBuildConfigSO.Instance; var preData = new BundleBuildPreProcessData { Config = config, TargetPlatform = targetPlatform }; OnBundleBuildPreProcess(preData); if (isBuildPatch) { string manifestPath = EditorUtil.GetCacheManifestPath(config.OutputRootDirectory, targetPlatform); string assetCacheManifestPath = EditorUtil.GetAssetCacheManifestPath(config.OutputRootDirectory); if (!File.Exists(manifestPath) || !File.Exists(assetCacheManifestPath)) { isBuildPatch = false ; } } string fullOutputFolder = CreateFullOutputFolder(config, targetPlatform); BundleBuildInfoParam buildInfoParam = new BundleBuildInfoParam(); BundleBuildConfigParam configParam = new BundleBuildConfigParam(config, targetPlatform,isBuildPatch); BundleBuildParameters buildParam = GetSBPParameters(config, targetPlatform, fullOutputFolder); BundleBuildContent content = new BundleBuildContent(); List<IBuildTask> taskList = GetSBPInternalBuildTask(!isBuildPatch); taskList.Insert(0 ,new CalculateBundleBuilds()); taskList.Add(new BuildRawBundles()); taskList.Add(new BuildManifest()); taskList.Add(new EncryptBundles()); taskList.Add(new CalculateVerifyInfo()); if (HasOption(config.Options,BundleBuildOptions.AppendHash)) { taskList.Add(new AppendHash()); } if (isBuildPatch) { taskList.Add(new RemoveNonPatchDependency()); taskList.Add(new MergePatchManifest()); } taskList.Add(new WriteManifestFile()); if (!isBuildPatch) { taskList.Add(new WriteCacheFile()); } if (config.IsCopyToReadOnlyDirectory && config.TargetPlatforms.Count == 1 ) { taskList.Add(new CopyToReadOnlyDirectory()); taskList.Add(new WriteManifestFile()); } Stopwatch sw = Stopwatch.StartNew(); ReturnCode returnCode = ContentPipeline.BuildAssetBundles(buildParam, content, out IBundleBuildResults result, taskList, buildInfoParam,configParam); if (returnCode == ReturnCode.Success) { Debug.Log($"资源包构建成功:{returnCode} ,耗时:{sw.Elapsed.Hours} 时{sw.Elapsed.Minutes} 分{sw.Elapsed.Seconds} 秒" ); } else { Debug.LogError($"资源包构建未成功:{returnCode} ,耗时:{sw.Elapsed.Hours} 时{sw.Elapsed.Minutes} 分{sw.Elapsed.Seconds} 秒" ); } var postData = new BundleBuildPostProcessData { Config = config, TargetPlatform = targetPlatform, OutputFolder = fullOutputFolder, ReturnCode = returnCode, Result = result, }; OnBundleBuildPostProcess(postData); return returnCode; }
总的来说,构建补丁包需要3个步骤:
在进行完整构建时记录资源缓存信息
计算补丁资源
合并资源清单
生成完整构建时的资源缓存 资源缓存信息清单的类定义
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 using System;using System.Collections.Generic;using System.IO;using CatAsset.Runtime;namespace CatAsset.Editor { [Serializable ] public class AssetCacheManifest { [Serializable ] public struct AssetCacheInfo : IEquatable<AssetCacheInfo> { public string Name; public long LastWriteTime; public long MetaLastWriteTime; public static AssetCacheInfo Create (string assetName ) { AssetCacheInfo assetCacheInfo = new AssetCacheInfo { Name = assetName, LastWriteTime = File.GetLastWriteTime(assetName).Ticks, MetaLastWriteTime = File.GetLastWriteTime($"{assetName} .meta" ).Ticks, }; return assetCacheInfo; } public static bool operator ==(AssetCacheInfo a,AssetCacheInfo b) { return Equals(a, b); } public static bool operator !=(AssetCacheInfo a,AssetCacheInfo b) { return !(a == b); } public bool Equals (AssetCacheInfo other ) { return Name == other.Name && LastWriteTime == other.LastWriteTime && MetaLastWriteTime == other.MetaLastWriteTime; } public override bool Equals (object obj ) { return obj is AssetCacheInfo other && Equals(other); } public override int GetHashCode ( ) { unchecked { var hashCode = (Name != null ? Name.GetHashCode() : 0 ); hashCode = (hashCode * 397 ) ^ LastWriteTime.GetHashCode(); hashCode = (hashCode * 397 ) ^ MetaLastWriteTime.GetHashCode(); return hashCode; } } } public const string ManifestJsonFileName = "AssetCacheManifest.json" ; public List <AssetCacheInfo > Caches = new List<AssetCacheInfo>(); public Dictionary<string , AssetCacheInfo> GetCacheDict ( ) { Dictionary<string , AssetCacheInfo> result = new Dictionary<string , AssetCacheInfo>(); foreach (AssetCacheInfo assetCache in Caches) { result.Add(assetCache.Name,assetCache); } return result; } } }
正如之前提到的,除了资源文件本身的最后写入时间 外还需要记录Meta文件的最后写入时间
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 using System.IO;using CatAsset.Runtime;using UnityEditor;using UnityEditor.Build.Pipeline;using UnityEditor.Build.Pipeline.Injector;using UnityEditor.Build.Pipeline.Interfaces;using UnityEngine;namespace CatAsset.Editor { public class WriteCacheFile : IBuildTask { [InjectContext(ContextUsage.In) ] private IBundleBuildConfigParam configParam; [InjectContext(ContextUsage.In) ] private IManifestParam manifestParam; [InjectContext(ContextUsage.In) ] private IBundleBuildParameters buildParam; public int Version { get ; } public ReturnCode Run ( ) { string folder = EditorUtil.GetBundleCacheFolder(configParam.Config.OutputRootDirectory, configParam.TargetPlatform); EditorUtil.CreateEmptyDirectory(folder); EditorUtil.CopyDirectory(((BundleBuildParameters)buildParam).OutputFolder,folder); folder = EditorUtil.GetAssetCacheManifestFolder(configParam.Config.OutputRootDirectory); EditorUtil.CreateEmptyDirectory(folder); AssetCacheManifest assetCacheManifest = new AssetCacheManifest(); foreach (BundleManifestInfo bundleManifestInfo in manifestParam.Manifest.Bundles) { foreach (AssetManifestInfo assetManifestInfo in bundleManifestInfo.Assets) { AssetCacheManifest.AssetCacheInfo cacheInfo = AssetCacheManifest.AssetCacheInfo.Create(assetManifestInfo.Name); assetCacheManifest.Caches.Add(cacheInfo); } } string json = EditorJsonUtility.ToJson(assetCacheManifest,true ); string path = RuntimeUtil.GetRegularPath(Path.Combine(folder, AssetCacheManifest.ManifestJsonFileName)); using (StreamWriter sw = new StreamWriter(path)) { sw.Write(json); } return ReturnCode.Success; } } }
除了记录资源最后写入时间 外,还需要将完整构建产出的资源包复制到资源包缓存目录下,用于在补丁构建后进行合并形成最终的完整资源包输出
计算补丁资源 补丁资源的计算大致由以下步骤组成:
判断自身是否变化
判断自身依赖的资源是否为补丁资源
判断【资源是否变化】的步骤则为:
是否为新资源
是否为变化的旧资源
是否为被移动到新包里的旧资源
具体代码如下:
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 using System.Collections.Generic;using System.Diagnostics;using System.IO;using System.Linq;using CatAsset.Runtime;using UnityEditor;using UnityEditor.Build.Pipeline;using UnityEditor.Build.Pipeline.Injector;using UnityEditor.Build.Pipeline.Interfaces;using UnityEditor.Build.Pipeline.Utilities;using UnityEngine;using Debug = UnityEngine.Debug;namespace CatAsset.Editor { public class CalculateBundleBuilds : IBuildTask { public int Version { get ; } [InjectContext(ContextUsage.In) ] private IBundleBuildParameters buildParam; [InjectContext(ContextUsage.InOut) ] private IBundleBuildInfoParam buildInfoParam; [InjectContext(ContextUsage.In) ] private IBundleBuildConfigParam configParam; [InjectContext(ContextUsage.In) ] private IBuildCache buildCache; [InjectContext(ContextUsage.InOut) ] private IBundleBuildContent content; [InjectContext(ContextUsage.InOut) ] private IBuildContent content2; public ReturnCode Run ( ) { BundleBuildConfigSO config = configParam.Config; if (!configParam.IsBuildPatch) { buildInfoParam = new BundleBuildInfoParam(config.GetAssetBundleBuilds(), config.GetNormalBundleBuilds(), config.GetRawBundleBuilds()); } else { Stopwatch sw = Stopwatch.StartNew(); var clonedConfig = new PatchAssetCalculateHelper().Calculate(config, configParam.TargetPlatform); sw.Stop(); Debug.Log($"计算补丁资源耗时:{sw.Elapsed.TotalSeconds:0.00 } 秒" ); buildInfoParam = new BundleBuildInfoParam(clonedConfig.GetAssetBundleBuilds(), clonedConfig.GetNormalBundleBuilds(), clonedConfig.GetRawBundleBuilds()); } ((BundleBuildParameters)buildParam).SetBundleBuilds(buildInfoParam.NormalBundleBuilds); content = new BundleBuildContent(buildInfoParam.AssetBundleBuilds); content2 = content; return ReturnCode.Success; } } }
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 using System.Collections.Generic;using System.IO;using CatAsset.Runtime;using UnityEditor;using UnityEngine;namespace CatAsset.Editor { public class PatchAssetCalculateHelper { private Dictionary <string , List <string >> upStreamDict = new Dictionary<string , List<string >>(); private Dictionary <string , string > assetToBundle = new Dictionary<string , string >(); private Dictionary <string , string > cacheAssetToBundle = new Dictionary<string , string >(); private AssetCacheManifest assetCacheManifest; private Dictionary<string , AssetCacheManifest.AssetCacheInfo> assetCacheDict; private Dictionary<string , AssetCacheManifest.AssetCacheInfo> curAssetCacheDict = new Dictionary<string , AssetCacheManifest.AssetCacheInfo>(); private Dictionary <string , bool > assetChangeStateDict = new Dictionary<string , bool >(); private Dictionary <string , bool > assetPatchStateDict = new Dictionary<string , bool >(); public BundleBuildConfigSO Calculate (BundleBuildConfigSO config, BuildTarget buildTarget ) { assetCacheManifest = ReadAssetCache(config); assetCacheDict = assetCacheManifest.GetCacheDict(); BundleBuildConfigSO clonedConfig = Object.Instantiate(config); GetDependencyChain(config); ReadCachedManifest(config,buildTarget); CalPatchAsset(config, clonedConfig); return clonedConfig; } private void CalPatchAsset (BundleBuildConfigSO config, BundleBuildConfigSO clonedConfig ) { int index = 0 ; for (int i = clonedConfig.Bundles.Count - 1 ; i >= 0 ; i--) { var bundle = clonedConfig.Bundles[i]; bool isAllPatch = true ; for (int j = bundle.Assets.Count - 1 ; j >= 0 ; j--) { var asset = bundle.Assets[j]; index++; EditorUtility.DisplayProgressBar($"计算补丁资源" , $"{asset.Name} " , index / (config.AssetCount * 1.0f )); bool isPatch = IsPatchAsset(asset.Name); if (isPatch) { Debug.Log($"发现补丁资源:{asset.Name} " ); } else { bundle.Assets.RemoveAt(j); isAllPatch = false ; } } if (bundle.Assets.Count > 0 ) { if (!isAllPatch) { var part = bundle.BundleName.Split('.' ); bundle.BundleName = $"{part[0 ]} _patch.{part[1 ]} " ; bundle.BundleIdentifyName = BundleBuildInfo.GetBundleIdentifyName(bundle.DirectoryName, bundle.BundleName); } } else { clonedConfig.Bundles.RemoveAt(i); } } EditorUtility.ClearProgressBar(); } private bool IsPatchAsset (string assetName ) { if (assetPatchStateDict.TryGetValue(assetName, out bool isPatch)) { return isPatch; } isPatch = IsChangedAsset(assetName); if (isPatch) { assetPatchStateDict.Add(assetName,true ); return true ; } if (upStreamDict.TryGetValue(assetName, out var upStreamList)) { foreach (string upStream in upStreamList) { isPatch = IsPatchAsset(upStream); if (isPatch) { assetPatchStateDict.Add(assetName,true ); return true ; } } } assetPatchStateDict.Add(assetName,false ); return false ; } private bool IsChangedAsset (string assetName ) { if (assetChangeStateDict.TryGetValue(assetName, out bool isPatch)) { return isPatch; } if (!assetCacheDict.TryGetValue(assetName, out var assetCache)) { assetChangeStateDict.Add(assetName, true ); return true ; } if (!curAssetCacheDict.TryGetValue(assetName, out var curAssetCache)) { curAssetCache = AssetCacheManifest.AssetCacheInfo.Create(assetName); curAssetCacheDict.Add(assetName, curAssetCache); } if (curAssetCache != assetCache) { assetChangeStateDict.Add(assetName, true ); return true ; } if (assetToBundle[assetName] != cacheAssetToBundle[assetName]) { assetChangeStateDict.Add(assetName, true ); return true ; } assetChangeStateDict.Add(assetName, false ); return false ; } } }
上述代码会在完整构建的资源列表基础上,移除所有非补丁资源,以及不包含补丁资源的资源包,以此形成用于最终补丁构建的资源列表
另外值得一提的是,为了降低补丁包带来的复杂度与资源冗余,CatAsset规定了每个正式包最多只会有一个补丁包,同时如果某个补丁包包含了正式包的所有资源,则此补丁包会自动转正为正式包
合并资源清单 在构建补丁包完成后,需要将新的补丁资源包与上一次完整构建时输出的资源包进行合并,以得到最终输出的完整资源包与资源清单
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 using System.Collections.Generic;using System.IO;using CatAsset.Runtime;using UnityEditor;using UnityEditor.Build.Pipeline;using UnityEditor.Build.Pipeline.Injector;using UnityEditor.Build.Pipeline.Interfaces;using UnityEngine;namespace CatAsset.Editor { public class MergePatchManifest : IBuildTask { [InjectContext(ContextUsage.In) ] private IBundleBuildParameters buildParam; [InjectContext(ContextUsage.In) ] private IBundleBuildConfigParam configParam; [InjectContext(ContextUsage.InOut) ] private IManifestParam manifestParam; public int Version => 1 ; public ReturnCode Run ( ) { var config = configParam.Config; var bundleCacheFolder = EditorUtil.GetBundleCacheFolder(config.OutputRootDirectory, configParam.TargetPlatform); HashSet<string > patchAssets = new HashSet<string >(); foreach (BundleManifestInfo bundleManifestInfo in manifestParam.Manifest.Bundles) { foreach (AssetManifestInfo assetManifestInfo in bundleManifestInfo.Assets) { patchAssets.Add(assetManifestInfo.Name); } } string path = RuntimeUtil.GetRegularPath(Path.Combine(bundleCacheFolder, CatAssetManifest.ManifestJsonFileName)); CatAssetManifest cachedManifest = CatAssetManifest.DeserializeFromJson(File.ReadAllText(path)); for (int i = cachedManifest.Bundles.Count - 1 ; i >= 0 ; i--) { BundleManifestInfo bundleManifestInfo = cachedManifest.Bundles[i]; if (bundleManifestInfo.BundleName == RuntimeUtil.BuiltInShaderBundleName) { continue ; } for (int j = bundleManifestInfo.Assets.Count - 1 ; j >= 0 ; j--) { AssetManifestInfo assetManifestInfo = bundleManifestInfo.Assets[j]; if (patchAssets.Contains(assetManifestInfo.Name)) { bundleManifestInfo.Assets.RemoveAt(j); } } if (bundleManifestInfo.Assets.Count == 0 ) { cachedManifest.Bundles.RemoveAt(i); } } string outputFolder = ((BundleBuildParameters)buildParam).OutputFolder; foreach (BundleManifestInfo bundleManifestInfo in cachedManifest.Bundles) { string sourcePath = RuntimeUtil.GetRegularPath(Path.Combine(bundleCacheFolder, bundleManifestInfo.RelativePath)); string destPath = RuntimeUtil.GetRegularPath(Path.Combine(outputFolder,bundleManifestInfo.RelativePath)); if (!string .IsNullOrEmpty(bundleManifestInfo.Directory)) { string fullDirectory = RuntimeUtil.GetRegularPath(Path.Combine(outputFolder,bundleManifestInfo.Directory)); if (!Directory.Exists(fullDirectory)) { Directory.CreateDirectory(fullDirectory); } } File.Copy(sourcePath,destPath); } cachedManifest.Bundles.AddRange(manifestParam.Manifest.Bundles); manifestParam = new ManifestParam(cachedManifest, manifestParam.WriteFolder); return ReturnCode.Success; } } }
此操作会将在原来被缓存的资源清单中存在的补丁资源, 以及所有资源都是补丁资源的资源包从原资源清单中移除,然后与补丁构建产生的资源清单合并,最终得到一份新的资源清单用于正确的初始化资源信息
以之前的例子来说,补丁构建产生的资源清单只会包含A_patch和资源b的信息,并且会将原资源清单中的资源b信息移除,然后将二者进行合并以得到新的资源清单