diff --git a/grzyClothTool/Constants/GlobalConstants.cs b/grzyClothTool/Constants/GlobalConstants.cs index 9d27bcd..a5f9b62 100644 --- a/grzyClothTool/Constants/GlobalConstants.cs +++ b/grzyClothTool/Constants/GlobalConstants.cs @@ -6,6 +6,8 @@ public static class GlobalConstants { public const int MAX_DRAWABLES_IN_ADDON = 128; //todo: somewhere in the future, this should depend on a resource type (fivem has limit of 128, but sp and ragemp have 255 I believe) public const int MAX_DRAWABLE_TEXTURES = 26; + public const long MAX_RESOURCE_SIZE_BYTES = 838860800; // 800 MB in bytes + public const long MAX_MAIN_RESOURCE_SIZE_BYTES = 734003200; // 700 MB in bytes (800 * 1024 * 1024) public static readonly Uri DISCORD_INVITE_URL = new("https://discord.gg/HCQutNhxWt"); public static readonly string GRZY_TOOLS_URL = "https://grzy.tools"; } diff --git a/grzyClothTool/Helpers/BuildResourceHelper.cs b/grzyClothTool/Helpers/BuildResourceHelper.cs index 0688bd3..04dac21 100644 --- a/grzyClothTool/Helpers/BuildResourceHelper.cs +++ b/grzyClothTool/Helpers/BuildResourceHelper.cs @@ -1,5 +1,4 @@ using CodeWalker.GameFiles; -using CodeWalker.GameFiles; using CodeWalker.Utils; using grzyClothTool.Constants; using grzyClothTool.Models; @@ -25,22 +24,30 @@ public class BuildResourceHelper private readonly string _projectName; private string _buildPath; private readonly string _baseBuildPath; + private readonly string _parentBuildPath; // Parent directory for creating extra resources as siblings private readonly bool _splitAddons; + private readonly bool _splitBySize; private readonly IProgress _progress; private readonly bool shouldUseNumber = false; private readonly List firstPersonFiles = []; private BuildResourceType _buildResourceType; + + // Size tracking for resource splitting + private long _currentResourceSize = 0; + private int _extraResourceCounter = 0; - public BuildResourceHelper(string name, string path, IProgress progress, BuildResourceType resourceType, bool splitAddons) + public BuildResourceHelper(string name, string path, IProgress progress, BuildResourceType resourceType, bool splitAddons, bool splitBySize = false) { _projectName = name; _buildPath = path; _baseBuildPath = path; // store base build path, as we will be modyfiyng _buildPath, when splitting addons + _parentBuildPath = Directory.GetParent(path)?.FullName ?? path; // Store parent directory for extra resources _progress = progress; _buildResourceType = resourceType; _splitAddons = splitAddons; + _splitBySize = splitBySize; shouldUseNumber = MainWindow.AddonManager.Addons.Count > 1; } @@ -65,6 +72,59 @@ public string GetProjectName(int? number = null) number ??= _number; return shouldUseNumber ? $"{_projectName}_{number:D2}" : _projectName; } + + public string GetProjectNameWithExtra() + { + var baseName = GetProjectName(); + return _extraResourceCounter > 0 ? $"{baseName}-EXTRA_{_extraResourceCounter}" : baseName; + } + + public void ResetSizeTracking() + { + _currentResourceSize = 0; + _extraResourceCounter = 0; + } + + public bool ShouldCreateNewExtraResource(long nextFileSize = 0) + { + if (!_splitBySize || _buildResourceType != BuildResourceType.FiveM) + return false; + + // Use 700MB limit for main resource, 800MB for extra resources + long sizeLimit = _extraResourceCounter == 0 + ? GlobalConstants.MAX_MAIN_RESOURCE_SIZE_BYTES + : GlobalConstants.MAX_RESOURCE_SIZE_BYTES; + + return (_currentResourceSize + nextFileSize) >= sizeLimit; + } + + public void AddToResourceSize(long bytes) + { + _currentResourceSize += bytes; + } + + public long GetFileSize(string filePath) + { + try + { + return new FileInfo(filePath).Length; + } + catch + { + return 0; + } + } + + public void IncrementExtraCounter() + { + _extraResourceCounter++; + _currentResourceSize = 0; + } + + public bool IsInExtraResource() + { + return _extraResourceCounter > 0; + } #region FiveM @@ -74,6 +134,7 @@ private async Task BuildFiveMFilesAsync(SexType sex, byte[] ymtBytes, int counte { var pedName = GetPedName(sex); var projectName = GetProjectName(counter); + var originalBuildPath = _buildPath; // Store original path to restore later var drawables = _addon.Drawables.Where(x => x.Sex == sex).ToList(); var drawableGroups = drawables.Select((x, i) => new { Index = i, Value = x }) @@ -90,12 +151,86 @@ private async Task BuildFiveMFilesAsync(SexType sex, byte[] ymtBytes, int counte var ymtPath = Path.Combine(streamDirectory, $"{pedName}_{projectName}.ymt"); fileOperations.Add(File.WriteAllBytesAsync(ymtPath, ymtBytes)); + AddToResourceSize(ymtBytes.Length); foreach (var group in drawableGroups) { foreach (var d in group) { var tempYddPath = yddPathsDict[d]; + var yddFileSize = GetFileSize(tempYddPath); + + // Pre-calculate actual texture sizes (since optimization changes file size) + var textureData = new List<(string buildName, byte[] data)>(); + long texturesSizeTotal = 0; + foreach (var t in d.Textures) + { + var buildName = RemoveInvalidChars(t.GetBuildName()); + byte[] txtBytes = null; + + if (t.IsOptimizedDuringBuild) + { + txtBytes = await ImgHelper.Optimize(t); + } + else + { + if(t.Extension != ".ytd") + { + txtBytes = await ImgHelper.Optimize(t, true); + } + else + { + txtBytes = await FileHelper.ReadAllBytesAsync(t.FilePath); + } + } + + if (txtBytes != null) + { + textureData.Add((buildName, txtBytes)); + texturesSizeTotal += txtBytes.Length; + } + } + + // Calculate total size with actual processed sizes + long totalDrawableSize = yddFileSize + texturesSizeTotal; + if (!string.IsNullOrEmpty(d.ClothPhysicsPath)) + { + totalDrawableSize += GetFileSize(d.ClothPhysicsPath); + } + if (!string.IsNullOrEmpty(d.FirstPersonPath)) + { + totalDrawableSize += GetFileSize(d.FirstPersonPath); + } + + // Check if we need to split into a new resource (only for stream files when size splitting is enabled) + if (ShouldCreateNewExtraResource(totalDrawableSize) && totalDrawableSize > 0) + { + // Wait for pending operations to complete before creating new resource + if (fileOperations.Count > 0) + { + await Task.WhenAll(fileOperations); + fileOperations.Clear(); + } + + // Create manifest for current resource (only if we're already in an extra resource) + if (IsInExtraResource()) + { + await CreateSizeSplitFxManifest(sex, projectName); + } + + // Move to next resource (first extra or subsequent) + IncrementExtraCounter(); + + // Update build path for new extra resource (create as sibling, not subdirectory) + _buildPath = Path.Combine(_parentBuildPath, GetProjectNameWithExtra()); + streamDirectory = Path.Combine(_buildPath, "stream"); + Directory.CreateDirectory(streamDirectory); + + // Copy YMT to new resource and wait for it to complete + ymtPath = Path.Combine(streamDirectory, $"{pedName}_{projectName}.ymt"); + await File.WriteAllBytesAsync(ymtPath, ymtBytes); + AddToResourceSize(ymtBytes.Length); + } var drawablePedName = d.IsProp ? $"{pedName}_p" : pedName; @@ -105,45 +240,32 @@ private async Task BuildFiveMFilesAsync(SexType sex, byte[] ymtBytes, int counte var prefix = RemoveInvalidChars($"{drawablePedName}_{projectName}^"); var finalPath = Path.Combine(folderPath, $"{prefix}{d.Name}{Path.GetExtension(d.FilePath)}"); fileOperations.Add(FileHelper.CopyAsync(tempYddPath, finalPath)); + AddToResourceSize(yddFileSize); if (!string.IsNullOrEmpty(d.ClothPhysicsPath)) { + var clothSize = GetFileSize(d.ClothPhysicsPath); fileOperations.Add(FileHelper.CopyAsync(d.ClothPhysicsPath, Path.Combine(folderPath, $"{prefix}{d.Name}{Path.GetExtension(d.ClothPhysicsPath)}"))); + AddToResourceSize(clothSize); } if (!string.IsNullOrEmpty(d.FirstPersonPath)) { //todo: this probably shouldn't be hardcoded to "_1", handle it when there is option to add more alternate drawable versions + var fpSize = GetFileSize(d.FirstPersonPath); fileOperations.Add(FileHelper.CopyAsync(d.FirstPersonPath, Path.Combine(folderPath, $"{prefix}{d.Name}_1{Path.GetExtension(d.FirstPersonPath)}"))); + AddToResourceSize(fpSize); var name = $"{prefix}{d.Name}".Replace("^", "/"); firstPersonFiles.Add(name); } - foreach (var t in d.Textures) + // Write pre-processed textures + foreach (var (buildName, txtBytes) in textureData) { - var buildName = RemoveInvalidChars(t.GetBuildName()); var finalTexPath = Path.Combine(folderPath, $"{prefix}{buildName}.ytd"); - - byte[] txtBytes = null; - if (t.IsOptimizedDuringBuild) - { - txtBytes = await ImgHelper.Optimize(t); - fileOperations.Add(File.WriteAllBytesAsync(finalTexPath, txtBytes)); - } - else - { - if(t.Extension != ".ytd") - { - txtBytes = await ImgHelper.Optimize(t, true); - } - else - { - txtBytes = await FileHelper.ReadAllBytesAsync(t.FilePath); - } - - fileOperations.Add(File.WriteAllBytesAsync(finalTexPath, txtBytes)); - } + fileOperations.Add(File.WriteAllBytesAsync(finalTexPath, txtBytes)); + AddToResourceSize(txtBytes.Length); } } } @@ -172,6 +294,9 @@ await Task.Run(() => generated?.Save(streamDirectory + "/mp_creaturemetadata_" + GetGenderLetter(sex) + "_" + projectName + ".ymt"); } ); + + // Restore original build path after processing + _buildPath = originalBuildPath; } public async Task BuildFiveMResource() @@ -191,11 +316,23 @@ public async Task BuildFiveMResource() SetAddon(selectedAddon); SetNumber(counter); UpdateBuildPath(); + ResetSizeTracking(); // Reset size tracking for each addon when splitting by addons AddBuildTasksForSex(selectedAddon, SexType.male, tasks, metaFiles, counter); AddBuildTasksForSex(selectedAddon, SexType.female, tasks, metaFiles, counter); await Task.WhenAll(tasks); + + // If size splitting is enabled and extra resources were created, finalize the last one + if (_splitBySize && _extraResourceCounter > 0) + { + // The last extra resource needs its manifest + await FinalizeSizeSplitResource(selectedAddon); + + // Restore the original build path for meta files + _buildPath = Path.Combine(_baseBuildPath, GetProjectName()); + } + BuildFirstPersonAlternatesMeta(); BuildPedAlternativeVariationsMeta(); BuildFxManifest(metaFiles); @@ -209,6 +346,8 @@ public async Task BuildFiveMResource() } else { + ResetSizeTracking(); // Reset size tracking when not splitting by addons + foreach (var selectedAddon in MainWindow.AddonManager.Addons) { SetAddon(selectedAddon); @@ -221,12 +360,67 @@ public async Task BuildFiveMResource() } await Task.WhenAll(tasks); + + // If size splitting is enabled and extra resources were created, finalize the last one + if (_splitBySize && _extraResourceCounter > 0) + { + await FinalizeSizeSplitResource(null); + + // Restore the original build path for meta files + _buildPath = _baseBuildPath; + } + BuildFirstPersonAlternatesMeta(); BuildPedAlternativeVariationsMeta(); BuildFxManifest(metaFiles); } } + + private async Task FinalizeSizeSplitResource(Addon addon) + { + // Create the final fxmanifest for the last extra resource + if (_extraResourceCounter > 0) + { + var currentPath = _buildPath; + _buildPath = Path.Combine(_parentBuildPath, GetProjectNameWithExtra()); + + StringBuilder contentBuilder = new(); + contentBuilder.AppendLine("-- This resource was generated by grzyClothTool :)"); + contentBuilder.AppendLine($"-- {GlobalConstants.DISCORD_INVITE_URL}"); + contentBuilder.AppendLine($"-- EXTRA resource {_extraResourceCounter} for size splitting (stream files only)"); + contentBuilder.AppendLine(); + contentBuilder.AppendLine("fx_version 'cerulean'"); + contentBuilder.AppendLine("game 'gta5'"); + contentBuilder.AppendLine("author 'grzyClothTool'"); + + var finalPath = Path.Combine(_buildPath, "fxmanifest.lua"); + await File.WriteAllTextAsync(finalPath, contentBuilder.ToString()); + + _buildPath = currentPath; + } + } + + private async Task CreateSizeSplitFxManifest(SexType sex, string projectName) + { + var pedName = GetPedName(sex); + var metaFileName = $"{pedName}_{projectName}.meta"; + + StringBuilder contentBuilder = new(); + contentBuilder.AppendLine("-- This resource was generated by grzyClothTool :)"); + contentBuilder.AppendLine($"-- {GlobalConstants.DISCORD_INVITE_URL}"); + contentBuilder.AppendLine($"-- EXTRA resource {_extraResourceCounter} for size splitting"); + contentBuilder.AppendLine(); + contentBuilder.AppendLine("fx_version 'cerulean'"); + contentBuilder.AppendLine("game 'gta5'"); + contentBuilder.AppendLine("author 'grzyClothTool'"); + contentBuilder.AppendLine(); + + // Only include stream files, no meta files in extra resources + + var finalPath = Path.Combine(_buildPath, "fxmanifest.lua"); + await File.WriteAllTextAsync(finalPath, contentBuilder.ToString()); + } private void BuildFxManifest(List metaFiles) { @@ -1232,6 +1426,12 @@ public static async Task ResaveYdd(GDrawable dr) string uniqueFileName = $"{dr.Id}_{Path.GetFileName(inputPath)}"; string outputPath = Path.Combine(tempDir, uniqueFileName); + // If temp file already exists, return it (happens when building multiple resources with same drawables) + if (File.Exists(outputPath)) + { + return outputPath; + } + // If drawable is encrypted or has no embedded textures, just copy the original file without processing if (dr?.IsEncrypted == true || dr.Details?.EmbeddedTextures == null || dr.Details.EmbeddedTextures.Count == 0 || dr.Details.EmbeddedTextures.All(x => x.Value.Details.Width == 0)) { diff --git a/grzyClothTool/Helpers/FileHelper.cs b/grzyClothTool/Helpers/FileHelper.cs index 8e8eeca..264aa03 100644 --- a/grzyClothTool/Helpers/FileHelper.cs +++ b/grzyClothTool/Helpers/FileHelper.cs @@ -100,8 +100,8 @@ public static async Task CreateDrawableAsync(string filePath, Enums.S public static async Task CopyAsync(string sourcePath, string destinationPath) { - using var sourceStream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan); using var destinationStream = new FileStream(destinationPath, FileMode.CreateNew, FileAccess.Write, FileShare.None, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan); + using var sourceStream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan); await sourceStream.CopyToAsync(destinationStream); } diff --git a/grzyClothTool/Views/BuildWindow.xaml b/grzyClothTool/Views/BuildWindow.xaml index d293417..d9b09ae 100644 --- a/grzyClothTool/Views/BuildWindow.xaml +++ b/grzyClothTool/Views/BuildWindow.xaml @@ -69,6 +69,13 @@ Margin="10,10,10,0" IsChecked="{Binding SplitAddons, Mode=TwoWay}"/> + + _splitBySize; + set + { + if (_splitBySize != value) + { + _splitBySize = value; + OnPropertyChanged(nameof(SplitBySize)); + } + } + } + private bool _isWarningVisible; public bool IsWarningVisible { @@ -126,6 +140,7 @@ public BuildWindow() private void Window_Loaded(object sender, RoutedEventArgs e) { split_addons.IsEnabled = MainWindow.AddonManager.Addons.Count > 1; + split_by_size.IsEnabled = true; // Enabled by default for FiveM (the default selection) CheckAddons(); } @@ -204,7 +219,7 @@ private async void build_MyBtnClickEvent(object sender, RoutedEventArgs e) timer.Start(); var progress = new Progress(value => ProgressValue += value); - var buildHelper = new BuildResourceHelper(ProjectName, BuildPath, progress, _resourceType, SplitAddons); + var buildHelper = new BuildResourceHelper(ProjectName, BuildPath, progress, _resourceType, SplitAddons, SplitBySize); await Task.Run(() => BuildResource(buildHelper)); // moved out of ui thread, so users don't think tool stopped responding @@ -244,15 +259,29 @@ private void RadioButton_ChangedEvent(object sender, RoutedEventArgs e) }; - // Singleplayer doesn't support splitting addons + // Singleplayer doesn't support splitting addons or by size if (_resourceType == BuildResourceType.Singleplayer) { SplitAddons = false; split_addons.IsEnabled = false; + SplitBySize = false; + split_by_size.IsEnabled = false; + } + // AltV doesn't support splitting by size + else if (_resourceType == BuildResourceType.AltV) + { + SplitBySize = false; + split_by_size.IsEnabled = false; + if (DataContext != null) // check if DataContext exist, to prevent error (happens on initialization) + { + split_addons.IsEnabled = MainWindow.AddonManager.Addons.Count > 1; + } } + // FiveM supports both options else if (DataContext != null) // check if DataContext exist, to prevent error (happens on initialization) { split_addons.IsEnabled = MainWindow.AddonManager.Addons.Count > 1; + split_by_size.IsEnabled = true; } } }