Skip to content

Feat: FiveM resource-splitting #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tencreator wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Feat: FiveM resource-splitting #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fcd7649
Added resource-splitting & fixed a bug when attempting to build twice…
tencreator Dec 4, 2025
87b27d3
Now it actually keeps the main resource under 1GB in size, small enou…
tencreator Dec 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions grzyClothTool/Constants/GlobalConstants.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ public static class GlobalConstants
{
public const int MAX_DRAWABLES_IN_ADDON = 128; //todo: somewhere in the future, this should depend on a resource type (fivem has limit of 128, but sp and ragemp have 255 I believe)
public const int MAX_DRAWABLE_TEXTURES = 26;
public const long MAX_RESOURCE_SIZE_BYTES = 838860800; // 800 MB in bytes
public const long MAX_MAIN_RESOURCE_SIZE_BYTES = 734003200; // 700 MB in bytes (800 * 1024 * 1024)
public static readonly Uri DISCORD_INVITE_URL = new("https://discord.gg/HCQutNhxWt");
public static readonly string GRZY_TOOLS_URL = "https://grzy.tools";
}
248 changes: 224 additions & 24 deletions grzyClothTool/Helpers/BuildResourceHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
using CodeWalker.GameFiles;
using CodeWalker.GameFiles;
using CodeWalker.Utils;
using grzyClothTool.Constants;
using grzyClothTool.Models;
Expand All @@ -25,22 +24,30 @@ public class BuildResourceHelper
private readonly string _projectName;
private string _buildPath;
private readonly string _baseBuildPath;
private readonly string _parentBuildPath; // Parent directory for creating extra resources as siblings
private readonly bool _splitAddons;
private readonly bool _splitBySize;
private readonly IProgress<int> _progress;

private readonly bool shouldUseNumber = false;

private readonly List<string> firstPersonFiles = [];
private BuildResourceType _buildResourceType;

// Size tracking for resource splitting
private long _currentResourceSize = 0;
private int _extraResourceCounter = 0;

public BuildResourceHelper(string name, string path, IProgress<int> progress, BuildResourceType resourceType, bool splitAddons)
public BuildResourceHelper(string name, string path, IProgress<int> progress, BuildResourceType resourceType, bool splitAddons, bool splitBySize = false)
{
_projectName = name;
_buildPath = path;
_baseBuildPath = path; // store base build path, as we will be modyfiyng _buildPath, when splitting addons
_parentBuildPath = Directory.GetParent(path)?.FullName ?? path; // Store parent directory for extra resources
_progress = progress;
_buildResourceType = resourceType;
_splitAddons = splitAddons;
_splitBySize = splitBySize;

shouldUseNumber = MainWindow.AddonManager.Addons.Count > 1;
}
Expand All @@ -65,6 +72,59 @@ public string GetProjectName(int? number = null)
number ??= _number;
return shouldUseNumber ? $"{_projectName}_{number:D2}" : _projectName;
}

public string GetProjectNameWithExtra()
{
var baseName = GetProjectName();
return _extraResourceCounter > 0 ? $"{baseName}-EXTRA_{_extraResourceCounter}" : baseName;
}

public void ResetSizeTracking()
{
_currentResourceSize = 0;
_extraResourceCounter = 0;
}

public bool ShouldCreateNewExtraResource(long nextFileSize = 0)
{
if (!_splitBySize || _buildResourceType != BuildResourceType.FiveM)
return false;

// Use 700MB limit for main resource, 800MB for extra resources
long sizeLimit = _extraResourceCounter == 0
? GlobalConstants.MAX_MAIN_RESOURCE_SIZE_BYTES
: GlobalConstants.MAX_RESOURCE_SIZE_BYTES;

return (_currentResourceSize + nextFileSize) >= sizeLimit;
}

public void AddToResourceSize(long bytes)
{
_currentResourceSize += bytes;
}

public long GetFileSize(string filePath)
{
try
{
return new FileInfo(filePath).Length;
}
catch
{
return 0;
}
}

public void IncrementExtraCounter()
{
_extraResourceCounter++;
_currentResourceSize = 0;
}

public bool IsInExtraResource()
{
return _extraResourceCounter > 0;
}


#region FiveM
Expand All @@ -74,6 +134,7 @@ private async Task BuildFiveMFilesAsync(SexType sex, byte[] ymtBytes, int counte
{
var pedName = GetPedName(sex);
var projectName = GetProjectName(counter);
var originalBuildPath = _buildPath; // Store original path to restore later

var drawables = _addon.Drawables.Where(x => x.Sex == sex).ToList();
var drawableGroups = drawables.Select((x, i) => new { Index = i, Value = x })
Expand All @@ -90,12 +151,86 @@ private async Task BuildFiveMFilesAsync(SexType sex, byte[] ymtBytes, int counte

var ymtPath = Path.Combine(streamDirectory, $"{pedName}_{projectName}.ymt");
fileOperations.Add(File.WriteAllBytesAsync(ymtPath, ymtBytes));
AddToResourceSize(ymtBytes.Length);

foreach (var group in drawableGroups)
{
foreach (var d in group)
{
var tempYddPath = yddPathsDict[d];
var yddFileSize = GetFileSize(tempYddPath);

// Pre-calculate actual texture sizes (since optimization changes file size)
var textureData = new List<(string buildName, byte[] data)>();
long texturesSizeTotal = 0;
foreach (var t in d.Textures)
{
var buildName = RemoveInvalidChars(t.GetBuildName());
byte[] txtBytes = null;

if (t.IsOptimizedDuringBuild)
{
txtBytes = await ImgHelper.Optimize(t);
}
else
{
if(t.Extension != ".ytd")
{
txtBytes = await ImgHelper.Optimize(t, true);
}
else
{
txtBytes = await FileHelper.ReadAllBytesAsync(t.FilePath);
}
}

if (txtBytes != null)
{
textureData.Add((buildName, txtBytes));
texturesSizeTotal += txtBytes.Length;
}
}

// Calculate total size with actual processed sizes
long totalDrawableSize = yddFileSize + texturesSizeTotal;
if (!string.IsNullOrEmpty(d.ClothPhysicsPath))
{
totalDrawableSize += GetFileSize(d.ClothPhysicsPath);
}
if (!string.IsNullOrEmpty(d.FirstPersonPath))
{
totalDrawableSize += GetFileSize(d.FirstPersonPath);
}

// Check if we need to split into a new resource (only for stream files when size splitting is enabled)
if (ShouldCreateNewExtraResource(totalDrawableSize) && totalDrawableSize > 0)
{
// Wait for pending operations to complete before creating new resource
if (fileOperations.Count > 0)
{
await Task.WhenAll(fileOperations);
fileOperations.Clear();
}

// Create manifest for current resource (only if we're already in an extra resource)
if (IsInExtraResource())
{
await CreateSizeSplitFxManifest(sex, projectName);
}

// Move to next resource (first extra or subsequent)
IncrementExtraCounter();

// Update build path for new extra resource (create as sibling, not subdirectory)
_buildPath = Path.Combine(_parentBuildPath, GetProjectNameWithExtra());
streamDirectory = Path.Combine(_buildPath, "stream");
Directory.CreateDirectory(streamDirectory);

// Copy YMT to new resource and wait for it to complete
ymtPath = Path.Combine(streamDirectory, $"{pedName}_{projectName}.ymt");
await File.WriteAllBytesAsync(ymtPath, ymtBytes);
AddToResourceSize(ymtBytes.Length);
}

var drawablePedName = d.IsProp ? $"{pedName}_p" : pedName;

Expand All @@ -105,45 +240,32 @@ private async Task BuildFiveMFilesAsync(SexType sex, byte[] ymtBytes, int counte
var prefix = RemoveInvalidChars($"{drawablePedName}_{projectName}^");
var finalPath = Path.Combine(folderPath, $"{prefix}{d.Name}{Path.GetExtension(d.FilePath)}");
fileOperations.Add(FileHelper.CopyAsync(tempYddPath, finalPath));
AddToResourceSize(yddFileSize);

if (!string.IsNullOrEmpty(d.ClothPhysicsPath))
{
var clothSize = GetFileSize(d.ClothPhysicsPath);
fileOperations.Add(FileHelper.CopyAsync(d.ClothPhysicsPath, Path.Combine(folderPath, $"{prefix}{d.Name}{Path.GetExtension(d.ClothPhysicsPath)}")));
AddToResourceSize(clothSize);
}

if (!string.IsNullOrEmpty(d.FirstPersonPath))
{
//todo: this probably shouldn't be hardcoded to "_1", handle it when there is option to add more alternate drawable versions
var fpSize = GetFileSize(d.FirstPersonPath);
fileOperations.Add(FileHelper.CopyAsync(d.FirstPersonPath, Path.Combine(folderPath, $"{prefix}{d.Name}_1{Path.GetExtension(d.FirstPersonPath)}")));
AddToResourceSize(fpSize);

var name = $"{prefix}{d.Name}".Replace("^", "/");
firstPersonFiles.Add(name);
}

foreach (var t in d.Textures)
// Write pre-processed textures
foreach (var (buildName, txtBytes) in textureData)
{
var buildName = RemoveInvalidChars(t.GetBuildName());
var finalTexPath = Path.Combine(folderPath, $"{prefix}{buildName}.ytd");

byte[] txtBytes = null;
if (t.IsOptimizedDuringBuild)
{
txtBytes = await ImgHelper.Optimize(t);
fileOperations.Add(File.WriteAllBytesAsync(finalTexPath, txtBytes));
}
else
{
if(t.Extension != ".ytd")
{
txtBytes = await ImgHelper.Optimize(t, true);
}
else
{
txtBytes = await FileHelper.ReadAllBytesAsync(t.FilePath);
}

fileOperations.Add(File.WriteAllBytesAsync(finalTexPath, txtBytes));
}
fileOperations.Add(File.WriteAllBytesAsync(finalTexPath, txtBytes));
AddToResourceSize(txtBytes.Length);
}
}
}
Expand Down Expand Up @@ -172,6 +294,9 @@ await Task.Run(() =>
generated?.Save(streamDirectory + "/mp_creaturemetadata_" + GetGenderLetter(sex) + "_" + projectName + ".ymt");
}
);

// Restore original build path after processing
_buildPath = originalBuildPath;
}

public async Task BuildFiveMResource()
Expand All @@ -191,11 +316,23 @@ public async Task BuildFiveMResource()
SetAddon(selectedAddon);
SetNumber(counter);
UpdateBuildPath();
ResetSizeTracking(); // Reset size tracking for each addon when splitting by addons

AddBuildTasksForSex(selectedAddon, SexType.male, tasks, metaFiles, counter);
AddBuildTasksForSex(selectedAddon, SexType.female, tasks, metaFiles, counter);

await Task.WhenAll(tasks);

// If size splitting is enabled and extra resources were created, finalize the last one
if (_splitBySize && _extraResourceCounter > 0)
{
// The last extra resource needs its manifest
await FinalizeSizeSplitResource(selectedAddon);

// Restore the original build path for meta files
_buildPath = Path.Combine(_baseBuildPath, GetProjectName());
}

BuildFirstPersonAlternatesMeta();
BuildPedAlternativeVariationsMeta();
BuildFxManifest(metaFiles);
Expand All @@ -209,6 +346,8 @@ public async Task BuildFiveMResource()
}
else
{
ResetSizeTracking(); // Reset size tracking when not splitting by addons

foreach (var selectedAddon in MainWindow.AddonManager.Addons)
{
SetAddon(selectedAddon);
Expand All @@ -221,12 +360,67 @@ public async Task BuildFiveMResource()
}

await Task.WhenAll(tasks);

// If size splitting is enabled and extra resources were created, finalize the last one
if (_splitBySize && _extraResourceCounter > 0)
{
await FinalizeSizeSplitResource(null);

// Restore the original build path for meta files
_buildPath = _baseBuildPath;
}

BuildFirstPersonAlternatesMeta();
BuildPedAlternativeVariationsMeta();
BuildFxManifest(metaFiles);
}

}

private async Task FinalizeSizeSplitResource(Addon addon)
{
// Create the final fxmanifest for the last extra resource
if (_extraResourceCounter > 0)
{
var currentPath = _buildPath;
_buildPath = Path.Combine(_parentBuildPath, GetProjectNameWithExtra());

StringBuilder contentBuilder = new();
contentBuilder.AppendLine("-- This resource was generated by grzyClothTool :)");
contentBuilder.AppendLine($"-- {GlobalConstants.DISCORD_INVITE_URL}");
contentBuilder.AppendLine($"-- EXTRA resource {_extraResourceCounter} for size splitting (stream files only)");
contentBuilder.AppendLine();
contentBuilder.AppendLine("fx_version 'cerulean'");
contentBuilder.AppendLine("game 'gta5'");
contentBuilder.AppendLine("author 'grzyClothTool'");

var finalPath = Path.Combine(_buildPath, "fxmanifest.lua");
await File.WriteAllTextAsync(finalPath, contentBuilder.ToString());

_buildPath = currentPath;
}
}

private async Task CreateSizeSplitFxManifest(SexType sex, string projectName)
{
var pedName = GetPedName(sex);
var metaFileName = $"{pedName}_{projectName}.meta";

StringBuilder contentBuilder = new();
contentBuilder.AppendLine("-- This resource was generated by grzyClothTool :)");
contentBuilder.AppendLine($"-- {GlobalConstants.DISCORD_INVITE_URL}");
contentBuilder.AppendLine($"-- EXTRA resource {_extraResourceCounter} for size splitting");
contentBuilder.AppendLine();
contentBuilder.AppendLine("fx_version 'cerulean'");
contentBuilder.AppendLine("game 'gta5'");
contentBuilder.AppendLine("author 'grzyClothTool'");
contentBuilder.AppendLine();

// Only include stream files, no meta files in extra resources

var finalPath = Path.Combine(_buildPath, "fxmanifest.lua");
await File.WriteAllTextAsync(finalPath, contentBuilder.ToString());
}

private void BuildFxManifest(List<string> metaFiles)
{
Expand Down Expand Up @@ -1232,6 +1426,12 @@ public static async Task<string> ResaveYdd(GDrawable dr)
string uniqueFileName = $"{dr.Id}_{Path.GetFileName(inputPath)}";
string outputPath = Path.Combine(tempDir, uniqueFileName);

// If temp file already exists, return it (happens when building multiple resources with same drawables)
if (File.Exists(outputPath))
{
return outputPath;
}

// If drawable is encrypted or has no embedded textures, just copy the original file without processing
if (dr?.IsEncrypted == true || dr.Details?.EmbeddedTextures == null || dr.Details.EmbeddedTextures.Count == 0 || dr.Details.EmbeddedTextures.All(x => x.Value.Details.Width == 0))
{
Expand Down
2 changes: 1 addition & 1 deletion grzyClothTool/Helpers/FileHelper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,8 @@ public static async Task<GDrawable> CreateDrawableAsync(string filePath, Enums.S

public static async Task CopyAsync(string sourcePath, string destinationPath)
{
using var sourceStream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan);
using var destinationStream = new FileStream(destinationPath, FileMode.CreateNew, FileAccess.Write, FileShare.None, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan);
using var sourceStream = new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan);
await sourceStream.CopyToAsync(destinationStream);
}

Expand Down
Loading