Skip to content

Trying to fix memory using AI #275

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ArGr1 wants to merge 20 commits into
base: master
Choose a base branch
from
Draft

Trying to fix memory using AI #275

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
5703065
Write hierarchy db instead of export
ArGr1 Mar 28, 2025
f936f24
Convert hierarchy nodes outside of exporting task to lessen memory load.
ArGr1 Mar 28, 2025
22b4fac
remove unnecessary garbage collection
ArGr1 Mar 28, 2025
f0f02f7
Remove writing hierarchy on side thread.
ArGr1 Apr 2, 2025
a41f141
debugging console writes
ArGr1 Apr 2, 2025
564362c
dlete console writelines
ArGr1 Apr 3, 2025
dcaed1f
Check memory usage and write to console
ArGr1 Apr 3, 2025
c9d24c1
IEnumerable instead of list
ArGr1 Apr 3, 2025
5d5ae5d
do not cast to array
ArGr1 Apr 3, 2025
ee048ab
indent lines
ArGr1 Apr 3, 2025
6e94e16
formatting
ArGr1 Apr 3, 2025
c3bb929
Really hacky batching of node dict creation.
ArGr1 Apr 3, 2025
cad236f
linting
ArGr1 Apr 3, 2025
a8ecd4e
Memory improvement suggestion node dict creation
ArGr1 Apr 7, 2025
23cdbf2
console write
ArGr1 Apr 7, 2025
a0e4bca
Batch size test
ArGr1 Apr 7, 2025
f735e5c
update to dotnet 9
ArGr1 Apr 8, 2025
35ac28a
do not use string interning in RvmProvider
ArGr1 Apr 8, 2025
6773921
writing len of different datas to console
ArGr1 Apr 8, 2025
60e6df9
Enable logging of DatabaseComposer
ArGr1 Apr 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CadRevealComposer.Exe/CadRevealComposer.Exe.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<Nullable>enable</Nullable>

<!-- all pipeline processing is culture invariant so disable globalization for improved performance and less surface area -->
Expand Down
2 changes: 1 addition & 1 deletion CadRevealComposer.Tests/CadRevealComposer.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<IsPackable>false</IsPackable>
</PropertyGroup>
Expand Down
2 changes: 1 addition & 1 deletion CadRevealComposer/CadRevealComposer.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
ο»Ώ<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<Nullable>enable</Nullable>
</PropertyGroup>

Expand Down
27 changes: 11 additions & 16 deletions CadRevealComposer/CadRevealComposerRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Threading.Tasks;
using Configuration;
using Devtools;
using IdProviders;
Expand Down Expand Up @@ -105,19 +104,10 @@ IReadOnlyList<IModelFormatProvider> modelFormatProviders

filtering.PrintFilteringStatsToConsole();

var exportHierarchyDatabaseTask = Task.Run(() =>
{
// Exporting hierarchy on side thread to allow it to run in parallel
var hierarchyExportTimer = Stopwatch.StartNew();
var databasePath = Path.GetFullPath(Path.Join(outputDirectory.FullName, "hierarchy.db"));
SceneCreator.ExportHierarchyDatabase(databasePath, nodesToExport);
Console.WriteLine(
$"Exported hierarchy database to path \"{databasePath}\" in {hierarchyExportTimer.Elapsed}"
);
});

geometriesToProcess = Simplify.OptimizeVertexCountInMeshes(geometriesToProcess);

WriteHierarchy(outputDirectory, nodesToExport);

var geometriesToProcessArray = geometriesToProcess.ToArray();
if (composerParameters.DevPrimitiveCacheFolder != null)
{
Expand All @@ -133,10 +123,6 @@ IReadOnlyList<IModelFormatProvider> modelFormatProviders
composerParameters
);

if (!exportHierarchyDatabaseTask.IsCompleted)
Console.WriteLine("Waiting for hierarchy export to complete...");
exportHierarchyDatabaseTask.Wait();

WriteParametersToParamsFile(modelParameters, composerParameters, outputDirectory);

ModifyHierarchyPostProcess(outputDirectory, splitExportResults);
Expand All @@ -145,6 +131,15 @@ IReadOnlyList<IModelFormatProvider> modelFormatProviders
Console.WriteLine($"Convert completed in {totalTimeElapsed.Elapsed}");
}

private static void WriteHierarchy(DirectoryInfo outputDirectory, IReadOnlyList<CadRevealNode> nodes)
{
var hierarchyNodes = HierarchyComposerConverter.ConvertToHierarchyNodes(nodes);
var hierarchyExportTimer = Stopwatch.StartNew();
var databasePath = Path.GetFullPath(Path.Join(outputDirectory.FullName, "hierarchy.db"));
SceneCreator.WriteToHierarchyDatabase(databasePath, hierarchyNodes);
Console.WriteLine($"Exported hierarchy database to path \"{databasePath}\" in {hierarchyExportTimer.Elapsed}");
}

public record SplitAndExportResults(List<TreeIndexSectorIdPair> TreeIndexToSectorIdDict);

public record TreeIndexSectorIdPair(uint TreeIndex, uint SectorId);
Expand Down
13 changes: 8 additions & 5 deletions CadRevealComposer/SceneCreator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using Commons.Utils;
using Configuration;
using HierarchyComposer.Functions;
using HierarchyComposer.Model;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Operations;
Expand Down Expand Up @@ -44,13 +45,15 @@ public static void ExportModelMetadata(DirectoryInfo outputDirectory, ModelMetad
File.WriteAllText(metadataPath, metadataString);
}

public static void ExportHierarchyDatabase(string databasePath, IReadOnlyList<CadRevealNode> allNodes)
public static void WriteToHierarchyDatabase(string databasePath, IReadOnlyList<HierarchyNode> allNodes)
{
var nodes = HierarchyComposerConverter.ConvertToHierarchyNodes(allNodes);

ILogger<DatabaseComposer> databaseLogger = NullLogger<DatabaseComposer>.Instance;
ILogger<DatabaseComposer> databaseLogger = LoggerFactory.Create(builder =>
{
builder.SetMinimumLevel(LogLevel.Debug); // Set the desired log level
}).CreateLogger<DatabaseComposer>();
//Logger<DatabaseComposer> databaseLogger = NullLogger<DatabaseComposer>.Instance;
var exporter = new DatabaseComposer(databaseLogger);
exporter.ComposeDatabase(nodes.ToList(), Path.GetFullPath(databasePath));
exporter.ComposeDatabase(allNodes, Path.GetFullPath(databasePath));
}

public static void AddPrioritizedSectorsToDatabase(
Expand Down
2 changes: 1 addition & 1 deletion CadRevealFbxProvider.Tests/CadRevealFbxProvider.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
ο»Ώ<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>

Expand Down
2 changes: 1 addition & 1 deletion CadRevealFbxProvider/CadRevealFbxProvider.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
Expand Down
2 changes: 1 addition & 1 deletion CadRevealObjProvider.Tests/CadRevealObjProvider.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>

Expand Down
2 changes: 1 addition & 1 deletion CadRevealObjProvider/CadRevealObjProvider.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
Expand Down
2 changes: 1 addition & 1 deletion CadRevealRvmProvider.Tests/CadRevealRvmProvider.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
ο»Ώ<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<IsPackable>false</IsPackable>
Expand Down
2 changes: 1 addition & 1 deletion CadRevealRvmProvider/CadRevealRvmProvider.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
ο»Ώ<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
Expand Down
2 changes: 1 addition & 1 deletion CadRevealRvmProvider/RvmProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ NodeNameFiltering nodeNameFiltering
});

var stringInternPool = new BenStringInternPool(new SharedInternPool());
var rvmStore = RvmWorkload.ReadRvmFiles(workload, progressReport, stringInternPool);
var rvmStore = RvmWorkload.ReadRvmFiles(workload, progressReport, null);

teamCityReadRvmFilesLogBlock.CloseBlock();

Expand Down
2 changes: 1 addition & 1 deletion Commons.Tests/Commons.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>

Expand Down
2 changes: 1 addition & 1 deletion Commons/Commons.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

<PropertyGroup>
<OutputType>Library</OutputType>
<TargetFrameworks>net8.0;netstandard2.1</TargetFrameworks>
<TargetFrameworks>net9.0;netstandard2.1</TargetFrameworks>
<Nullable>enable</Nullable>
<WarningsAsErrors>nullable;</WarningsAsErrors>
<LangVersion>latest</LangVersion>
Expand Down
2 changes: 1 addition & 1 deletion HierarchyComposer.Tests/HierarchyComposer.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<TargetFramework>net9.0</TargetFramework>
<IsPackable>false</IsPackable>
</PropertyGroup>

Expand Down
Loading
Loading