From a803fdb0b18e2341510aa0806992df3cf634dda5 Mon Sep 17 00:00:00 2001 From: Amrsatrio Date: Thu, 19 Aug 2021 21:48:26 +0700 Subject: [PATCH] Made object loader to be more stable and consistent --- README.md | 12 +- .../com/tb24/blenderumap/JWPSerializer.kt | 20 +-- src/main/java/com/tb24/blenderumap/Main.java | 166 ++++++------------ .../com/tb24/blenderumap/MyFileProvider.java | 4 +- src/main/resources/config.json | 11 +- 5 files changed, 68 insertions(+), 145 deletions(-) diff --git a/README.md b/README.md index c1369b4..c9c9c3b 100644 --- a/README.md +++ b/README.md @@ -35,19 +35,11 @@ This tool was made to fulfill my curiosity about exporting a Fortnite building i * `bDumpAssets`: Save assets as JSON format, useful for debugging. * `ObjectCacheSize`: Configure the object loader cache size to tune the performance, or set to 0 to disable. Defaults to 100. * `bReadMaterials`: Export materials. Materials are experimental! Not all imported materials will be perfect. **Min. 16GB of RAM recommended!** -* `bExportToDDSWhenPossible`: Prefer PNG over DDS? Set this to `false` and textures will always be exported as PNG. **Warning: Export times will significantly increase when this is set to `false`!** Defaults to enabled. +* `bExportToDDSWhenPossible`: Prefer PNG to DDS? Set this to `false` and textures will always be exported as PNG. **Warning: Export times will significantly increase when this is set to `false`!** Defaults to enabled. * `bExportBuildingFoundations`: You can turn off exporting sub-buildings in large POIs if you want to quickly port the base POI structures, by setting this to `false`. Defaults to enabled. * `bUseUModel`: Run UModel within the exporting process to export meshes, materials, and textures. * `UModelAdditionalArgs`: Additional command line args when starting UModel. * **`ExportPackage`: The .umap you want to export.** Accepts these path formats: - * `/Game/Maps/MapName.umap` - * `GameName/Content/Maps/MapName.umap` (file path) - - These are accepted for games with I/O store (.utoc and .ucas): * `/Game/Maps/MapName` (package path) * `/Game/Maps/MapName.MapName` (object path) - * `GameName/Content/Maps/MapName` (file path without extension) - - For assets in Fortnite game feature plugins: - * For example the full file path is `FortniteGame/Plugins/GameFeatures/GameFeatureName/Content/Maps/MapName.umap` - * You have to convert it into `/GameFeatureName/Maps/MapName` which is the package path that is supported by this tool. + * `GameName/Content/Maps/MapName.umap` (file path) \ No newline at end of file diff --git a/src/main/java/com/tb24/blenderumap/JWPSerializer.kt b/src/main/java/com/tb24/blenderumap/JWPSerializer.kt index c95ccae..08b3e1f 100644 --- a/src/main/java/com/tb24/blenderumap/JWPSerializer.kt +++ b/src/main/java/com/tb24/blenderumap/JWPSerializer.kt @@ -4,8 +4,6 @@ package com.tb24.blenderumap import com.google.gson.* -import me.fungames.jfortniteparse.ue4.assets.IoPackage -import me.fungames.jfortniteparse.ue4.assets.PakPackage import me.fungames.jfortniteparse.ue4.assets.UProperty import me.fungames.jfortniteparse.ue4.assets.exports.UCurveTable import me.fungames.jfortniteparse.ue4.assets.exports.UDataTable @@ -87,19 +85,11 @@ object JWPSerializer { if (src.isImport()) { val pkg = src.owner val arr = JsonArray().apply { - if (pkg is PakPackage) { - var current = pkg.run { src.getResource() } - while (current != null) { - add(current.objectName.text) - current = pkg.run { current!!.outerIndex.getResource() } - } - } else { - val initial = (pkg as IoPackage).resolveObjectIndex(pkg.importMap[src.toImport()]) - var current = initial - while (current != null) { - add(current.getName().text) - current = current.getOuter() - } + val initial = pkg?.findObjectMinimal(src) + var current = initial + while (current != null) { + add(current.getName().text) + current = current.getOuter() } } if (arr.size() > 0) arr else null diff --git a/src/main/java/com/tb24/blenderumap/Main.java b/src/main/java/com/tb24/blenderumap/Main.java index 613c963..d434288 100644 --- a/src/main/java/com/tb24/blenderumap/Main.java +++ b/src/main/java/com/tb24/blenderumap/Main.java @@ -3,43 +3,14 @@ */ package com.tb24.blenderumap; -import com.google.gson.FieldNamingPolicy; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; +import com.google.gson.*; import com.google.gson.reflect.TypeToken; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; - -import javax.imageio.ImageIO; - import kotlin.Lazy; import kotlin.io.FilesKt; import kotlin.text.StringsKt; import me.fungames.jfortniteparse.fort.exports.BuildingTextureData; import me.fungames.jfortniteparse.ue4.assets.Package; -import me.fungames.jfortniteparse.ue4.assets.exports.UBlueprintGeneratedClass; -import me.fungames.jfortniteparse.ue4.assets.exports.ULevel; -import me.fungames.jfortniteparse.ue4.assets.exports.UObject; -import me.fungames.jfortniteparse.ue4.assets.exports.UStaticMesh; -import me.fungames.jfortniteparse.ue4.assets.exports.UStruct; -import me.fungames.jfortniteparse.ue4.assets.exports.UWorld; +import me.fungames.jfortniteparse.ue4.assets.exports.*; import me.fungames.jfortniteparse.ue4.assets.exports.mats.UMaterialInstance; import me.fungames.jfortniteparse.ue4.assets.exports.mats.UMaterialInstance.FTextureParameterValue; import me.fungames.jfortniteparse.ue4.assets.exports.mats.UMaterialInterface; @@ -52,14 +23,22 @@ import me.fungames.jfortniteparse.ue4.converters.meshes.StaticMeshesKt; import me.fungames.jfortniteparse.ue4.converters.meshes.psk.ExportStaticMeshKt; import me.fungames.jfortniteparse.ue4.converters.textures.TexturesKt; +import me.fungames.jfortniteparse.ue4.objects.core.math.FQuat; import me.fungames.jfortniteparse.ue4.objects.core.math.FRotator; import me.fungames.jfortniteparse.ue4.objects.core.math.FVector; import me.fungames.jfortniteparse.ue4.objects.core.misc.FGuid; import me.fungames.jfortniteparse.ue4.objects.uobject.FName; import me.fungames.jfortniteparse.ue4.objects.uobject.FSoftObjectPath; -import me.fungames.jfortniteparse.ue4.versions.GameKt; import me.fungames.jfortniteparse.ue4.versions.Ue4Version; -import me.fungames.jfortniteparse.util.DataTypeConverterKt; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.imageio.ImageIO; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.util.*; import static com.tb24.blenderumap.AssetUtilsKt.getProp; import static com.tb24.blenderumap.AssetUtilsKt.getProps; @@ -113,19 +92,12 @@ public static void main(String[] args) { Package pkg = exportAndProduceProcessed(config.ExportPackage); if (pkg == null) return; - if (!exportQueue.isEmpty()) { - exportUmodel(); - } - File file = new File("processed.json"); LOGGER.info("Writing to " + file.getAbsolutePath()); try (FileWriter writer = new FileWriter(file)) { // GSON.toJson(components, writer); String pkgName = provider.compactFilePath(pkg.getName()); - if (pkgName.endsWith(".umap")) { - pkgName = pkgName.substring(0, pkgName.lastIndexOf('.')); - } GSON.toJson(pkgName, writer); } @@ -133,8 +105,10 @@ public static void main(String[] args) { } catch (Exception e) { if (e instanceof MainException) { LOGGER.info(e.getMessage()); + } else if (e instanceof JsonSyntaxException) { + LOGGER.error("Please check your config.json for syntax errors.\n{}", e.getCause().getMessage()); } else { - LOGGER.error("An unexpected error has occurred, please report", e); + LOGGER.error("An unexpected error has occurred, please check your config.json or report to the author", e); } System.exit(1); @@ -160,16 +134,13 @@ public static File getNewestUsmap(String directoryFilePath) { } private static Package exportAndProduceProcessed(String path) { - if (path.endsWith(".umap") && provider.mountedIoStoreReaders().size() > 0) { - path = path.substring(0, path.lastIndexOf('.')); - } UObject mainObject = provider.loadObject(path); if (mainObject == null) { - LOGGER.info("Object not found"); + LOGGER.info("Not found: " + path); return null; } if (!(mainObject instanceof UWorld)) { - LOGGER.info(mainObject.getPathName() + " is not a UWorld, won't try to export"); + LOGGER.info(mainObject.getFullName() + " is not a World, won't try to export"); return null; } UWorld world = (UWorld) mainObject; @@ -217,11 +188,7 @@ private static Package exportAndProduceProcessed(String path) { UStaticMesh meshExport = mesh.getValue(); if (meshExport != null) { - if (config.bUseUModel) { - exportQueue.add(mesh); - } else { - ExportStaticMeshKt.export(StaticMeshesKt.convertMesh(meshExport), false, false).writeToDir(getExportDir(meshExport)); - } + ExportStaticMeshKt.export(StaticMeshesKt.convertMesh(meshExport), false, false).writeToDir(getExportDir(meshExport)); if (config.bReadMaterials) { List staticMaterials = meshExport.StaticMaterials; @@ -278,8 +245,7 @@ private static Package exportAndProduceProcessed(String path) { if (config.bExportBuildingFoundations && additionalWorlds != null) { for (FSoftObjectPath additionalWorld : additionalWorlds) { - String text = additionalWorld.getAssetPathName().getText(); - Package cpkg = exportAndProduceProcessed(StringsKt.substringBeforeLast(text, '.', text) + ".umap"); + Package cpkg = exportAndProduceProcessed(additionalWorld.getAssetPathName().getText()); children.add(cpkg != null ? provider.compactFilePath(cpkg.getName()) : null); } } @@ -294,11 +260,33 @@ private static Package exportAndProduceProcessed(String path) { comp.add(children); } + /*if (config.bExportBuildingFoundations) { + for (Lazy streamingLevelLazy : world.getStreamingLevels()) { + UObject streamingLevel = streamingLevelLazy.getValue(); + if (streamingLevel == null) continue; + + JsonArray children = new JsonArray(); + Package cpkg = exportAndProduceProcessed(getProp(streamingLevel, "WorldAsset", FSoftObjectPath.class).getAssetPathName().getText()); + children.add(cpkg != null ? provider.compactFilePath(cpkg.getName()) : null); + + FTransform transform = getProp(streamingLevel, "LevelTransform", FTransform.class); + + JsonArray comp = new JsonArray(); + comp.add(JsonNull.INSTANCE); // GUID + comp.add(streamingLevel.getName()); + comp.add(JsonNull.INSTANCE); // mesh path + comp.add(JsonNull.INSTANCE); // materials + comp.add(JsonNull.INSTANCE); // texture data + comp.add(vector(transform.getTranslation())); // location + comp.add(quat(transform.getRotation())); // rotation + comp.add(vector(transform.getScale3D())); // scale + comp.add(children); + comps.add(comp); + } + }*/ + Package pkg = world.getOwner(); String pkgName = provider.compactFilePath(pkg.getName()); - if (pkgName.endsWith(".umap")) { - pkgName = pkgName.substring(0, pkgName.lastIndexOf('.')); - } File file = new File(MyFileProvider.JSONS_FOLDER, pkgName + ".processed.json"); file.getParentFile().mkdirs(); LOGGER.info("Writing to {}", file.getAbsolutePath()); @@ -322,11 +310,6 @@ private static void addToArray(JsonArray array, Lazy index) } private static void exportTexture(Lazy index) { - if (config.bUseUModel) { - exportQueue.add(index); - return; - } - try { UTexture2D texture = index.getValue() instanceof UTexture2D ? (UTexture2D) index.getValue() : null; if (texture == null) { @@ -387,55 +370,6 @@ public static String pkgIndexToDirPath(Lazy index) { return StringsKt.substringAfterLast(pkgPath, '/', pkgPath).equals(objectName) ? pkgPath : pkgPath + '/' + objectName; } - private static void exportUmodel() throws InterruptedException, IOException { - try (PrintWriter pw = new PrintWriter("umodel_cmd.txt")) { - pw.println("-path=\"" + config.PaksDirectory + '\"'); - pw.println("-game=ue4." + GameKt.GAME_UE4_GET_MINOR(config.UEVersion.getGame())); - - if (config.EncryptionKeys.size() > 0) { // TODO run umodel multiple times if there's more than one encryption key - pw.println("-aes=0x" + DataTypeConverterKt.printHexBinary(config.EncryptionKeys.get(0).Key)); - } - - pw.println("-out=\"" + new File("").getAbsolutePath() + '\"'); - - if (!isEmpty(config.UModelAdditionalArgs)) { - pw.println(config.UModelAdditionalArgs); - } - - boolean bFirst = true; - - for (Lazy export : exportQueue) { - if (export == null) continue; - - UObject object = export.getValue(); - String packagePath = provider.compactFilePath(object.getOwner().getName()); - String objectName = object.getName(); - - if (bFirst) { - bFirst = false; - pw.println("-export"); - pw.println(packagePath); - pw.println(objectName); - } else { - pw.println("-pkg=" + packagePath); - pw.println("-obj=" + objectName); - } - } - } - - ProcessBuilder pb = new ProcessBuilder(Arrays.asList("umodel", "@umodel_cmd.txt")); - pb.redirectOutput(ProcessBuilder.Redirect.INHERIT); - pb.redirectError(ProcessBuilder.Redirect.INHERIT); - LOGGER.info("Starting UModel process"); - int exitCode = pb.start().waitFor(); - - if (exitCode == 0) { - exportQueue.clear(); - } else { - LOGGER.warn("UModel returned exit code " + exitCode + ", some assets might weren't exported successfully"); - } - } - private static JsonArray vector(FVector vector) { if (vector == null) return null; JsonArray array = new JsonArray(3); @@ -454,8 +388,14 @@ private static JsonArray rotator(FRotator rotator) { return array; } - private static boolean isEmpty(String s) { - return s == null || s.isEmpty(); + private static JsonArray quat(FQuat quat) { + if (quat == null) return null; + JsonArray array = new JsonArray(4); + array.add(quat.getX()); + array.add(quat.getY()); + array.add(quat.getZ()); + array.add(quat.getW()); + return array; } private static class Mat { @@ -572,8 +512,6 @@ public static class Config { public boolean bReadMaterials = true; public boolean bExportToDDSWhenPossible = true; public boolean bExportBuildingFoundations = true; - public boolean bUseUModel = false; - public String UModelAdditionalArgs = ""; public String ExportPackage; } diff --git a/src/main/java/com/tb24/blenderumap/MyFileProvider.java b/src/main/java/com/tb24/blenderumap/MyFileProvider.java index 2857b2a..12f33ec 100644 --- a/src/main/java/com/tb24/blenderumap/MyFileProvider.java +++ b/src/main/java/com/tb24/blenderumap/MyFileProvider.java @@ -44,7 +44,7 @@ protected int sizeOf(@NotNull FPackageId key, @NotNull IoPackage value) { cache = null; } - setIoStoreTocReadOptions(0); + //setIoStoreTocReadOptions(0); Let the long package paths be loaded so long plugin paths can be resolved Map keysToSubmit = new HashMap<>(); for (EncryptionKey entry : encryptionKeys) { @@ -67,7 +67,7 @@ protected int sizeOf(@NotNull FPackageId key, @NotNull IoPackage value) { @Override public Package loadGameFile(@NotNull GameFile file) { return MapsKt.getOrPut(loaded, file, () -> { - LOGGER.info("Loading " + file); + //LOGGER.info("Loading " + file); Package loadedPkg = super.loadGameFile(file); if (loadedPkg != null && bDumpAssets) { diff --git a/src/main/resources/config.json b/src/main/resources/config.json index 7979f20..7f2f6b8 100644 --- a/src/main/resources/config.json +++ b/src/main/resources/config.json @@ -2,13 +2,16 @@ "_Documentation": "https://github.com/Amrsatrio/BlenderUmap/blob/master/README.md", "PaksDirectory": "C:\\Program Files\\Epic Games\\Fortnite\\FortniteGame\\Content\\Paks", "UEVersion": "GAME_UE4_LATEST", - "EncryptionKeys": [], + "EncryptionKeys": [ + { + "Guid": "00000000000000000000000000000000", + "Key": "0x0000000000000000000000000000000000000000000000000000000000000000" + } + ], "bDumpAssets": false, "ObjectCacheSize": 100, "bReadMaterials": true, "bExportToDDSWhenPossible": true, "bExportBuildingFoundations": true, - "bUseUModel": false, - "UModelAdditionalArgs": "", - "ExportPackage": "/Game/Content/Athena/Apollo/Maps/Buildings/3x3/Apollo_3x3_BoatRental.umap" + "ExportPackage": "/Game/Athena/Apollo/Maps/Buildings/3x3/Apollo_3x3_BoatRental" } \ No newline at end of file