Skip to content

Commit

Permalink
Made object loader to be more stable and consistent
Browse files Browse the repository at this point in the history
  • Loading branch information
@Amrsatrio
Amrsatrio committed Aug 19, 2021
1 parent 7f4d78d commit a803fdb
Show file tree
Hide file tree
Showing 5 changed files with 68 additions and 145 deletions.
12 changes: 2 additions & 10 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,19 +35,11 @@ This tool was made to fulfill my curiosity about exporting a Fortnite building i
* `bDumpAssets`: Save assets as JSON format, useful for debugging.
* `ObjectCacheSize`: Configure the object loader cache size to tune the performance, or set to 0 to disable. Defaults to 100.
* `bReadMaterials`: Export materials. Materials are experimental! Not all imported materials will be perfect. **Min. 16GB of RAM recommended!**
* `bExportToDDSWhenPossible`: Prefer PNG over DDS? Set this to `false` and textures will always be exported as PNG. **Warning: Export times will significantly increase when this is set to `false`!** Defaults to enabled.
* `bExportToDDSWhenPossible`: Prefer PNG to DDS? Set this to `false` and textures will always be exported as PNG. **Warning: Export times will significantly increase when this is set to `false`!** Defaults to enabled.
* `bExportBuildingFoundations`: You can turn off exporting sub-buildings in large POIs if you want to quickly port the base POI structures, by setting this to `false`. Defaults to enabled.
* `bUseUModel`: Run UModel within the exporting process to export meshes, materials, and textures.
* `UModelAdditionalArgs`: Additional command line args when starting UModel.
* **`ExportPackage`: The .umap you want to export.** Accepts these path formats:
* `/Game/Maps/MapName.umap`
* `GameName/Content/Maps/MapName.umap` (file path)

These are accepted for games with I/O store (.utoc and .ucas):
* `/Game/Maps/MapName` (package path)
* `/Game/Maps/MapName.MapName` (object path)
* `GameName/Content/Maps/MapName` (file path without extension)

For assets in Fortnite game feature plugins:
* For example the full file path is `FortniteGame/Plugins/GameFeatures/GameFeatureName/Content/Maps/MapName.umap`
* You have to convert it into `/GameFeatureName/Maps/MapName` which is the package path that is supported by this tool.
* `GameName/Content/Maps/MapName.umap` (file path)
20 changes: 5 additions & 15 deletions src/main/java/com/tb24/blenderumap/JWPSerializer.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@
package com.tb24.blenderumap

import com.google.gson.*
import me.fungames.jfortniteparse.ue4.assets.IoPackage
import me.fungames.jfortniteparse.ue4.assets.PakPackage
import me.fungames.jfortniteparse.ue4.assets.UProperty
import me.fungames.jfortniteparse.ue4.assets.exports.UCurveTable
import me.fungames.jfortniteparse.ue4.assets.exports.UDataTable
Expand Down Expand Up @@ -87,19 +85,11 @@ object JWPSerializer {
if (src.isImport()) {
val pkg = src.owner
val arr = JsonArray().apply {
if (pkg is PakPackage) {
var current = pkg.run { src.getResource() }
while (current != null) {
add(current.objectName.text)
current = pkg.run { current!!.outerIndex.getResource() }
}
} else {
val initial = (pkg as IoPackage).resolveObjectIndex(pkg.importMap[src.toImport()])
var current = initial
while (current != null) {
add(current.getName().text)
current = current.getOuter()
}
val initial = pkg?.findObjectMinimal(src)
var current = initial
while (current != null) {
add(current.getName().text)
current = current.getOuter()
}
}
if (arr.size() > 0) arr else null
Expand Down
166 changes: 52 additions & 114 deletions src/main/java/com/tb24/blenderumap/Main.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,43 +3,14 @@
*/
package com.tb24.blenderumap;

import com.google.gson.FieldNamingPolicy;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.*;
import com.google.gson.reflect.TypeToken;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;

import javax.imageio.ImageIO;

import kotlin.Lazy;
import kotlin.io.FilesKt;
import kotlin.text.StringsKt;
import me.fungames.jfortniteparse.fort.exports.BuildingTextureData;
import me.fungames.jfortniteparse.ue4.assets.Package;
import me.fungames.jfortniteparse.ue4.assets.exports.UBlueprintGeneratedClass;
import me.fungames.jfortniteparse.ue4.assets.exports.ULevel;
import me.fungames.jfortniteparse.ue4.assets.exports.UObject;
import me.fungames.jfortniteparse.ue4.assets.exports.UStaticMesh;
import me.fungames.jfortniteparse.ue4.assets.exports.UStruct;
import me.fungames.jfortniteparse.ue4.assets.exports.UWorld;
import me.fungames.jfortniteparse.ue4.assets.exports.*;
import me.fungames.jfortniteparse.ue4.assets.exports.mats.UMaterialInstance;
import me.fungames.jfortniteparse.ue4.assets.exports.mats.UMaterialInstance.FTextureParameterValue;
import me.fungames.jfortniteparse.ue4.assets.exports.mats.UMaterialInterface;
Expand All @@ -52,14 +23,22 @@
import me.fungames.jfortniteparse.ue4.converters.meshes.StaticMeshesKt;
import me.fungames.jfortniteparse.ue4.converters.meshes.psk.ExportStaticMeshKt;
import me.fungames.jfortniteparse.ue4.converters.textures.TexturesKt;
import me.fungames.jfortniteparse.ue4.objects.core.math.FQuat;
import me.fungames.jfortniteparse.ue4.objects.core.math.FRotator;
import me.fungames.jfortniteparse.ue4.objects.core.math.FVector;
import me.fungames.jfortniteparse.ue4.objects.core.misc.FGuid;
import me.fungames.jfortniteparse.ue4.objects.uobject.FName;
import me.fungames.jfortniteparse.ue4.objects.uobject.FSoftObjectPath;
import me.fungames.jfortniteparse.ue4.versions.GameKt;
import me.fungames.jfortniteparse.ue4.versions.Ue4Version;
import me.fungames.jfortniteparse.util.DataTypeConverterKt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.imageio.ImageIO;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.*;

import static com.tb24.blenderumap.AssetUtilsKt.getProp;
import static com.tb24.blenderumap.AssetUtilsKt.getProps;
Expand Down Expand Up @@ -113,28 +92,23 @@ public static void main(String[] args) {
Package pkg = exportAndProduceProcessed(config.ExportPackage);
if (pkg == null) return;

if (!exportQueue.isEmpty()) {
exportUmodel();
}

File file = new File("processed.json");
LOGGER.info("Writing to " + file.getAbsolutePath());

try (FileWriter writer = new FileWriter(file)) {
// GSON.toJson(components, writer);
String pkgName = provider.compactFilePath(pkg.getName());
if (pkgName.endsWith(".umap")) {
pkgName = pkgName.substring(0, pkgName.lastIndexOf('.'));
}
GSON.toJson(pkgName, writer);
}

LOGGER.info(String.format("All done in %,.1f sec. In the Python script, replace the line with data_dir with this line below:\n\ndata_dir = r\"%s\"", (System.currentTimeMillis() - start) / 1000.0F, new File("").getAbsolutePath()));
} catch (Exception e) {
if (e instanceof MainException) {
LOGGER.info(e.getMessage());
} else if (e instanceof JsonSyntaxException) {
LOGGER.error("Please check your config.json for syntax errors.\n{}", e.getCause().getMessage());
} else {
LOGGER.error("An unexpected error has occurred, please report", e);
LOGGER.error("An unexpected error has occurred, please check your config.json or report to the author", e);
}

System.exit(1);
Expand All @@ -160,16 +134,13 @@ public static File getNewestUsmap(String directoryFilePath) {
}

private static Package exportAndProduceProcessed(String path) {
if (path.endsWith(".umap") && provider.mountedIoStoreReaders().size() > 0) {
path = path.substring(0, path.lastIndexOf('.'));
}
UObject mainObject = provider.loadObject(path);
if (mainObject == null) {
LOGGER.info("Object not found");
LOGGER.info("Not found: " + path);
return null;
}
if (!(mainObject instanceof UWorld)) {
LOGGER.info(mainObject.getPathName() + " is not a UWorld, won't try to export");
LOGGER.info(mainObject.getFullName() + " is not a World, won't try to export");
return null;
}
UWorld world = (UWorld) mainObject;
Expand Down Expand Up @@ -217,11 +188,7 @@ private static Package exportAndProduceProcessed(String path) {
UStaticMesh meshExport = mesh.getValue();

if (meshExport != null) {
if (config.bUseUModel) {
exportQueue.add(mesh);
} else {
ExportStaticMeshKt.export(StaticMeshesKt.convertMesh(meshExport), false, false).writeToDir(getExportDir(meshExport));
}
ExportStaticMeshKt.export(StaticMeshesKt.convertMesh(meshExport), false, false).writeToDir(getExportDir(meshExport));

if (config.bReadMaterials) {
List<FStaticMaterial> staticMaterials = meshExport.StaticMaterials;
Expand Down Expand Up @@ -278,8 +245,7 @@ private static Package exportAndProduceProcessed(String path) {

if (config.bExportBuildingFoundations && additionalWorlds != null) {
for (FSoftObjectPath additionalWorld : additionalWorlds) {
String text = additionalWorld.getAssetPathName().getText();
Package cpkg = exportAndProduceProcessed(StringsKt.substringBeforeLast(text, '.', text) + ".umap");
Package cpkg = exportAndProduceProcessed(additionalWorld.getAssetPathName().getText());
children.add(cpkg != null ? provider.compactFilePath(cpkg.getName()) : null);
}
}
Expand All @@ -294,11 +260,33 @@ private static Package exportAndProduceProcessed(String path) {
comp.add(children);
}

/*if (config.bExportBuildingFoundations) {
for (Lazy<UObject> streamingLevelLazy : world.getStreamingLevels()) {
UObject streamingLevel = streamingLevelLazy.getValue();
if (streamingLevel == null) continue;
JsonArray children = new JsonArray();
Package cpkg = exportAndProduceProcessed(getProp(streamingLevel, "WorldAsset", FSoftObjectPath.class).getAssetPathName().getText());
children.add(cpkg != null ? provider.compactFilePath(cpkg.getName()) : null);
FTransform transform = getProp(streamingLevel, "LevelTransform", FTransform.class);
JsonArray comp = new JsonArray();
comp.add(JsonNull.INSTANCE); // GUID
comp.add(streamingLevel.getName());
comp.add(JsonNull.INSTANCE); // mesh path
comp.add(JsonNull.INSTANCE); // materials
comp.add(JsonNull.INSTANCE); // texture data
comp.add(vector(transform.getTranslation())); // location
comp.add(quat(transform.getRotation())); // rotation
comp.add(vector(transform.getScale3D())); // scale
comp.add(children);
comps.add(comp);
}
}*/

Package pkg = world.getOwner();
String pkgName = provider.compactFilePath(pkg.getName());
if (pkgName.endsWith(".umap")) {
pkgName = pkgName.substring(0, pkgName.lastIndexOf('.'));
}
File file = new File(MyFileProvider.JSONS_FOLDER, pkgName + ".processed.json");
file.getParentFile().mkdirs();
LOGGER.info("Writing to {}", file.getAbsolutePath());
Expand All @@ -322,11 +310,6 @@ private static void addToArray(JsonArray array, Lazy<? extends UTexture> index)
}

private static void exportTexture(Lazy<? extends UTexture> index) {
if (config.bUseUModel) {
exportQueue.add(index);
return;
}

try {
UTexture2D texture = index.getValue() instanceof UTexture2D ? (UTexture2D) index.getValue() : null;
if (texture == null) {
Expand Down Expand Up @@ -387,55 +370,6 @@ public static String pkgIndexToDirPath(Lazy<? extends UObject> index) {
return StringsKt.substringAfterLast(pkgPath, '/', pkgPath).equals(objectName) ? pkgPath : pkgPath + '/' + objectName;
}

private static void exportUmodel() throws InterruptedException, IOException {
try (PrintWriter pw = new PrintWriter("umodel_cmd.txt")) {
pw.println("-path=\"" + config.PaksDirectory + '\"');
pw.println("-game=ue4." + GameKt.GAME_UE4_GET_MINOR(config.UEVersion.getGame()));

if (config.EncryptionKeys.size() > 0) { // TODO run umodel multiple times if there's more than one encryption key
pw.println("-aes=0x" + DataTypeConverterKt.printHexBinary(config.EncryptionKeys.get(0).Key));
}

pw.println("-out=\"" + new File("").getAbsolutePath() + '\"');

if (!isEmpty(config.UModelAdditionalArgs)) {
pw.println(config.UModelAdditionalArgs);
}

boolean bFirst = true;

for (Lazy<? extends UObject> export : exportQueue) {
if (export == null) continue;

UObject object = export.getValue();
String packagePath = provider.compactFilePath(object.getOwner().getName());
String objectName = object.getName();

if (bFirst) {
bFirst = false;
pw.println("-export");
pw.println(packagePath);
pw.println(objectName);
} else {
pw.println("-pkg=" + packagePath);
pw.println("-obj=" + objectName);
}
}
}

ProcessBuilder pb = new ProcessBuilder(Arrays.asList("umodel", "@umodel_cmd.txt"));
pb.redirectOutput(ProcessBuilder.Redirect.INHERIT);
pb.redirectError(ProcessBuilder.Redirect.INHERIT);
LOGGER.info("Starting UModel process");
int exitCode = pb.start().waitFor();

if (exitCode == 0) {
exportQueue.clear();
} else {
LOGGER.warn("UModel returned exit code " + exitCode + ", some assets might weren't exported successfully");
}
}

private static JsonArray vector(FVector vector) {
if (vector == null) return null;
JsonArray array = new JsonArray(3);
Expand All @@ -454,8 +388,14 @@ private static JsonArray rotator(FRotator rotator) {
return array;
}

private static boolean isEmpty(String s) {
return s == null || s.isEmpty();
private static JsonArray quat(FQuat quat) {
if (quat == null) return null;
JsonArray array = new JsonArray(4);
array.add(quat.getX());
array.add(quat.getY());
array.add(quat.getZ());
array.add(quat.getW());
return array;
}

private static class Mat {
Expand Down Expand Up @@ -572,8 +512,6 @@ public static class Config {
public boolean bReadMaterials = true;
public boolean bExportToDDSWhenPossible = true;
public boolean bExportBuildingFoundations = true;
public boolean bUseUModel = false;
public String UModelAdditionalArgs = "";
public String ExportPackage;
}

Expand Down
4 changes: 2 additions & 2 deletions src/main/java/com/tb24/blenderumap/MyFileProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ protected int sizeOf(@NotNull FPackageId key, @NotNull IoPackage value) {
cache = null;
}

setIoStoreTocReadOptions(0);
//setIoStoreTocReadOptions(0); Let the long package paths be loaded so long plugin paths can be resolved

Map<FGuid, byte[]> keysToSubmit = new HashMap<>();
for (EncryptionKey entry : encryptionKeys) {
Expand All @@ -67,7 +67,7 @@ protected int sizeOf(@NotNull FPackageId key, @NotNull IoPackage value) {
@Override
public Package loadGameFile(@NotNull GameFile file) {
return MapsKt.getOrPut(loaded, file, () -> {
LOGGER.info("Loading " + file);
//LOGGER.info("Loading " + file);
Package loadedPkg = super.loadGameFile(file);

if (loadedPkg != null && bDumpAssets) {
Expand Down
11 changes: 7 additions & 4 deletions src/main/resources/config.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,16 @@
"_Documentation": "https://github.com/Amrsatrio/BlenderUmap/blob/master/README.md",
"PaksDirectory": "C:\\Program Files\\Epic Games\\Fortnite\\FortniteGame\\Content\\Paks",
"UEVersion": "GAME_UE4_LATEST",
"EncryptionKeys": [],
"EncryptionKeys": [
{
"Guid": "00000000000000000000000000000000",
"Key": "0x0000000000000000000000000000000000000000000000000000000000000000"
}
],
"bDumpAssets": false,
"ObjectCacheSize": 100,
"bReadMaterials": true,
"bExportToDDSWhenPossible": true,
"bExportBuildingFoundations": true,
"bUseUModel": false,
"UModelAdditionalArgs": "",
"ExportPackage": "/Game/Content/Athena/Apollo/Maps/Buildings/3x3/Apollo_3x3_BoatRental.umap"
"ExportPackage": "/Game/Athena/Apollo/Maps/Buildings/3x3/Apollo_3x3_BoatRental"
}

0 comments on commit a803fdb

Please sign in to comment.