From d093fc7f830b68035f4940ad6eeeac2a1c855306 Mon Sep 17 00:00:00 2001 From: Amrsatrio Date: Thu, 23 Jul 2020 04:36:34 +0700 Subject: [PATCH] Added support for loading stuff within packages with more than one export --- .gitignore | 1 + build.gradle | 4 +- .../java/com/tb24/blenderumap/AssetUtils.kt | 39 ++ .../com/tb24/blenderumap/JWPSerializer.kt | 70 +++- src/main/java/com/tb24/blenderumap/Main.java | 354 ++++++++---------- .../com/tb24/blenderumap/MyFileProvider.java | 175 +++++++++ umap.py | 35 +- 7 files changed, 457 insertions(+), 221 deletions(-) create mode 100644 src/main/java/com/tb24/blenderumap/AssetUtils.kt create mode 100644 src/main/java/com/tb24/blenderumap/MyFileProvider.java diff --git a/.gitignore b/.gitignore index 3cb3f42..db71b38 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /.idea /build /run +/.unused diff --git a/build.gradle b/build.gradle index d7ed30a..f933065 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ plugins { } group 'com.tb24' -version '0.2.1' +version '0.2.2' sourceCompatibility = 1.8 @@ -42,7 +42,9 @@ repositories { } dependencies { + implementation fileTree(dir: 'libs', include: ['*.jar']) implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8' + implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.8' implementation 'com.google.code.gson:gson:2.8.6' // implementation 'com.mojang:brigadier:1.0.17' implementation 'me.fungames:JFortniteParse:+' // :3.0.2' diff --git a/src/main/java/com/tb24/blenderumap/AssetUtils.kt b/src/main/java/com/tb24/blenderumap/AssetUtils.kt new file mode 100644 index 0000000..5f08a8a --- /dev/null +++ b/src/main/java/com/tb24/blenderumap/AssetUtils.kt @@ -0,0 +1,39 @@ +package com.tb24.blenderumap + +import me.fungames.jfortniteparse.ue4.FGuid +import me.fungames.jfortniteparse.ue4.assets.exports.UExport +import me.fungames.jfortniteparse.ue4.assets.exports.UObject +import me.fungames.jfortniteparse.ue4.assets.objects.FPropertyTag +import java.util.* + +fun getProp(properties: List, name: String, clazz: Class): T? { + for (prop in properties) { + if (name == prop.name.text) { + return prop.getTagTypeValue(clazz, null) as T? + } + } + return null +} + +fun getProps(properties: List, name: String, clazz: Class): Array { + val collected: MutableList = ArrayList() + var maxIndex = -1 + for (prop in properties) { + if (prop.name.text == name) { + collected.add(prop) + maxIndex = Math.max(maxIndex, prop.arrayIndex) + } + } + val out = java.lang.reflect.Array.newInstance(clazz, maxIndex + 1) as Array + for (prop in collected) { + out[prop.arrayIndex] = prop.getTagTypeValue(clazz, null) as T? + } + return out +} + +fun UExport.getProp(name: String, clazz: Class) = baseObject.getProp(name, clazz) +fun UObject.getProp(name: String, clazz: Class) = getProp(properties, name, clazz) +inline operator fun UExport.get(name: String): T? = getProp(name, T::class.java) +inline operator fun UObject.get(name: String): T? = getProp(name, T::class.java) + +fun FGuid?.asString() = if (this == null) null else "%08x%08x%08x%08x".format(part1, part2, part3, part4) diff --git a/src/main/java/com/tb24/blenderumap/JWPSerializer.kt b/src/main/java/com/tb24/blenderumap/JWPSerializer.kt index ad05cca..6ac5c57 100644 --- a/src/main/java/com/tb24/blenderumap/JWPSerializer.kt +++ b/src/main/java/com/tb24/blenderumap/JWPSerializer.kt @@ -7,7 +7,6 @@ import com.google.gson.* import me.fungames.jfortniteparse.ue4.FGuid import me.fungames.jfortniteparse.ue4.assets.exports.UDataTable import me.fungames.jfortniteparse.ue4.assets.exports.UExport -import me.fungames.jfortniteparse.ue4.assets.exports.UObject import me.fungames.jfortniteparse.ue4.assets.objects.* import me.fungames.jfortniteparse.ue4.assets.util.FName import me.fungames.jfortniteparse.util.parseHexBinary @@ -20,10 +19,13 @@ import java.util.* */ @ExperimentalUnsignedTypes object JWPSerializer { + @JvmField + var sUseNonstandardFormat = false + @JvmField val GSON: Gson = GsonBuilder() .disableHtmlEscaping() - .setPrettyPrinting() + //.setPrettyPrinting() .serializeNulls() .registerTypeAdapter(ByteArray::class.java, ByteArraySerializer()) .registerTypeAdapter(UByte::class.java, JsonSerializer { src, typeOfSrc, context -> JsonPrimitive(src.toByte()) }) @@ -42,6 +44,14 @@ object JWPSerializer { JsonObject().apply { add("min", context.serialize(src.min)) add("max", context.serialize(src.max)) + add("valid", context.serialize(src.isValid)) + } + }) + .registerTypeAdapter(FBox2D::class.java, JsonSerializer { src, typeOfSrc, context -> + JsonObject().apply { + add("min", context.serialize(src.min)) + add("max", context.serialize(src.max)) + add("valid", context.serialize(src.isValid)) } }) .registerTypeAdapter(FGameplayTagContainer::class.java, JsonSerializer { src, typeOfSrc, context -> @@ -62,24 +72,36 @@ object JWPSerializer { JsonPrimitive(src.text) }) .registerTypeAdapter(FPackageIndex::class.java, JsonSerializer { src, typeOfSrc, context -> - var out: JsonElement? = null - val importObject = src.importObject + val out: JsonElement - if (src.index >= 0) { - out = JsonObject() - out.addProperty("export", src.index) - } else if (importObject != null) { + if (src.index < 0) { + val importObject = src.importObject out = JsonArray() - out.add(context.serialize(importObject.objectName)) + out.add(context.serialize(importObject!!.objectName)) out.add(context.serialize(src.outerImportObject!!.objectName)) if (src.outerImportObject!!.outerIndex.importObject != null) { out.add(context.serialize(src.outerImportObject!!.outerIndex.importObject!!.objectName)) } + } else { + out = JsonObject() + out.addProperty("export", src.index) + + /*if (src.index > 0) { + out.addProperty("__object_name", src.exportObject!!.objectName.text) + }*/ } out }) + .registerTypeAdapter(FQuat::class.java, JsonSerializer { src, typeOfSrc, context -> + JsonObject().apply { + addProperty("x", src.x) + addProperty("y", src.y) + addProperty("z", src.z) + addProperty("w", src.w) + } + }) .registerTypeAdapter(FRotator::class.java, JsonSerializer { src, typeOfSrc, context -> JsonObject().apply { addProperty("pitch", src.pitch) @@ -141,21 +163,37 @@ object JWPSerializer { .create() private fun serializeProperties(obj: JsonObject, properties: List, context: JsonSerializationContext) { - for (propertyTag in properties) { - obj.add(propertyTag.name.text, context.serialize(propertyTag.tag)) + properties.forEach { + obj.add(it.name.text + (if (it.arrayIndex != 0) "[${it.arrayIndex}]" else ""), context.serialize(it.tag)) } } private class ExportSerializer : JsonSerializer { override fun serialize(src: UExport, typeOfSrc: Type, context: JsonSerializationContext): JsonElement? { val obj = JsonObject() + if (sUseNonstandardFormat && src.export != null) obj.addProperty("object_name", src.export!!.objectName.text) obj.addProperty("export_type", src.exportType) - if (src is UObject) { - serializeProperties(obj, src.properties, context) - } else if (src is UDataTable) { - for ((key, value) in src.rows) { - obj.add(key.text, context.serialize(value)) + if (src !is UDataTable || sUseNonstandardFormat) + serializeProperties(obj, src.baseObject.properties, context) + + if (src is UDataTable) { + if (sUseNonstandardFormat) { + obj.add("rows", JsonObject().apply { + for ((key, value) in src.rows) { + add(key.text, JsonObject().apply { + addProperty("export_type", "RowStruct") + serializeProperties(this, value.properties, context) + }) + } + }) + } else { + for ((key, value) in src.rows) { + obj.add(key.text, JsonObject().apply { + addProperty("export_type", "RowStruct") + serializeProperties(this, value.properties, context) + }) + } } } diff --git a/src/main/java/com/tb24/blenderumap/Main.java b/src/main/java/com/tb24/blenderumap/Main.java index b9f09a4..1a995e0 100644 --- a/src/main/java/com/tb24/blenderumap/Main.java +++ b/src/main/java/com/tb24/blenderumap/Main.java @@ -1,5 +1,5 @@ /* - * (C) 2020 amrsatrio. All rights reserved. + * (C) amrsatrio. All rights reserved. */ package com.tb24.blenderumap; @@ -15,45 +15,48 @@ import java.io.FileWriter; import java.io.IOException; import java.io.PrintWriter; -import java.lang.reflect.Array; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.UUID; -import kotlin.collections.MapsKt; +import javax.imageio.ImageIO; + import kotlin.text.StringsKt; -import me.fungames.jfortniteparse.fileprovider.DefaultFileProvider; +import me.fungames.jfortniteparse.converters.ue4.meshes.StaticMeshesKt; +import me.fungames.jfortniteparse.converters.ue4.meshes.psk.ExportStaticMeshKt; +import me.fungames.jfortniteparse.converters.ue4.textures.TexturesKt; import me.fungames.jfortniteparse.ue4.FGuid; import me.fungames.jfortniteparse.ue4.assets.Package; import me.fungames.jfortniteparse.ue4.assets.exports.UExport; +import me.fungames.jfortniteparse.ue4.assets.exports.UStaticMesh; +import me.fungames.jfortniteparse.ue4.assets.exports.tex.UTexture; +import me.fungames.jfortniteparse.ue4.assets.objects.FObjectExport; import me.fungames.jfortniteparse.ue4.assets.objects.FPackageIndex; -import me.fungames.jfortniteparse.ue4.assets.objects.FPropertyTag; import me.fungames.jfortniteparse.ue4.assets.objects.FRotator; import me.fungames.jfortniteparse.ue4.assets.objects.FSoftObjectPath; import me.fungames.jfortniteparse.ue4.assets.objects.FStructFallback; import me.fungames.jfortniteparse.ue4.assets.objects.FVector; import me.fungames.jfortniteparse.ue4.assets.util.FName; import me.fungames.jfortniteparse.ue4.assets.util.StructFallbackReflectionUtilKt; -import me.fungames.jfortniteparse.ue4.pak.GameFile; -import me.fungames.jfortniteparse.ue4.pak.PakFileReader; import me.fungames.jfortniteparse.ue4.versions.GameKt; import me.fungames.jfortniteparse.ue4.versions.Ue4Version; +import static com.tb24.blenderumap.AssetUtilsKt.asString; +import static com.tb24.blenderumap.AssetUtilsKt.getProp; +import static com.tb24.blenderumap.AssetUtilsKt.getProps; import static com.tb24.blenderumap.JWPSerializer.GSON; public class Main { private static final Logger LOGGER = LoggerFactory.getLogger("BlenderUmap"); private static Config config; - private static File jsonsFolder = new File("jsons"); - private static DefaultFileProvider provider; - private static Map loaded = new HashMap<>(); - private static Set toExport = new HashSet<>(); + private static MyFileProvider provider; + private static Set exportQueue = new HashSet<>(); private static long start = System.currentTimeMillis(); public static void main(String[] args) { @@ -83,29 +86,12 @@ public static void main(String[] args) { throw new MainException("Please specify ExportPackage."); } - provider = new DefaultFileProvider(paksDir, config.UEVersion); - Map keysToSubmit = new HashMap<>(); - - for (Config.EncryptionKey entry : config.EncryptionKeys) { - if (isEmpty(entry.FileName)) { - keysToSubmit.put(entry.Guid, entry.Key); - } else { - Optional foundGuid = provider.getUnloadedPaks().stream().filter(it -> it.getFileName().equals(entry.FileName)).findFirst(); + provider = new MyFileProvider(paksDir, config.UEVersion, config.EncryptionKeys, config.bDumpAssets); - if (foundGuid.isPresent()) { - keysToSubmit.put(foundGuid.get().getPakInfo().getEncryptionKeyGuid(), entry.Key); - } else { - LOGGER.warn("PAK file not found: " + entry.FileName); - } - } - } - - provider.submitKeys(keysToSubmit); JsonArray components = exportAndProduceProcessed(config.ExportPackage); - if (components == null) return; - if (config.bRunUModel && !toExport.isEmpty()) { + if (!exportQueue.isEmpty()) { exportUmodel(); } @@ -121,7 +107,7 @@ public static void main(String[] args) { if (e instanceof MainException) { LOGGER.info(e.getMessage()); } else { - LOGGER.error("Uncaught exception", e); + LOGGER.error("An unexpected error has occurred, please report", e); } System.exit(1); @@ -129,7 +115,7 @@ public static void main(String[] args) { } private static JsonArray exportAndProduceProcessed(String s) { - Package pkg = loadIfNot(s); + Package pkg = provider.loadIfNot(s); if (pkg == null) { return null; @@ -140,43 +126,31 @@ private static JsonArray exportAndProduceProcessed(String s) { JsonArray comps = new JsonArray(); - for (UExport export : pkg.getExports()) { + for (FObjectExport objectExport : pkg.getExportMap()) { + UExport export = objectExport.exportObject.getValue(); String exportType = export.getExportType(); + if (exportType.equals("LODActor")) continue; - if (exportType.equals("LODActor")) { - continue; - } - - FPackageIndex smc = getProp(export, "StaticMeshComponent", FPackageIndex.class); - - if (smc == null) { - continue; - } - - UExport refSMC = pkg.getExports().get(smc.getIndex() - 1); + UExport staticMeshExp = provider.loadObject(getProp(export, "StaticMeshComponent", FPackageIndex.class)); + if (staticMeshExp == null) continue; // identifiers JsonArray comp = new JsonArray(); comps.add(comp); FGuid guid = getProp(export, "MyGuid", FGuid.class); - comp.add(guid != null ? guidAsString(guid) : UUID.randomUUID().toString().replace("-", "")); + comp.add(guid != null ? asString(guid) : UUID.randomUUID().toString().replace("-", "")); comp.add(exportType); // region mesh - String meshS = null; - FPackageIndex mesh = getProp(refSMC, "StaticMesh", FPackageIndex.class); + FPackageIndex mesh = getProp(staticMeshExp, "StaticMesh", FPackageIndex.class); if (mesh == null || mesh.getIndex() == 0) { // read the actor class to find the mesh - Package actorPkg = loadIfNot(export.getExport().getClassIndex().getOuterImportObject().getObjectName().getText()); - - if (actorPkg != null) { - for (UExport actorExp : actorPkg.getExports()) { - if (actorExp.getExportType().endsWith("StaticMeshComponent")) { - mesh = getProp(actorExp, "StaticMesh", FPackageIndex.class); + UExport actorBlueprint = provider.loadObject(objectExport.getClassIndex()); - if (mesh != null && mesh.getIndex() != 0) { - break; - } + if (actorBlueprint != null) { + for (UExport actorExp : actorBlueprint.getOwner().getExports()) { + if (actorExp.getExportType().endsWith("StaticMeshComponent") && (mesh = getProp(actorExp, "StaticMesh", FPackageIndex.class)) != null && mesh.getIndex() != 0) { + break; } } } @@ -188,24 +162,21 @@ private static JsonArray exportAndProduceProcessed(String s) { List materials = new ArrayList<>(); if (mesh != null && mesh.getIndex() != 0) { - toExport.add(meshS = mesh.getOuterImportObject().getObjectName().getText()); - - if (config.bReadMaterials) { - Package meshPkg = loadIfNot(meshS); + UExport meshExport = provider.loadObject(mesh); - if (meshPkg != null) { - for (UExport meshExport : meshPkg.getExports()) { - if (meshExport.getExportType().equals("StaticMesh")) { - //ExportStaticMeshKt.export(StaticMeshesKt.convertMesh((UStaticMesh) meshExport)).writeToDir(new File("TestExportMesh/" + meshS.substring(1)).getParentFile()); - FStructFallback[] staticMaterials = getProp(meshExport, "StaticMaterials", FStructFallback[].class); + if (meshExport != null) { + if (config.bUseUModel) { + exportQueue.add(mesh); + } else { + ExportStaticMeshKt.export(StaticMeshesKt.convertMesh((UStaticMesh) meshExport)).writeToDir(getExportDir(meshExport)); + } - if (staticMaterials != null) { - for (FStructFallback staticMaterial : staticMaterials) { - materials.add(new Mat(getProp(staticMaterial.getProperties(), "MaterialInterface", FPackageIndex.class))); - } - } + if (config.bReadMaterials) { + FStructFallback[] staticMaterials = getProp(meshExport, "StaticMaterials", FStructFallback[].class); - break; + if (staticMaterials != null) { + for (FStructFallback staticMaterial : staticMaterials) { + materials.add(new Mat(getProp(staticMaterial.getProperties(), "MaterialInterface", FPackageIndex.class))); } } } @@ -213,30 +184,27 @@ private static JsonArray exportAndProduceProcessed(String s) { } if (config.bReadMaterials) { - FPackageIndex material = getProp(refSMC, "BaseMaterial", FPackageIndex.class); + FPackageIndex material = getProp(staticMeshExp, "BaseMaterial", FPackageIndex.class); FPackageIndex[] overrideMaterials = getProp(export, "OverrideMaterials", FPackageIndex[].class); - for (FPackageIndex textureData : getProps(export.getBaseObject().getProperties(), "TextureData", FPackageIndex.class)) { - if (textureData != null && textureData.getIndex() != 0) { - String textureDataPath = textureData.getOuterImportObject().getObjectName().getText(); - Package texDataPkg = loadIfNot(textureDataPath); - - if (texDataPkg != null) { - BuildingTextureData td = StructFallbackReflectionUtilKt.mapToClass(texDataPkg.getExports().get(0).getBaseObject(), BuildingTextureData.class, null); - JsonArray textures = new JsonArray(); - addToArray(textures, td.Diffuse); - addToArray(textures, td.Normal); - addToArray(textures, td.Specular); - addToArray(textures, td.Emissive); - addToArray(textures, td.Mask); - JsonArray entry = new JsonArray(); - entry.add(textureDataPath); - entry.add(textures); - textureDataArr.add(entry); - - if (td.OverrideMaterial != null && td.OverrideMaterial.getIndex() != 0) { - material = td.OverrideMaterial; - } + for (FPackageIndex textureDataIdx : getProps(export.getBaseObject().getProperties(), "TextureData", FPackageIndex.class)) { + UExport texDataExp = provider.loadObject(textureDataIdx); + + if (texDataExp != null) { + BuildingTextureData td = StructFallbackReflectionUtilKt.mapToClass(texDataExp.getBaseObject(), BuildingTextureData.class, null); + JsonArray textures = new JsonArray(); + addToArray(textures, td.Diffuse); + addToArray(textures, td.Normal); + addToArray(textures, td.Specular); + addToArray(textures, td.Emissive); + addToArray(textures, td.Mask); + JsonArray entry = new JsonArray(); + entry.add(pkgIndexToDirPath(textureDataIdx)); + entry.add(textures); + textureDataArr.add(entry); + + if (td.OverrideMaterial != null && td.OverrideMaterial.getIndex() != 0) { + material = td.OverrideMaterial; } } else { textureDataArr.add((JsonElement) null); @@ -267,12 +235,12 @@ private static JsonArray exportAndProduceProcessed(String s) { } // endregion - comp.add(mesh != null && mesh.getIndex() != 0 ? meshS : null); + comp.add(pkgIndexToDirPath(mesh)); comp.add(matsObj); comp.add(textureDataArr); - comp.add(vector(getProp(refSMC, "RelativeLocation", FVector.class))); - comp.add(rotator(getProp(refSMC, "RelativeRotation", FRotator.class))); - comp.add(vector(getProp(refSMC, "RelativeScale3D", FVector.class))); + comp.add(vector(getProp(staticMeshExp, "RelativeLocation", FVector.class))); + comp.add(rotator(getProp(staticMeshExp, "RelativeRotation", FRotator.class))); + comp.add(vector(getProp(staticMeshExp, "RelativeScale3D", FVector.class))); comp.add(children); } @@ -281,44 +249,64 @@ private static JsonArray exportAndProduceProcessed(String s) { private static void addToArray(JsonArray array, FPackageIndex index) { if (index != null && index.getIndex() != 0) { - String s = index.getOuterImportObject().getObjectName().getText(); - toExport.add(s); - array.add(s); + exportTexture(index); + array.add(pkgIndexToDirPath(index)); } else { array.add((JsonElement) null); } } - private static Package loadIfNot(String pkg) { - GameFile gameFile = provider.findGameFile(pkg); + private static void exportTexture(FPackageIndex index) { + if (config.bUseUModel) { + exportQueue.add(index); + return; + } - if (gameFile != null) { - return loadIfNot(gameFile); - } else { - LOGGER.warn("Package " + pkg + " not found"); - return null; + try { + UTexture texExport = (UTexture) provider.loadObject(index); + File output = new File(getExportDir(texExport), texExport.getName() + ".png"); + + if (output.exists()) { + LOGGER.debug("Texture already exists, skipping: " + output.getAbsolutePath()); + } else { + LOGGER.info("Saving texture to " + output.getAbsolutePath()); + ImageIO.write(TexturesKt.toBufferedImage(texExport), "png", output); + } + } catch (IOException e) { + LOGGER.warn("Failed to save texture", e); } } - private static Package loadIfNot(GameFile pkg) { - return MapsKt.getOrPut(loaded, pkg, () -> { - LOGGER.info("Loading " + pkg); - Package loadedPkg = provider.loadGameFile(pkg); + public static File getExportDir(UExport exportObj) { + String pkgPath = MyFileProvider.compactFilePath(exportObj.getOwner().getName()); + pkgPath = StringsKt.substringBeforeLast(pkgPath, '.', pkgPath); - if (loadedPkg != null && config.bDumpAssets) { - File file = new File(jsonsFolder, pkg.getPathWithoutExtension() + ".json"); - LOGGER.info("Writing JSON to " + file.getAbsolutePath()); - file.getParentFile().mkdirs(); + if (pkgPath.startsWith("/")) { + pkgPath = pkgPath.substring(1); + } - try (FileWriter writer = new FileWriter(file)) { - GSON.toJson(loadedPkg.getExports(), writer); - } catch (IOException e) { - LOGGER.error("Writing failed", e); - } - } + File outputDir = new File(pkgPath).getParentFile(); + String pkgName = StringsKt.substringAfterLast(pkgPath, '/', pkgPath); - return loadedPkg; - }); + if (!exportObj.getName().equals(pkgName)) { + outputDir = new File(outputDir, pkgName); + } + + outputDir.mkdirs(); + return outputDir; + } + + public static String pkgIndexToDirPath(FPackageIndex index) { + if (index == null) return null; + + int i = index.getIndex(); + if (i == 0) return null; + + String pkgPath = MyFileProvider.compactFilePath(index.getOwner().getName()); + pkgPath = StringsKt.substringBeforeLast(pkgPath, '.', pkgPath); + pkgPath = i > 0 ? pkgPath : index.getOuterImportObject().getObjectName().getText(); + String objectName = (i > 0 ? index.getExportObject().getObjectName() : index.getImportObject().getObjectName()).getText(); + return StringsKt.substringAfterLast(pkgPath, '/', pkgPath).equals(objectName) ? pkgPath : pkgPath + '/' + objectName; } private static void exportUmodel() throws InterruptedException, IOException { @@ -326,8 +314,8 @@ private static void exportUmodel() throws InterruptedException, IOException { pw.println("-path=\"" + config.PaksDirectory + '\"'); pw.println("-game=ue4." + GameKt.GAME_UE4_GET_MINOR(config.UEVersion.getGame())); - if (config.EncryptionKeys.length > 0) { - pw.println("-aes=0x" + ByteArrayUtils.encode(config.EncryptionKeys[0].Key)); + if (config.EncryptionKeys.size() > 0) { // TODO run umodel multiple times if there's more than one encryption key + pw.println("-aes=0x" + ByteArrayUtils.encode(config.EncryptionKeys.get(0).Key)); } pw.println("-out=\"" + new File("").getAbsolutePath() + '\"'); @@ -338,13 +326,21 @@ private static void exportUmodel() throws InterruptedException, IOException { boolean bFirst = true; - for (String export : toExport) { + for (FPackageIndex export : exportQueue) { + int i = export.getIndex(); + if (i == 0) continue; + + String packagePath = i > 0 ? MyFileProvider.compactFilePath(export.getOwner().getName()) : export.getOuterImportObject().getObjectName().getText(); + String objectName = (i > 0 ? export.getExportObject().getObjectName() : export.getImportObject().getObjectName()).getText(); + if (bFirst) { bFirst = false; pw.println("-export"); - pw.println(export); + pw.println(packagePath); + pw.println(objectName); } else { - pw.println("-pkg=" + export); + pw.println("-pkg=" + packagePath); + pw.println("-obj=" + objectName); } } } @@ -356,50 +352,12 @@ private static void exportUmodel() throws InterruptedException, IOException { int exitCode = pb.start().waitFor(); if (exitCode == 0) { - toExport.clear(); + exportQueue.clear(); } else { LOGGER.warn("UModel returned exit code " + exitCode + ", some assets might weren't exported successfully"); } } - private static T getProp(List properties, String name, Class clazz) { - for (FPropertyTag prop : properties) { - if (name.equals(prop.getName().getText())) { - return (T) prop.getTagTypeValue(clazz, null); - } - } - - return null; - } - - private static T getProp(UExport export, String name, Class clazz) { - return getProp(export.getBaseObject().getProperties(), name, clazz); - } - - public static T[] getProps(List properties, String name, Class clazz) { - List collected = new ArrayList<>(); - int maxIndex = -1; - - for (FPropertyTag prop : properties) { - if (prop.getName().getText().equals(name)) { - collected.add(prop); - maxIndex = Math.max(maxIndex, prop.getArrayIndex()); - } - } - - T[] out = (T[]) Array.newInstance(clazz, maxIndex + 1); - - for (FPropertyTag prop : collected) { - out[prop.getArrayIndex()] = (T) prop.getTagTypeValue(clazz, null); - } - - return out; - } - - private static String guidAsString(FGuid guid) { - return String.format("%08x%08x%08x%08x", guid.getPart1(), guid.getPart2(), guid.getPart3(), guid.getPart4()); - } - private static JsonArray vector(FVector vector) { if (vector == null) return null; JsonArray array = new JsonArray(3); @@ -424,7 +382,7 @@ private static boolean isEmpty(String s) { private static class Mat { public FPackageIndex name; - public Map textureMap = new HashMap<>(); + public Map textureMap = new HashMap<>(); public Mat(FPackageIndex name) { this.name = name; @@ -436,10 +394,11 @@ public void populateTextures() { public void populateTextures(FPackageIndex pkgIndex) { if (pkgIndex.getIndex() == 0) return; - Package matPkg = loadIfNot(pkgIndex.getOuterImportObject().getObjectName().getText()); - if (matPkg == null) return; - UExport matFirstExp = matPkg.getExports().get(0); - FStructFallback[] textureParameterValues = getProp(matFirstExp, "TextureParameterValues", FStructFallback[].class); + + UExport material = provider.loadObject(pkgIndex); + if (material == null) return; + + FStructFallback[] textureParameterValues = getProp(material, "TextureParameterValues", FStructFallback[].class); if (textureParameterValues != null) { for (FStructFallback textureParameterValue : textureParameterValues) { @@ -449,13 +408,13 @@ public void populateTextures(FPackageIndex pkgIndex) { FPackageIndex parameterValue = getProp(textureParameterValue.getProperties(), "ParameterValue", FPackageIndex.class); if (parameterValue != null && parameterValue.getIndex() != 0 && !textureMap.containsKey(name.getText())) { - textureMap.put(name.getText(), parameterValue.getOuterImportObject().getObjectName().getText()); + textureMap.put(name.getText(), parameterValue); } } } } - FPackageIndex parent = getProp(matFirstExp, "Parent", FPackageIndex.class); + FPackageIndex parent = getProp(material, "Parent", FPackageIndex.class); if (parent != null && parent.getIndex() != 0) { populateTextures(parent); @@ -463,9 +422,14 @@ public void populateTextures(FPackageIndex pkgIndex) { } public void addToObj(JsonObject obj) { - String[][] textures = { // d n s e a + if (name.getIndex() == 0) { + obj.add(Integer.toHexString(hashCode()), null); + return; + } + + FPackageIndex[][] textures = { // d n s e a { - textureMap.getOrDefault("Trunk_BaseColor", textureMap.get("Diffuse")), + textureMap.getOrDefault("Trunk_BaseColor", textureMap.getOrDefault("Diffuse", textureMap.get("DiffuseTexture"))), textureMap.getOrDefault("Trunk_Normal", textureMap.get("Normals")), textureMap.getOrDefault("Trunk_Specular", textureMap.get("SpecularMasks")), textureMap.get("EmissiveTexture"), @@ -494,23 +458,31 @@ public void addToObj(JsonObject obj) { } }; - for (int i = 0; i < textures.length; i++) { + JsonArray array = new JsonArray(textures.length); + + for (FPackageIndex[] texture : textures) { boolean empty = true; - for (int j = 0; j < textures[i].length; j++) { - empty &= textures[i][j] == null; + for (FPackageIndex index : texture) { + empty &= index == null || index.getIndex() == 0; - if (textures[i][j] != null) { - toExport.add(textures[i][j]); + if (index != null && index.getIndex() != 0) { + exportTexture(index); } } - if (empty) { - textures[i] = new String[0]; + JsonArray subArray = new JsonArray(texture.length); + + if (!empty) { + for (FPackageIndex index : texture) { + subArray.add(pkgIndexToDirPath(index)); + } } + + array.add(subArray); } - obj.add(name.getIndex() != 0 ? name.getOuterImportObject().getObjectName().getText() : Integer.toHexString(hashCode()), GSON.toJsonTree(textures)); + obj.add(pkgIndexToDirPath(name), array); } } @@ -525,21 +497,15 @@ private static class BuildingTextureData { // public Float ResourceCost; } - private static class Config { + public static class Config { public String PaksDirectory = "C:\\Program Files\\Epic Games\\Fortnite\\FortniteGame\\Content\\Paks"; public Ue4Version UEVersion = Ue4Version.GAME_UE4_LATEST; - public EncryptionKey[] EncryptionKeys = {}; + public List EncryptionKeys = Collections.emptyList(); public boolean bReadMaterials = false; - public boolean bRunUModel = true; + public boolean bUseUModel = true; public String UModelAdditionalArgs = ""; public boolean bDumpAssets = false; public String ExportPackage; - - private static class EncryptionKey { - public FGuid Guid = FGuid.Companion.getMainGuid(); - public String FileName; - public byte[] Key = {}; - } } private static class MainException extends Exception { diff --git a/src/main/java/com/tb24/blenderumap/MyFileProvider.java b/src/main/java/com/tb24/blenderumap/MyFileProvider.java new file mode 100644 index 0000000..8afe4c2 --- /dev/null +++ b/src/main/java/com/tb24/blenderumap/MyFileProvider.java @@ -0,0 +1,175 @@ +package com.tb24.blenderumap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +import kotlin.collections.MapsKt; +import me.fungames.jfortniteparse.fileprovider.DefaultFileProvider; +import me.fungames.jfortniteparse.ue4.FGuid; +import me.fungames.jfortniteparse.ue4.assets.Package; +import me.fungames.jfortniteparse.ue4.assets.exports.UExport; +import me.fungames.jfortniteparse.ue4.assets.objects.FObjectExport; +import me.fungames.jfortniteparse.ue4.assets.objects.FObjectImport; +import me.fungames.jfortniteparse.ue4.assets.objects.FPackageIndex; +import me.fungames.jfortniteparse.ue4.pak.GameFile; +import me.fungames.jfortniteparse.ue4.pak.PakFileReader; +import me.fungames.jfortniteparse.ue4.versions.Ue4Version; + +import static com.tb24.blenderumap.JWPSerializer.GSON; + +public class MyFileProvider extends DefaultFileProvider { + private static final Logger LOGGER = LoggerFactory.getLogger("FileProvider"); + public static final File JSONS_FOLDER = new File("jsons"); + private final boolean bDumpAssets; + private final Map loaded = new HashMap<>(); + + public MyFileProvider(File folder, Ue4Version game, Iterable encryptionKeys, boolean bDumpAssets) { + super(folder, game); + this.bDumpAssets = bDumpAssets; + + Map keysToSubmit = new HashMap<>(); + + for (EncryptionKey entry : encryptionKeys) { + if (entry.FileName != null && !entry.FileName.isEmpty()) { + keysToSubmit.put(entry.Guid, entry.Key); + } else { + Optional foundGuid = getUnloadedPaks().stream().filter(it -> it.getFileName().equals(entry.FileName)).findFirst(); + + if (foundGuid.isPresent()) { + keysToSubmit.put(foundGuid.get().getPakInfo().getEncryptionKeyGuid(), entry.Key); + } else { + LOGGER.warn("PAK file not found: " + entry.FileName); + } + } + } + + submitKeys(keysToSubmit); + } + + public Package loadIfNot(String pkg) { + GameFile gameFile = findGameFile(pkg); + + if (gameFile != null) { + return loadIfNot(gameFile); + } else { + LOGGER.warn("Package " + pkg + " not found"); + return null; + } + } + + public Package loadIfNot(GameFile pkg) { + return MapsKt.getOrPut(loaded, pkg, () -> { + LOGGER.info("Loading " + pkg); + Package loadedPkg = loadGameFile(pkg); + + if (loadedPkg != null && bDumpAssets) { + File file = new File(JSONS_FOLDER, pkg.getPathWithoutExtension() + ".json"); + LOGGER.info("Writing JSON to " + file.getAbsolutePath()); + file.getParentFile().mkdirs(); + + try (FileWriter writer = new FileWriter(file)) { + GSON.toJson(loadedPkg.getExports(), writer); + } catch (IOException e) { + LOGGER.error("Writing failed", e); + } + } + + return loadedPkg; + }); + } + + public UExport loadObjectPath(String objectPath) { + if (objectPath == null) return null; + + int dotIndex = objectPath.lastIndexOf('.'); + String pkgPath, objectName; + + if (dotIndex != -1) { + pkgPath = objectPath.substring(0, dotIndex); + objectName = objectPath.substring(dotIndex + 1); + } else { + pkgPath = objectPath; + objectName = objectPath.substring(objectPath.lastIndexOf('/') + 1); + } + + Package pkg = loadIfNot(pkgPath); + + for (FObjectExport export : pkg.getExportMap()) { + if (export.getObjectName().getText().equals(objectName)) { + return export.getExportObject().getValue(); + } + } + + return null; + } + + public UExport loadObject(FPackageIndex index) { + if (index == null) return null; + + Package owner = index.getOwner(); + int i = index.getIndex(); + + if (i < 0) { // import + FObjectImport imp = owner.getImportMap().get(-i - 1); + Package pkg = loadIfNot(owner.getImportMap().get(-imp.getOuterIndex().getIndex() - 1).getObjectName().getText()); + + if (pkg != null) { + for (FObjectExport export : pkg.getExportMap()) { + if (export.getClassIndex().getName().equals(imp.getClassName().getText()) && export.getObjectName().getText().equals(imp.getObjectName().getText())) { + return export.getExportObject().getValue(); + } + } + } + } else if (i > 0) { // export + return owner.getExportMap().get(i - 1).getExportObject().getValue(); + } + + return null; + } + + public static String compactFilePath(String path) { + if (path.charAt(0) == '/') { + return path; + } + + if (path.startsWith("Engine/Content")) { // -> /Engine + return "/Engine" + path.substring("Engine/Content".length()); + } + + if (path.startsWith("Engine/Plugins")) { // -> /Plugins + return path.substring("Engine".length()); + } + + int delim = path.indexOf("/Content/"); + + if (delim == -1) { + return path; + } + + // GameName/Content -> /Game + return "/Game" + path.substring(delim + "/Content".length()); + } + + public static class EncryptionKey { + public FGuid Guid; + public String FileName; + public byte[] Key; + + public EncryptionKey() { + Guid = FGuid.Companion.getMainGuid(); + Key = new byte[]{}; + } + + public EncryptionKey(FGuid guid, byte[] key) { + Guid = guid; + Key = key; + } + } +} diff --git a/umap.py b/umap.py index 4dcf9d7..d143a3d 100644 --- a/umap.py +++ b/umap.py @@ -1,6 +1,6 @@ """ -BlenderUmap v0.2.0 -(C) 2020 amrsatrio. All rights reserved. +BlenderUmap v0.2.2 +(C) amrsatrio. All rights reserved. """ import bpy import json @@ -48,9 +48,9 @@ def import_umap(comps: list, attach_parent: bpy.types.Object = None) -> None: td_suffix = "" if mats and len(mats) > 0: - key += "_{:08x}".format(abs(string_hash_code(";".join(mats.keys())))) + key += f"_{abs(string_hash_code(';'.join(mats.keys()))):08x}" if texture_data and len(texture_data) > 0: - td_suffix = "_{:08x}".format(abs(string_hash_code(";".join([it[0] if it else "" for it in texture_data])))) + td_suffix = f"_{abs(string_hash_code(';'.join([it[0] if it else '' for it in texture_data]))):08x}" key += td_suffix existing_mesh = bpy.data.meshes.get(key) if reuse_meshes else None @@ -96,7 +96,8 @@ def import_umap(comps: list, attach_parent: bpy.types.Object = None) -> None: bpy.ops.object.shade_smooth() for m_idx, (m_path, m_textures) in enumerate(mats.items()): - import_material(m_idx, m_path, td_suffix, m_textures, texture_data) + if m_textures: + import_material(m_idx, m_path, td_suffix, m_textures, texture_data) else: print("WARNING: Failure importing mesh, defaulting to fallback mesh") fallback() @@ -167,7 +168,7 @@ def group(sub_tex_idx, location): tree.links.new(d_tex.outputs[0], sh.inputs[tex_index]) if tex_index is 4: # change mat blend method if there's an alpha mask texture - m.blend_method = "HASHED" + m.blend_method = "CLIP" return sh @@ -195,6 +196,8 @@ def group(sub_tex_idx, location): if m_idx < len(bpy.context.active_object.data.materials): bpy.context.active_object.data.materials[m_idx] = m + return m + def fallback() -> None: bpy.ops.mesh.primitive_plane_add(size=1) @@ -202,15 +205,27 @@ def fallback() -> None: def get_or_load_img(img_path: str) -> bpy.types.Image: - img_path = os.path.join(data_dir, img_path[1:] + ".tga") - existing = bpy.data.images.get(os.path.basename(img_path)) + name = os.path.basename(img_path) + existing = bpy.data.images.get(name) if existing: return existing - elif os.path.exists(img_path): + + img_path = os.path.join(data_dir, img_path[1:]) + + if os.path.exists(img_path + ".tga"): + img_path += ".tga" + elif os.path.exists(img_path + ".png"): + img_path += ".png" + elif os.path.exists(img_path + ".dds"): + img_path += ".dds" + + if os.path.exists(img_path): if verbose: print(img_path) - return bpy.data.images.load(filepath=img_path) + loaded = bpy.data.images.load(filepath=img_path) + loaded.name = name + return loaded else: print("WARNING: " + img_path + " not found") return None