From b8a126a440407cefbfdb632a58653643968e4ad5 Mon Sep 17 00:00:00 2001 From: Amrsatrio Date: Wed, 13 Jan 2021 01:38:51 +0700 Subject: [PATCH] FetchFortniteAesKey now fetches mappings for latest Using object classes does look good, but it breaks compatibility for games other than Fortnite --- build.gradle | 2 +- .../java/com/tb24/blenderumap/AssetUtils.kt | 2 +- .../tb24/blenderumap/FetchFortniteAesKey.java | 115 ++++++++++++----- .../com/tb24/blenderumap/JWPSerializer.kt | 39 +++--- src/main/java/com/tb24/blenderumap/Main.java | 117 ++++++++++++------ src/main/resources/log4j2.xml | 17 ++- 6 files changed, 204 insertions(+), 88 deletions(-) diff --git a/build.gradle b/build.gradle index eaff658..90ebfae 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ plugins { } group 'com.tb24' -version '0.3.4' +version '0.4.0' sourceCompatibility = 1.8 targetCompatibility = 1.8 diff --git a/src/main/java/com/tb24/blenderumap/AssetUtils.kt b/src/main/java/com/tb24/blenderumap/AssetUtils.kt index 5f4adcf..e1e21c4 100644 --- a/src/main/java/com/tb24/blenderumap/AssetUtils.kt +++ b/src/main/java/com/tb24/blenderumap/AssetUtils.kt @@ -14,7 +14,7 @@ fun getProp(properties: List, name: String, clazz: Class): } fun getProps(properties: List, name: String, clazz: Class): Array { - val collected: MutableList = ArrayList() + val collected = mutableListOf() var maxIndex = -1 for (prop in properties) { if (prop.name.text == name) { diff --git a/src/main/java/com/tb24/blenderumap/FetchFortniteAesKey.java b/src/main/java/com/tb24/blenderumap/FetchFortniteAesKey.java index 0ba8a7d..6e9be0d 100644 --- a/src/main/java/com/tb24/blenderumap/FetchFortniteAesKey.java +++ b/src/main/java/com/tb24/blenderumap/FetchFortniteAesKey.java @@ -12,50 +12,96 @@ import java.io.FileWriter; import java.io.IOException; import java.io.Reader; +import java.security.GeneralSecurityException; +import java.security.MessageDigest; +import java.util.Arrays; +import java.util.Date; import java.util.Map; +import kotlin.io.FilesKt; +import me.fungames.jfortniteparse.util.DataTypeConverterKt; import okhttp3.OkHttpClient; import okhttp3.Request; public class FetchFortniteAesKey { private static final Gson GSON = new GsonBuilder().setPrettyPrinting().create(); + private static final OkHttpClient okHttpClient = new OkHttpClient(); public static void main(String[] args) { try { - Reader reader = new OkHttpClient().newCall(new Request.Builder().url("https://benbotfn.tk/api/v1/aes").build()).execute().body().charStream(); - System.out.println("Fetched, writing..."); - AesResponse response = GSON.fromJson(reader, AesResponse.class); - reader.close(); - File configFile = new File("config.json"); - FileReader fileReader = new FileReader(configFile); - JsonElement configTree = JsonParser.parseReader(fileReader); - fileReader.close(); - JsonArray keys = configTree.getAsJsonObject().getAsJsonArray("EncryptionKeys"); - - while (keys.size() > 0) { - keys.remove(0); - } - - JsonObject mainKey = new JsonObject(); - mainKey.addProperty("Guid", "00000000000000000000000000000000"); - mainKey.addProperty("Key", response.mainKey); - keys.add(mainKey); - - for (Map.Entry entry : response.dynamicKeys.entrySet()) { - JsonObject dynKey = new JsonObject(); - dynKey.addProperty("FileName", entry.getKey().substring(entry.getKey().lastIndexOf('/') + 1)); - dynKey.addProperty("Key", entry.getValue()); - keys.add(dynKey); - } - - FileWriter fileWriter = new FileWriter(configFile); - GSON.toJson(configTree, fileWriter); - fileWriter.close(); + updateEncryptionKeys(); + updateMappings(); System.out.println("Done"); System.exit(0); - } catch (IOException e) { + } catch (GeneralSecurityException | IOException e) { e.printStackTrace(); + System.exit(1); + } + } + + private static void updateEncryptionKeys() throws IOException { + System.out.println("Fetching encryption keys..."); + Reader reader = okHttpClient.newCall(new Request.Builder().url("https://benbotfn.tk/api/v1/aes").build()).execute().body().charStream(); + System.out.println("Updating config..."); + AesResponse response = GSON.fromJson(reader, AesResponse.class); + reader.close(); + File configFile = new File("config.json"); + FileReader fileReader = new FileReader(configFile); + JsonElement configTree = JsonParser.parseReader(fileReader); + fileReader.close(); + JsonArray keys = configTree.getAsJsonObject().getAsJsonArray("EncryptionKeys"); + while (keys.size() > 0) { + keys.remove(0); + } + JsonObject mainKey = new JsonObject(); + mainKey.addProperty("Guid", "00000000000000000000000000000000"); + mainKey.addProperty("Key", response.mainKey); + keys.add(mainKey); + for (Map.Entry entry : response.dynamicKeys.entrySet()) { + JsonObject dynKey = new JsonObject(); + dynKey.addProperty("FileName", entry.getKey().substring(entry.getKey().lastIndexOf('/') + 1)); + dynKey.addProperty("Key", entry.getValue()); + keys.add(dynKey); } + FileWriter fileWriter = new FileWriter(configFile); + GSON.toJson(configTree, fileWriter); + fileWriter.close(); + } + + private static void updateMappings() throws IOException, GeneralSecurityException { + System.out.println("Fetching available mappings..."); + Reader reader = okHttpClient.newCall(new Request.Builder().url("https://benbotfn.tk/api/v1/mappings").build()).execute().body().charStream(); + FileEntry[] response = GSON.fromJson(reader, FileEntry[].class); + FileEntry chosen = null; + for (FileEntry entry : response) { + if ("Oodle".equalsIgnoreCase(entry.meta.get("compressionMethod"))) { + chosen = entry; + break; + } + } + if (chosen == null) { + System.out.println("No mappings found. Please supply your own Oodle compressed .usmap mappings in mappings folder."); + return; + } + File mappingsFolder = new File("mappings"); + File mappingsFile = new File(mappingsFolder, chosen.fileName); + if (mappingsFile.exists()) { + MessageDigest sha1 = MessageDigest.getInstance("SHA1"); + sha1.update(FilesKt.readBytes(mappingsFile)); + if (Arrays.equals(sha1.digest(), DataTypeConverterKt.parseHexBinary(chosen.hash))) { + System.out.println("Mappings already up to date."); + return; + } else { + System.out.println("Integrity check failed."); + } + } + System.out.println("Downloading latest mappings..."); + byte[] usmapData = okHttpClient.newCall(new Request.Builder().url(chosen.url).build()).execute().body().bytes(); + System.out.println("Saving mappings to " + mappingsFile.getAbsolutePath() + "..."); + if (!mappingsFolder.exists()) { + mappingsFolder.mkdir(); + } + FilesKt.writeBytes(mappingsFile, usmapData); } public static class AesResponse { @@ -63,4 +109,13 @@ public static class AesResponse { public String mainKey; public Map dynamicKeys; } + + public static class FileEntry { + public String url; + public String fileName; + public String hash; + public Long length; + public Date uploaded; + public Map meta; + } } diff --git a/src/main/java/com/tb24/blenderumap/JWPSerializer.kt b/src/main/java/com/tb24/blenderumap/JWPSerializer.kt index 52f93e4..ab106b4 100644 --- a/src/main/java/com/tb24/blenderumap/JWPSerializer.kt +++ b/src/main/java/com/tb24/blenderumap/JWPSerializer.kt @@ -5,7 +5,6 @@ package com.tb24.blenderumap import com.google.gson.* import me.fungames.jfortniteparse.ue4.assets.IoPackage -import me.fungames.jfortniteparse.ue4.assets.IoPackage.* import me.fungames.jfortniteparse.ue4.assets.PakPackage import me.fungames.jfortniteparse.ue4.assets.UProperty import me.fungames.jfortniteparse.ue4.assets.exports.ECurveTableMode @@ -17,6 +16,7 @@ import me.fungames.jfortniteparse.ue4.objects.core.i18n.FText import me.fungames.jfortniteparse.ue4.objects.core.i18n.FTextHistory import me.fungames.jfortniteparse.ue4.objects.core.math.FBox import me.fungames.jfortniteparse.ue4.objects.core.math.FBox2D +import me.fungames.jfortniteparse.ue4.objects.core.math.FColor import me.fungames.jfortniteparse.ue4.objects.core.misc.FGuid import me.fungames.jfortniteparse.ue4.objects.gameplaytags.FGameplayTagContainer import me.fungames.jfortniteparse.ue4.objects.uobject.FName @@ -46,7 +46,7 @@ object JWPSerializer { ?: separateCamelCase(it.name, "_").toLowerCase(Locale.ENGLISH) } .registerTypeAdapter(ByteArray::class.java, ByteArraySerializer()) - .registerTypeAdapter(UByte::class.java, JsonSerializer { src, typeOfSrc, context -> JsonPrimitive(src.toShort()) }) + .registerTypeAdapter(UByte::class.java, JsonSerializer { src, typeOfSrc, context -> JsonPrimitive(src.toString()) }) .registerTypeAdapter(UShort::class.java, JsonSerializer { src, typeOfSrc, context -> JsonPrimitive(src.toInt()) }) .registerTypeAdapter(UInt::class.java, JsonSerializer { src, typeOfSrc, context -> JsonPrimitive(src.toLong()) }) .registerTypeAdapter(ULong::class.java, JsonSerializer { src, typeOfSrc, context -> JsonPrimitive(BigInteger(src.toString())) }) @@ -68,6 +68,14 @@ object JWPSerializer { add("valid", context.serialize(src.isValid)) } }) + .registerTypeAdapter(FColor::class.java, JsonSerializer { src, typeOfSrc, context -> + JsonObject().apply { + add("r", JsonPrimitive(src.r.toShort())) + add("g", JsonPrimitive(src.g.toShort())) + add("b", JsonPrimitive(src.b.toShort())) + add("a", JsonPrimitive(src.a.toShort())) + } + }) .registerTypeAdapter(FGameplayTagContainer::class.java, JsonSerializer { src, typeOfSrc, context -> JsonObject().apply { add("gameplay_tags", JsonArray().apply { src.gameplayTags.forEach { add(context.serialize(it)) } }) @@ -80,27 +88,23 @@ object JWPSerializer { .registerTypeAdapter(FPackageIndex::class.java, JsonSerializer { src, typeOfSrc, context -> if (src.isImport()) { val pkg = src.owner - if (pkg is PakPackage) { - JsonArray().apply { + JsonArray().apply { + if (pkg is PakPackage) { var current = pkg.run { src.getResource() } while (current != null) { add(current.objectName.text) current = pkg.run { current!!.outerIndex.getResource() } } - } - } else { - when (val resolved = (pkg as IoPackage).resolveObjectIndex(pkg.importMap[src.toImport()])) { - is ResolvedScriptObject -> JsonArray().apply { - var current: ResolvedObject = resolved - do { - add(current.getName().text) - } while (current.getOuter()?.also { current = it } != null) + } else { + val initial = (pkg as IoPackage).resolveObjectIndex(pkg.importMap[src.toImport()]) + var current = initial + while (current != null) { + add(current.getName().text) + current = current.getOuter() } - is ResolvedExportObject -> JsonArray().apply { - add(resolved.getName().text) - add(resolved.pkg.fileName) + if (initial is IoPackage.ResolvedExportObject) { + add(initial.pkg.name) } - else -> null } } } else { @@ -188,9 +192,10 @@ object JWPSerializer { val obj = JsonObject() //if (sUseNonstandardFormat && src.export != null) obj.addProperty("object_name", src.export!!.objectName.text) obj.addProperty("export_type", src.exportType) + obj.addProperty("path_name", src.getPathName(src.owner)) if (src !is UDataTable || sUseNonstandardFormat) - serializeProperties(obj, (src as UObject).properties, context) + serializeProperties(obj, src.properties, context) if (src is UDataTable) { if (sUseNonstandardFormat) { diff --git a/src/main/java/com/tb24/blenderumap/Main.java b/src/main/java/com/tb24/blenderumap/Main.java index eb96393..e09b288 100644 --- a/src/main/java/com/tb24/blenderumap/Main.java +++ b/src/main/java/com/tb24/blenderumap/Main.java @@ -29,21 +29,25 @@ import javax.imageio.ImageIO; import kotlin.Lazy; +import kotlin.io.FilesKt; import kotlin.text.StringsKt; import me.fungames.jfortniteparse.fort.exports.BuildingTextureData; -import me.fungames.jfortniteparse.fort.exports.actors.BuildingSMActor; import me.fungames.jfortniteparse.ue4.assets.Package; +import me.fungames.jfortniteparse.ue4.assets.exports.UBlueprintGeneratedClass; +import me.fungames.jfortniteparse.ue4.assets.exports.ULevel; import me.fungames.jfortniteparse.ue4.assets.exports.UObject; import me.fungames.jfortniteparse.ue4.assets.exports.UStaticMesh; import me.fungames.jfortniteparse.ue4.assets.exports.UStruct; -import me.fungames.jfortniteparse.ue4.assets.exports.components.UStaticMeshComponent; +import me.fungames.jfortniteparse.ue4.assets.exports.UWorld; import me.fungames.jfortniteparse.ue4.assets.exports.mats.UMaterialInstance; import me.fungames.jfortniteparse.ue4.assets.exports.mats.UMaterialInstance.FTextureParameterValue; import me.fungames.jfortniteparse.ue4.assets.exports.mats.UMaterialInterface; +import me.fungames.jfortniteparse.ue4.assets.exports.tex.FTexturePlatformData; import me.fungames.jfortniteparse.ue4.assets.exports.tex.UTexture; import me.fungames.jfortniteparse.ue4.assets.exports.tex.UTexture2D; +import me.fungames.jfortniteparse.ue4.assets.mappings.TypeMappingsProvider; +import me.fungames.jfortniteparse.ue4.assets.mappings.UsmapTypeMappingsProvider; import me.fungames.jfortniteparse.ue4.assets.objects.meshes.FStaticMaterial; -import me.fungames.jfortniteparse.ue4.assets.util.StructFallbackReflectionUtilKt; import me.fungames.jfortniteparse.ue4.converters.meshes.StaticMeshesKt; import me.fungames.jfortniteparse.ue4.converters.meshes.psk.ExportStaticMeshKt; import me.fungames.jfortniteparse.ue4.converters.textures.TexturesKt; @@ -56,8 +60,10 @@ import me.fungames.jfortniteparse.ue4.versions.Ue4Version; import static com.tb24.blenderumap.AssetUtilsKt.getProp; +import static com.tb24.blenderumap.AssetUtilsKt.getProps; import static com.tb24.blenderumap.JWPSerializer.GSON; +@SuppressWarnings("unchecked") public class Main { private static final Logger LOGGER = LoggerFactory.getLogger("BlenderUmap"); private static Config config; @@ -95,6 +101,12 @@ public static void main(String[] args) { } provider = new MyFileProvider(paksDir, config.UEVersion, config.EncryptionKeys, config.bDumpAssets); + File newestUsmap = getNewestUsmap("mappings"); + if (newestUsmap != null) { + TypeMappingsProvider usmap = new UsmapTypeMappingsProvider(newestUsmap); + usmap.reload(); + provider.setMappingsProvider(usmap); + } Package pkg = exportAndProduceProcessed(config.ExportPackage); if (pkg == null) return; @@ -127,41 +139,67 @@ public static void main(String[] args) { } } - private static Package exportAndProduceProcessed(String s) { - Package pkg = provider.loadGameFile(s); + public static File getNewestUsmap(String directoryFilePath) { + File directory = new File(directoryFilePath); + File[] files = directory.listFiles(); + long lastModifiedTime = Long.MIN_VALUE; + File chosenFile = null; + + if (files != null) { + for (File file : files) { + if (file.isFile() && file.getName().endsWith(".usmap") && file.lastModified() > lastModifiedTime) { + chosenFile = file; + lastModifiedTime = file.lastModified(); + } + } + } - /*if (!s.endsWith(".umap")) { - LOGGER.info("{} is not an .umap, won't try to export", s); - return null; - }*/ + return chosenFile; + } + private static Package exportAndProduceProcessed(String path) { + if (path.endsWith(".umap") && provider.mountedIoStoreReaders().size() > 0) { + path = path.substring(0, path.lastIndexOf('.')); + } + UObject mainObject = provider.loadObject(path); + if (mainObject == null) { + LOGGER.info("Object not found"); + return null; + } + if (!(mainObject instanceof UWorld)) { + LOGGER.info(mainObject.getPathName() + " is not a UWorld, won't try to export"); + return null; + } + UWorld world = (UWorld) mainObject; + ULevel persistentLevel = world.getPersistentLevel().getValue(); JsonArray comps = new JsonArray(); - for (UObject object : pkg.getExports()) { - if (!(object instanceof BuildingSMActor)) { - continue; - } - BuildingSMActor actor = (BuildingSMActor) object; + for (Lazy actorLazy : persistentLevel.getActors()) { + if (actorLazy == null) continue; + UObject actor = actorLazy.getValue(); + if (actor.getExportType().equals("LODActor")) continue; - UStaticMeshComponent staticMeshExp = actor.StaticMeshComponent.getValue(); + Lazy staticMeshExpLazy = getProp(actor, "StaticMeshComponent", Lazy.class); // /Script/Engine.StaticMeshActor:StaticMeshComponent or /Script/FortniteGame.BuildingSMActor:StaticMeshComponent + if (staticMeshExpLazy == null) continue; + UObject staticMeshExp = staticMeshExpLazy.getValue(); if (staticMeshExp == null) continue; // identifiers JsonArray comp = new JsonArray(); comps.add(comp); - FGuid guid = actor.MyGuid; - comp.add(guid != null ? guid.toString() : UUID.randomUUID().toString().replace("-", "")); + FGuid guid = getProp(actor, "MyGuid", FGuid.class); // /Script/FortniteGame.BuildingActor:MyGuid + comp.add(guid != null ? guid.toString().toLowerCase() : UUID.randomUUID().toString().replace("-", "")); comp.add(actor.getName()); // region mesh - Lazy mesh = staticMeshExp.StaticMesh; + Lazy mesh = getProp(staticMeshExp, "StaticMesh", Lazy.class); // /Script/Engine.StaticMeshComponent:StaticMesh if (mesh == null) { // read the actor class to find the mesh UStruct actorBlueprint = actor.getClazz(); - if (actorBlueprint != null) { + if (actorBlueprint instanceof UBlueprintGeneratedClass) { for (UObject actorExp : actorBlueprint.getOwner().getExports()) { - if (actorExp instanceof UStaticMeshComponent && (mesh = ((UStaticMeshComponent) actorExp).StaticMesh) != null) { + if ((mesh = getProp(actorExp, "StaticMesh", Lazy.class)) != null) { break; } } @@ -195,16 +233,14 @@ private static Package exportAndProduceProcessed(String s) { } } - if (config.bReadMaterials) { - Lazy material = actor.BaseMaterial; - List> overrideMaterials = staticMeshExp.OverrideMaterials; - Lazy[] textureDataArr_ = actor.TextureData != null ? actor.TextureData : new Lazy[0]; + if (config.bReadMaterials /*&& actor instanceof BuildingSMActor*/) { + Lazy material = getProp(actor, "BaseMaterial", Lazy.class); // /Script/FortniteGame.BuildingSMActor:BaseMaterial + List> overrideMaterials = getProp(staticMeshExp, "OverrideMaterials", List.class); // /Script/Engine.MeshComponent:OverrideMaterials - for (Lazy textureDataIdx : textureDataArr_) { - BuildingTextureData texDataExp = textureDataIdx != null ? textureDataIdx.getValue() : null; + for (Lazy textureDataIdx : getProps(actor.getProperties(), "TextureData", Lazy.class)) { // /Script/FortniteGame.BuildingSMActor:TextureData + BuildingTextureData td = textureDataIdx != null ? textureDataIdx.getValue() : null; - if (texDataExp != null) { - BuildingTextureData td = StructFallbackReflectionUtilKt.mapToClass(texDataExp, BuildingTextureData.class); + if (td != null) { JsonArray textures = new JsonArray(); addToArray(textures, td.Diffuse); addToArray(textures, td.Normal); @@ -236,7 +272,7 @@ private static Package exportAndProduceProcessed(String s) { // region additional worlds JsonArray children = new JsonArray(); - FSoftObjectPath[] additionalWorlds = getProp(actor, "AdditionalWorlds", FSoftObjectPath[].class); + List additionalWorlds = getProp(actor, "AdditionalWorlds", List.class); // /Script/FortniteGame.BuildingFoundation:AdditionalWorlds if (additionalWorlds != null) { for (FSoftObjectPath additionalWorld : additionalWorlds) { @@ -250,12 +286,13 @@ private static Package exportAndProduceProcessed(String s) { comp.add(pkgIndexToDirPath(mesh)); comp.add(matsObj); comp.add(textureDataArr); - comp.add(vector(staticMeshExp.RelativeLocation)); - comp.add(rotator(staticMeshExp.RelativeRotation)); - comp.add(vector(staticMeshExp.RelativeScale3D)); + comp.add(vector(getProp(staticMeshExp, "RelativeLocation", FVector.class))); // /Script/Engine.SceneComponent:RelativeLocation + comp.add(rotator(getProp(staticMeshExp, "RelativeRotation", FRotator.class))); // /Script/Engine.SceneComponent:RelativeRotation + comp.add(vector(getProp(staticMeshExp, "RelativeScale3D", FVector.class))); // /Script/Engine.SceneComponent:RelativeScale3D comp.add(children); } + Package pkg = world.getOwner(); String pkgName = provider.compactFilePath(pkg.getName()); if (!pkgName.endsWith(".umap")) { pkgName += ".umap"; @@ -289,14 +326,24 @@ private static void exportTexture(Lazy index) { } try { - UTexture texExport = index.getValue(); - File output = new File(getExportDir(texExport), texExport.getName() + ".png"); + UTexture2D texture = index.getValue() instanceof UTexture2D ? (UTexture2D) index.getValue() : null; + if (texture == null) { + return; + } + FTexturePlatformData platformData = texture.getFirstTexture(); + char[] fourCC = TexturesKt.getDdsFourCC(platformData); + File output = new File(getExportDir(texture), texture.getName() + (fourCC != null ? ".dds" : ".png")); if (output.exists()) { LOGGER.debug("Texture already exists, skipping: {}", output.getAbsolutePath()); } else { LOGGER.info("Saving texture to {}", output.getAbsolutePath()); - ImageIO.write(TexturesKt.toBufferedImage((UTexture2D) texExport), "png", output); + + if (fourCC != null) { + FilesKt.writeBytes(output, TexturesKt.toDdsArray(texture, platformData, platformData.getFirstLoadedMip())); + } else { + ImageIO.write(TexturesKt.toBufferedImage(texture, platformData, platformData.getFirstLoadedMip()), "png", output); + } } } catch (Exception e) { LOGGER.warn("Failed to save texture", e); diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml index a7aa74a..70957b7 100644 --- a/src/main/resources/log4j2.xml +++ b/src/main/resources/log4j2.xml @@ -1,16 +1,25 @@ + + %c: %p: %m%n%throwable + - - + + + + + + + - + - + +