Skip to content

Commit

Permalink
load ondemand archives
Browse files Browse the repository at this point in the history
  • Loading branch information
@4sval
4sval committed Nov 8, 2023
1 parent 5ef205c commit 88adcd0
Show file tree
Hide file tree
Showing 5 changed files with 32 additions and 107 deletions.
2 changes: 0 additions & 2 deletions .github/workflows/qa.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ name: FModel QA Builder
on:
push:
branches: [ dev ]
pull_request:
branches: [ dev ]

jobs:
build:
Expand Down
2 changes: 1 addition & 1 deletion CUE4Parse
Submodule CUE4Parse updated 11 files
+2 −0 CUE4Parse/FileProvider/ApkFileProvider.cs
+1 −1 CUE4Parse/FileProvider/Objects/VfcGameFile.cs
+27 −11 CUE4Parse/FileProvider/Vfs/AbstractVfsFileProvider.cs
+89 −0 CUE4Parse/UE4/IO/IoChunkToc.cs
+108 −0 CUE4Parse/UE4/IO/IoStoreOnDemandReader.cs
+7 −11 CUE4Parse/UE4/IO/IoStoreReader.cs
+1 −1 CUE4Parse/UE4/IO/Objects/FIoStoreTocResource.cs
+25 −0 CUE4Parse/UE4/IO/Objects/FOnDemandTocContainerEntry.cs
+25 −0 CUE4Parse/UE4/IO/Objects/FOnDemandTocEntry.cs
+56 −0 CUE4Parse/UE4/IO/Objects/FOnDemandTocHeader.cs
+18 −0 CUE4Parse/UE4/IO/Objects/FTocMeta.cs
3 changes: 1 addition & 2 deletions FModel/MainWindow.xaml.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,7 @@ private async void OnLoaded(object sender, RoutedEventArgs e)
#endif
await Task.WhenAll(
_applicationView.CUE4Parse.VerifyConsoleVariables(),
_applicationView.CUE4Parse.VerifyVirtualCache(),
_applicationView.CUE4Parse.VerifyContentBuildManifest(),
_applicationView.CUE4Parse.VerifyOnDemandArchives(),
_applicationView.CUE4Parse.InitMappings(),
_applicationView.InitImGuiSettings(newOrUpdated),
_applicationView.InitVgmStream(),
Expand Down
34 changes: 7 additions & 27 deletions FModel/ViewModels/ApiEndpoints/EpicApiEndpoint.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,12 @@ public class EpicApiEndpoint : AbstractApiProvider
private const string _OAUTH_URL = "https://account-public-service-prod03.ol.epicgames.com/account/api/oauth/token";
private const string _BASIC_TOKEN = "basic MzQ0NmNkNzI2OTRjNGE0NDg1ZDgxYjc3YWRiYjIxNDE6OTIwOWQ0YTVlMjVhNDU3ZmI5YjA3NDg5ZDMxM2I0MWE=";
private const string _APP_URL = "https://launcher-public-service-prod06.ol.epicgames.com/launcher/api/public/assets/v2/platform/Windows/namespace/fn/catalogItem/4fe75bbc5a674f4f9b356b5c90567da5/app/Fortnite/label/Live";
private const string _CBM_URL = "https://launcher-public-service-prod06.ol.epicgames.com/launcher/api/public/assets/Windows/5cb97847cee34581afdbc445400e2f77/FortniteContentBuilds";

public EpicApiEndpoint(RestClient client) : base(client) { }

public async Task<ManifestInfo> GetManifestAsync(CancellationToken token)
{
if (await IsExpired().ConfigureAwait(false))
{
var auth = await GetAuthAsync(token).ConfigureAwait(false);
if (auth != null)
{
UserSettings.Default.LastAuthResponse = auth;
}
}
await VerifyAuth(token).ConfigureAwait(false);

var request = new FRestRequest(_APP_URL);
request.AddHeader("Authorization", $"bearer {UserSettings.Default.LastAuthResponse.AccessToken}");
Expand All @@ -36,7 +28,12 @@ public async Task<ManifestInfo> GetManifestAsync(CancellationToken token)
return response.IsSuccessful ? new ManifestInfo(response.Content) : null;
}

public async Task<ContentBuildManifestInfo> GetContentBuildManifestAsync(CancellationToken token, string label)
public ManifestInfo GetManifest(CancellationToken token)
{
return GetManifestAsync(token).GetAwaiter().GetResult();
}

public async Task VerifyAuth(CancellationToken token)
{
if (await IsExpired().ConfigureAwait(false))
{
Expand All @@ -46,23 +43,6 @@ public async Task<ContentBuildManifestInfo> GetContentBuildManifestAsync(Cancell
UserSettings.Default.LastAuthResponse = auth;
}
}

var request = new FRestRequest(_CBM_URL);
request.AddHeader("Authorization", $"bearer {UserSettings.Default.LastAuthResponse.AccessToken}");
request.AddQueryParameter("label", label);
var response = await _client.ExecuteAsync(request, token).ConfigureAwait(false);
Log.Information("[{Method}] [{Status}({StatusCode})] '{Resource}'", request.Method, response.StatusDescription, (int) response.StatusCode, response.ResponseUri?.OriginalString);
return response.IsSuccessful ? new ContentBuildManifestInfo(response.Content) : null;
}

public ManifestInfo GetManifest(CancellationToken token)
{
return GetManifestAsync(token).GetAwaiter().GetResult();
}

public ContentBuildManifestInfo GetContentBuildManifest(CancellationToken token, string label)
{
return GetContentBuildManifestAsync(token, label).GetAwaiter().GetResult();
}

private async Task<AuthResponse> GetAuthAsync(CancellationToken token)
Expand Down
98 changes: 23 additions & 75 deletions FModel/ViewModels/CUE4ParseViewModel.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http.Headers;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
Expand Down Expand Up @@ -31,7 +32,6 @@
using CUE4Parse.UE4.Wwise;
using CUE4Parse_Conversion;
using CUE4Parse_Conversion.Sounds;
using CUE4Parse.FileProvider.Objects;
using CUE4Parse.UE4.Objects.Core.Serialization;
using EpicManifestParser.Objects;
using FModel.Creator;
Expand Down Expand Up @@ -121,7 +121,7 @@ public Snooper SnooperViewer
public AssetsFolderViewModel AssetsFolder { get; }
public SearchViewModel SearchVm { get; }
public TabControlViewModel TabControl { get; }
public ConfigIni BuildInfo { get; }
public ConfigIni IoStoreOnDemand { get; }

public CUE4ParseViewModel()
{
Expand Down Expand Up @@ -175,7 +175,7 @@ public CUE4ParseViewModel()
AssetsFolder = new AssetsFolderViewModel();
SearchVm = new SearchViewModel();
TabControl = new TabControlViewModel();
BuildInfo = new ConfigIni(nameof(BuildInfo));
IoStoreOnDemand = new ConfigIni(nameof(IoStoreOnDemand));
}

public async Task Initialize()
Expand Down Expand Up @@ -216,9 +216,9 @@ await _threadWorkerView.Begin(cancellationToken =>

foreach (var fileManifest in manifest.FileManifests)
{
if (fileManifest.Name.Equals("Cloud/BuildInfo.ini", StringComparison.OrdinalIgnoreCase))
if (fileManifest.Name.Equals("Cloud/IoStoreOnDemand.ini", StringComparison.OrdinalIgnoreCase))
{
BuildInfo.Read(new StreamReader(fileManifest.GetStream()));
IoStoreOnDemand.Read(new StreamReader(fileManifest.GetStream()));
continue;
}
if (!_fnLive.IsMatch(fileManifest.Name)) continue;
Expand Down Expand Up @@ -252,8 +252,8 @@ await _threadWorkerView.Begin(cancellationToken =>

break;
case DefaultFileProvider:
var buildInfoPath = Path.Combine(UserSettings.Default.GameDirectory, "..\\..\\..\\Cloud\\BuildInfo.ini");
if (File.Exists(buildInfoPath)) BuildInfo.Read(new StringReader(File.ReadAllText(buildInfoPath)));
var ioStoreOnDemandPath = Path.Combine(UserSettings.Default.GameDirectory, "..\\..\\..\\Cloud\\IoStoreOnDemand.ini");
if (File.Exists(ioStoreOnDemandPath)) IoStoreOnDemand.Read(new StringReader(File.ReadAllText(ioStoreOnDemandPath)));
break;
}

Expand Down Expand Up @@ -294,7 +294,7 @@ public void LoadVfs(CancellationToken token, IEnumerable<FileItem> aesKeys)
if (Provider.MountedVfs.FirstOrDefault(x => x.Name == file.Name) is not { } vfs)
{
if (Provider.UnloadedVfs.FirstOrDefault(x => x.Name == file.Name) is IoStoreReader store)
file.FileCount = (int) store.Info.TocEntryCount - 1;
file.FileCount = (int) store.TocResource.Header.TocEntryCount - 1;

continue;
}
Expand Down Expand Up @@ -436,87 +436,35 @@ public Task VerifyConsoleVariables()
});
}

private int _vfcCount { get; set; }
public Task VerifyVirtualCache()
{
if (Provider is StreamedFileProvider { LiveGame: "FortniteLive" } || _vfcCount > 0)
return Task.CompletedTask;

return Task.Run(() =>
{
_vfcCount = Provider.LoadVirtualCache();
if (_vfcCount > 0)
FLogger.Append(ELog.Information,
() => FLogger.Text($"{_vfcCount} cached packages loaded", Constants.WHITE, true));
});
}

public Task VerifyContentBuildManifest()
public Task VerifyOnDemandArchives()
{
// only local fortnite
if (Provider is not DefaultFileProvider || !Provider.InternalGameName.Equals("FortniteGame", StringComparison.OrdinalIgnoreCase))
return Task.CompletedTask;

// scuffed but working
var persistentDownloadDir = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "FortniteGame/Saved/PersistentDownloadDir");
var vfcMetadata = Path.Combine(persistentDownloadDir, "VFC", "vfc.meta");
if (!File.Exists(vfcMetadata))
var iasFileInfo = new FileInfo(Path.Combine(persistentDownloadDir, "ias", "ias.cache.0"));
if (!iasFileInfo.Exists || iasFileInfo.Length == 0)
return Task.CompletedTask;

// load if local fortnite with ondemand disabled
// VFC folder is created at launch if ondemand
// VFC folder is deleted at launch if not ondemand anymore
return Task.Run(() =>
return Task.Run(async () =>
{
var inst = new List<InstructionToken>();
BuildInfo.FindPropertyInstructions("Content", "Label", inst);
IoStoreOnDemand.FindPropertyInstructions("Endpoint", "TocPath", inst);
if (inst.Count <= 0) return;

var manifestInfo = _apiEndpointView.EpicApi.GetContentBuildManifest(default, inst[0].Value);
var manifestDir = new DirectoryInfo(Path.Combine(persistentDownloadDir, "ManifestCache"));
var manifestPath = Path.Combine(manifestDir.FullName, manifestInfo?.FileName ?? "");

byte[] manifestData;
if (File.Exists(manifestPath))
{
manifestData = File.ReadAllBytes(manifestPath);
}
else if (manifestInfo != null)
{
manifestData = manifestInfo.DownloadManifestData();
File.WriteAllBytes(manifestPath, manifestData);
}
else if (manifestDir.Exists && manifestDir.GetFiles("*.manifest") is { Length: > 0} cachedManifests)
{
manifestData = File.ReadAllBytes(cachedManifests[0].FullName);
}
else return;
var ioStoreOnDemandPath = Path.Combine(UserSettings.Default.GameDirectory, "..\\..\\..\\Cloud", inst[0].Value.SubstringAfterLast("/").SubstringBefore("\""));
if (!File.Exists(ioStoreOnDemandPath)) return;

var manifest = new Manifest(manifestData, new ManifestOptions
await _apiEndpointView.EpicApi.VerifyAuth(default);
await Provider.RegisterVfs(new IoChunkToc(ioStoreOnDemandPath), new IoStoreOnDemandOptions
{
ChunkBaseUri = new Uri("http://epicgames-download1.akamaized.net/Builds/Fortnite/Content/CloudDir/ChunksV4/", UriKind.Absolute),
ChunkCacheDirectory = Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, ".data"))
ChunkBaseUri = new Uri("https://download.epicgames.com/ias/fortnite/", UriKind.Absolute),
ChunkCacheDirectory = Directory.CreateDirectory(Path.Combine(UserSettings.Default.OutputDirectory, ".data")),
Authorization = new AuthenticationHeaderValue("Bearer", UserSettings.Default.LastAuthResponse.AccessToken)
});

var onDemandFiles = new Dictionary<string, GameFile>();
foreach (var fileManifest in manifest.FileManifests)
{
if (Provider.Files.TryGetValue(fileManifest.Name, out _)) continue;

var onDemandFile = new StreamedGameFile(fileManifest.Name, fileManifest.GetStream(), Provider.Versions);
if (Provider.IsCaseInsensitive) onDemandFiles[onDemandFile.Path.ToLowerInvariant()] = onDemandFile;
else onDemandFiles[onDemandFile.Path] = onDemandFile;
}

(Provider.Files as FileProviderDictionary)?.AddFiles(onDemandFiles);
if (onDemandFiles.Count > 0)
FLogger.Append(ELog.Information,
() => FLogger.Text($"{onDemandFiles.Count} streamed packages loaded", Constants.WHITE, true));
#if DEBUG

var missing = manifest.FileManifests.Count - onDemandFiles.Count;
if (missing > 0)
FLogger.Append(ELog.Debug,
() => FLogger.Text($"{missing} packages were already loaded by regular archives", Constants.WHITE, true));
#endif
await Provider.MountAsync();
});
}

Expand Down

0 comments on commit 88adcd0

Please sign in to comment.