Skip to content

Attempt to parallelize YouTube Stream extractor #751

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: dev
Choose a base branch
from
Open

Attempt to parallelize YouTube Stream extractor #751

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ad08250
Parallelize YouTubeStreamExtractor with a ExecutorService.
FireMasterK Sep 7, 2022
7fd580f
Lock keyAndVersionExtracted while being read/written.
FireMasterK Jun 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import org.schabi.newpipe.extractor.localization.Localization;

import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
Expand All @@ -38,6 +40,8 @@ public final class NewPipe {
private static Localization preferredLocalization;
private static ContentCountry preferredContentCountry;

private static ExecutorService executorService;

private NewPipe() {
}

Expand All @@ -51,9 +55,15 @@ public static void init(final Downloader d, final Localization l) {
}

public static void init(final Downloader d, final Localization l, final ContentCountry c) {
init(d, l, c, Executors.newCachedThreadPool());
}

public static void init(final Downloader d, final Localization l, final ContentCountry c,
final ExecutorService e) {
downloader = d;
preferredLocalization = l;
preferredContentCountry = c;
executorService = e;
}

public static Downloader getDownloader() {
Expand Down Expand Up @@ -132,4 +142,13 @@ public static ContentCountry getPreferredContentCountry() {
public static void setPreferredContentCountry(final ContentCountry preferredContentCountry) {
NewPipe.preferredContentCountry = preferredContentCountry;
}

@Nonnull
public static ExecutorService getExecutorService() {
return executorService;
}

public static void setExecutorService(final ExecutorService executorService) {
NewPipe.executorService = executorService;
}
}
122 changes: 64 additions & 58 deletions ...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ private YoutubeParsingHelper() {
private static String[] youtubeMusicKey;

private static boolean keyAndVersionExtracted = false;
private static final Object KEY_AND_VERSION_EXTRACTED_LOCK = new Object();
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private static Optional<Boolean> hardcodedClientVersionAndKeyValid = Optional.empty();

Expand Down Expand Up @@ -575,80 +576,85 @@ public static boolean areHardcodedClientVersionAndKeyValid()

private static void extractClientVersionAndKeyFromSwJs()
throws IOException, ExtractionException {
if (keyAndVersionExtracted) {
return;
}
final String url = "https://www.youtube.com/sw.js";
final var headers = getOriginReferrerHeaders("https://www.youtube.com");
final String response = getDownloader().get(url, headers).responseBody();
try {
clientVersion = getStringResultFromRegexArray(response,
INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
key = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1);
} catch (final Parser.RegexException e) {
throw new ParsingException("Could not extract YouTube WEB InnerTube client version "
+ "and API key from sw.js", e);
synchronized (KEY_AND_VERSION_EXTRACTED_LOCK) {
if (keyAndVersionExtracted) {
return;
}
final String url = "https://www.youtube.com/sw.js";
final var headers = getOriginReferrerHeaders("https://www.youtube.com");
final String response = getDownloader().get(url, headers).responseBody();
try {
clientVersion = getStringResultFromRegexArray(response,
INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
key = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1);
} catch (final Parser.RegexException e) {
throw new ParsingException("Could not extract YouTube WEB InnerTube client version "
+ "and API key from sw.js", e);
}
keyAndVersionExtracted = true;
}
keyAndVersionExtracted = true;
}

private static void extractClientVersionAndKeyFromHtmlSearchResultsPage()
throws IOException, ExtractionException {
// Don't extract the client version and the InnerTube key if it has been already extracted
if (keyAndVersionExtracted) {
return;
}
synchronized (KEY_AND_VERSION_EXTRACTED_LOCK) {
// Don't extract the client version and the InnerTube key
// if it has been already extracted
if (keyAndVersionExtracted) {
return;
}

// Don't provide a search term in order to have a smaller response
final String url = "https://www.youtube.com/results?search_query=&ucbcb=1";
final String html = getDownloader().get(url, getCookieHeader()).responseBody();
final JsonObject initialData = getInitialData(html);
final JsonArray serviceTrackingParams = initialData.getObject("responseContext")
.getArray("serviceTrackingParams");
// Don't provide a search term in order to have a smaller response
final String url = "https://www.youtube.com/results?search_query=&ucbcb=1";
final String html = getDownloader().get(url, getCookieHeader()).responseBody();
final JsonObject initialData = getInitialData(html);
final JsonArray serviceTrackingParams = initialData.getObject("responseContext")
.getArray("serviceTrackingParams");

// Try to get version from initial data first
final Stream<JsonObject> serviceTrackingParamsStream = serviceTrackingParams.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast);
// Try to get version from initial data first
final Stream<JsonObject> serviceTrackingParamsStream = serviceTrackingParams.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast);

clientVersion = getClientVersionFromServiceTrackingParam(
serviceTrackingParamsStream, "CSI", "cver");

clientVersion = getClientVersionFromServiceTrackingParam(
serviceTrackingParamsStream, "CSI", "cver");
if (clientVersion == null) {
try {
clientVersion = getStringResultFromRegexArray(html,
INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
} catch (final Parser.RegexException ignored) {
}
}

// Fallback to get a shortened client version which does not contain the last two
// digits
if (isNullOrEmpty(clientVersion)) {
clientVersion = getClientVersionFromServiceTrackingParam(
serviceTrackingParamsStream, "ECATCHER", "client.version");
}

if (clientVersion == null) {
try {
clientVersion = getStringResultFromRegexArray(html,
INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1);
key = getStringResultFromRegexArray(html, INNERTUBE_API_KEY_REGEXES, 1);
} catch (final Parser.RegexException ignored) {
}
}

// Fallback to get a shortened client version which does not contain the last two
// digits
if (isNullOrEmpty(clientVersion)) {
clientVersion = getClientVersionFromServiceTrackingParam(
serviceTrackingParamsStream, "ECATCHER", "client.version");
}

try {
key = getStringResultFromRegexArray(html, INNERTUBE_API_KEY_REGEXES, 1);
} catch (final Parser.RegexException ignored) {
}
if (isNullOrEmpty(key)) {
throw new ParsingException(
// CHECKSTYLE:OFF
"Could not extract YouTube WEB InnerTube API key from HTML search results page");
// CHECKSTYLE:ON
}

if (isNullOrEmpty(key)) {
throw new ParsingException(
// CHECKSTYLE:OFF
"Could not extract YouTube WEB InnerTube API key from HTML search results page");
// CHECKSTYLE:ON
}
if (clientVersion == null) {
throw new ParsingException(
// CHECKSTYLE:OFF
"Could not extract YouTube WEB InnerTube client version from HTML search results page");
// CHECKSTYLE:ON
}

if (clientVersion == null) {
throw new ParsingException(
// CHECKSTYLE:OFF
"Could not extract YouTube WEB InnerTube client version from HTML search results page");
// CHECKSTYLE:ON
keyAndVersionExtracted = true;
}

keyAndVersionExtracted = true;
}

@Nullable
Expand Down
78 changes: 73 additions & 5 deletions ...java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.MultiInfoItemsCollector;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException;
Expand Down Expand Up @@ -99,6 +100,8 @@
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.stream.Collectors;

import javax.annotation.Nonnull;
Expand Down Expand Up @@ -821,6 +824,35 @@ public void onFetchPage(@Nonnull final Downloader downloader)
final ContentCountry contentCountry = getExtractorContentCountry();
html5Cpn = generateContentPlaybackNonce();

final Future<Void> nextFuture = NewPipe.getExecutorService().submit(() -> {
try {
final byte[] body = JsonWriter.string(
prepareDesktopJsonBuilder(localization, contentCountry)
.value(VIDEO_ID, videoId)
.value(CONTENT_CHECK_OK, true)
.value(RACY_CHECK_OK, true)
.done())
.getBytes(StandardCharsets.UTF_8);
nextResponse = getJsonPostResponse(NEXT, body, localization);
} catch (final Exception e) {
throw new RuntimeException(e);
}
return null;
});

Future<Void> androidPlayerFuture = null;

if (isAndroidClientFetchForced) {
androidPlayerFuture = getAndroidFetchFuture(contentCountry, localization, videoId);
}

Future<Void> iosPlayerFuture = null;

if (isIosClientFetchForced) {
iosPlayerFuture = getIosFetchFuture(contentCountry, localization, videoId);
}


playerResponse = getJsonPostResponse(PLAYER,
createDesktopPlayerBody(localization, contentCountry, videoId, sts, false,
html5Cpn),
Expand Down Expand Up @@ -885,24 +917,60 @@ public void onFetchPage(@Nonnull final Downloader downloader)
// setStreamType()), so this block will be run only for POST_LIVE_STREAM and VIDEO_STREAM
// values if fetching of the ANDROID client is not forced
if ((!isAgeRestricted && streamType != StreamType.LIVE_STREAM)
|| isAndroidClientFetchForced) {
&& androidPlayerFuture == null) {
androidPlayerFuture = getAndroidFetchFuture(contentCountry, localization, videoId);
}

if ((!isAgeRestricted && streamType == StreamType.LIVE_STREAM)
&& iosPlayerFuture == null) {
iosPlayerFuture = getIosFetchFuture(contentCountry, localization, videoId);
}

try {
nextFuture.get();

if (androidPlayerFuture != null) {
androidPlayerFuture.get();
}
if (iosPlayerFuture != null) {
iosPlayerFuture.get();
}

} catch (final InterruptedException ignored) {
} catch (final ExecutionException e) {
final Throwable cause = e.getCause();
if (cause != null) {
throw new RuntimeException(cause);
}
}
}

private Future<Void> getAndroidFetchFuture(final ContentCountry contentCountry,
final Localization localization,
final String videoId) {
return NewPipe.getExecutorService().submit(() -> {
try {
fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId);
} catch (final Exception ignored) {
// Ignore exceptions related to ANDROID client fetch or parsing, as it is not
// compulsory to play contents
}
}
return null;
});
}

if ((!isAgeRestricted && streamType == StreamType.LIVE_STREAM)
|| isIosClientFetchForced) {
private Future<Void> getIosFetchFuture(final ContentCountry contentCountry,
final Localization localization,
final String videoId) {
return NewPipe.getExecutorService().submit(() -> {
try {
fetchIosMobileJsonPlayer(contentCountry, localization, videoId);
} catch (final Exception ignored) {
// Ignore exceptions related to IOS client fetch or parsing, as it is not
// compulsory to play contents
}
}
return null;
});
}

private void checkPlayabilityStatus(final JsonObject youtubePlayerResponse,
Expand Down