diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java b/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java index 7e29067711..85a01a38f5 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/NewPipe.java @@ -26,6 +26,8 @@ import org.schabi.newpipe.extractor.localization.Localization; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -38,6 +40,8 @@ public final class NewPipe { private static Localization preferredLocalization; private static ContentCountry preferredContentCountry; + private static ExecutorService executorService; + private NewPipe() { } @@ -51,9 +55,15 @@ public static void init(final Downloader d, final Localization l) { } public static void init(final Downloader d, final Localization l, final ContentCountry c) { + init(d, l, c, Executors.newCachedThreadPool()); + } + + public static void init(final Downloader d, final Localization l, final ContentCountry c, + final ExecutorService e) { downloader = d; preferredLocalization = l; preferredContentCountry = c; + executorService = e; } public static Downloader getDownloader() { @@ -132,4 +142,13 @@ public static ContentCountry getPreferredContentCountry() { public static void setPreferredContentCountry(final ContentCountry preferredContentCountry) { NewPipe.preferredContentCountry = preferredContentCountry; } + + @Nonnull + public static ExecutorService getExecutorService() { + return executorService; + } + + public static void setExecutorService(final ExecutorService executorService) { + NewPipe.executorService = executorService; + } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java index cb7c21bdc1..57a2eab42d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java @@ -205,6 +205,7 @@ private YoutubeParsingHelper() { private static String[] youtubeMusicKey; private static boolean keyAndVersionExtracted = false; + private static final Object KEY_AND_VERSION_EXTRACTED_LOCK = new Object(); @SuppressWarnings("OptionalUsedAsFieldOrParameterType") private static Optional hardcodedClientVersionAndKeyValid = Optional.empty(); @@ -575,80 +576,85 @@ public static boolean areHardcodedClientVersionAndKeyValid() private static void extractClientVersionAndKeyFromSwJs() throws IOException, ExtractionException { - if (keyAndVersionExtracted) { - return; - } - final String url = "https://www.youtube.com/sw.js"; - final var headers = getOriginReferrerHeaders("https://www.youtube.com"); - final String response = getDownloader().get(url, headers).responseBody(); - try { - clientVersion = getStringResultFromRegexArray(response, - INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1); - key = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1); - } catch (final Parser.RegexException e) { - throw new ParsingException("Could not extract YouTube WEB InnerTube client version " - + "and API key from sw.js", e); + synchronized (KEY_AND_VERSION_EXTRACTED_LOCK) { + if (keyAndVersionExtracted) { + return; + } + final String url = "https://www.youtube.com/sw.js"; + final var headers = getOriginReferrerHeaders("https://www.youtube.com"); + final String response = getDownloader().get(url, headers).responseBody(); + try { + clientVersion = getStringResultFromRegexArray(response, + INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1); + key = getStringResultFromRegexArray(response, INNERTUBE_API_KEY_REGEXES, 1); + } catch (final Parser.RegexException e) { + throw new ParsingException("Could not extract YouTube WEB InnerTube client version " + + "and API key from sw.js", e); + } + keyAndVersionExtracted = true; } - keyAndVersionExtracted = true; } private static void extractClientVersionAndKeyFromHtmlSearchResultsPage() throws IOException, ExtractionException { - // Don't extract the client version and the InnerTube key if it has been already extracted - if (keyAndVersionExtracted) { - return; - } + synchronized (KEY_AND_VERSION_EXTRACTED_LOCK) { + // Don't extract the client version and the InnerTube key + // if it has been already extracted + if (keyAndVersionExtracted) { + return; + } - // Don't provide a search term in order to have a smaller response - final String url = "https://www.youtube.com/results?search_query=&ucbcb=1"; - final String html = getDownloader().get(url, getCookieHeader()).responseBody(); - final JsonObject initialData = getInitialData(html); - final JsonArray serviceTrackingParams = initialData.getObject("responseContext") - .getArray("serviceTrackingParams"); + // Don't provide a search term in order to have a smaller response + final String url = "https://www.youtube.com/results?search_query=&ucbcb=1"; + final String html = getDownloader().get(url, getCookieHeader()).responseBody(); + final JsonObject initialData = getInitialData(html); + final JsonArray serviceTrackingParams = initialData.getObject("responseContext") + .getArray("serviceTrackingParams"); - // Try to get version from initial data first - final Stream serviceTrackingParamsStream = serviceTrackingParams.stream() - .filter(JsonObject.class::isInstance) - .map(JsonObject.class::cast); + // Try to get version from initial data first + final Stream serviceTrackingParamsStream = serviceTrackingParams.stream() + .filter(JsonObject.class::isInstance) + .map(JsonObject.class::cast); + + clientVersion = getClientVersionFromServiceTrackingParam( + serviceTrackingParamsStream, "CSI", "cver"); - clientVersion = getClientVersionFromServiceTrackingParam( - serviceTrackingParamsStream, "CSI", "cver"); + if (clientVersion == null) { + try { + clientVersion = getStringResultFromRegexArray(html, + INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1); + } catch (final Parser.RegexException ignored) { + } + } + + // Fallback to get a shortened client version which does not contain the last two + // digits + if (isNullOrEmpty(clientVersion)) { + clientVersion = getClientVersionFromServiceTrackingParam( + serviceTrackingParamsStream, "ECATCHER", "client.version"); + } - if (clientVersion == null) { try { - clientVersion = getStringResultFromRegexArray(html, - INNERTUBE_CONTEXT_CLIENT_VERSION_REGEXES, 1); + key = getStringResultFromRegexArray(html, INNERTUBE_API_KEY_REGEXES, 1); } catch (final Parser.RegexException ignored) { } - } - - // Fallback to get a shortened client version which does not contain the last two - // digits - if (isNullOrEmpty(clientVersion)) { - clientVersion = getClientVersionFromServiceTrackingParam( - serviceTrackingParamsStream, "ECATCHER", "client.version"); - } - try { - key = getStringResultFromRegexArray(html, INNERTUBE_API_KEY_REGEXES, 1); - } catch (final Parser.RegexException ignored) { - } + if (isNullOrEmpty(key)) { + throw new ParsingException( + // CHECKSTYLE:OFF + "Could not extract YouTube WEB InnerTube API key from HTML search results page"); + // CHECKSTYLE:ON + } - if (isNullOrEmpty(key)) { - throw new ParsingException( - // CHECKSTYLE:OFF - "Could not extract YouTube WEB InnerTube API key from HTML search results page"); - // CHECKSTYLE:ON - } + if (clientVersion == null) { + throw new ParsingException( + // CHECKSTYLE:OFF + "Could not extract YouTube WEB InnerTube client version from HTML search results page"); + // CHECKSTYLE:ON + } - if (clientVersion == null) { - throw new ParsingException( - // CHECKSTYLE:OFF - "Could not extract YouTube WEB InnerTube client version from HTML search results page"); - // CHECKSTYLE:ON + keyAndVersionExtracted = true; } - - keyAndVersionExtracted = true; } @Nullable diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java index 113f2d4a3d..4e09508da6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java @@ -50,6 +50,7 @@ import org.schabi.newpipe.extractor.MediaFormat; import org.schabi.newpipe.extractor.MetaInfo; import org.schabi.newpipe.extractor.MultiInfoItemsCollector; +import org.schabi.newpipe.extractor.NewPipe; import org.schabi.newpipe.extractor.StreamingService; import org.schabi.newpipe.extractor.downloader.Downloader; import org.schabi.newpipe.extractor.exceptions.AgeRestrictedContentException; @@ -99,6 +100,8 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -821,6 +824,35 @@ public void onFetchPage(@Nonnull final Downloader downloader) final ContentCountry contentCountry = getExtractorContentCountry(); html5Cpn = generateContentPlaybackNonce(); + final Future nextFuture = NewPipe.getExecutorService().submit(() -> { + try { + final byte[] body = JsonWriter.string( + prepareDesktopJsonBuilder(localization, contentCountry) + .value(VIDEO_ID, videoId) + .value(CONTENT_CHECK_OK, true) + .value(RACY_CHECK_OK, true) + .done()) + .getBytes(StandardCharsets.UTF_8); + nextResponse = getJsonPostResponse(NEXT, body, localization); + } catch (final Exception e) { + throw new RuntimeException(e); + } + return null; + }); + + Future androidPlayerFuture = null; + + if (isAndroidClientFetchForced) { + androidPlayerFuture = getAndroidFetchFuture(contentCountry, localization, videoId); + } + + Future iosPlayerFuture = null; + + if (isIosClientFetchForced) { + iosPlayerFuture = getIosFetchFuture(contentCountry, localization, videoId); + } + + playerResponse = getJsonPostResponse(PLAYER, createDesktopPlayerBody(localization, contentCountry, videoId, sts, false, html5Cpn), @@ -885,24 +917,60 @@ public void onFetchPage(@Nonnull final Downloader downloader) // setStreamType()), so this block will be run only for POST_LIVE_STREAM and VIDEO_STREAM // values if fetching of the ANDROID client is not forced if ((!isAgeRestricted && streamType != StreamType.LIVE_STREAM) - || isAndroidClientFetchForced) { + && androidPlayerFuture == null) { + androidPlayerFuture = getAndroidFetchFuture(contentCountry, localization, videoId); + } + + if ((!isAgeRestricted && streamType == StreamType.LIVE_STREAM) + && iosPlayerFuture == null) { + iosPlayerFuture = getIosFetchFuture(contentCountry, localization, videoId); + } + + try { + nextFuture.get(); + + if (androidPlayerFuture != null) { + androidPlayerFuture.get(); + } + if (iosPlayerFuture != null) { + iosPlayerFuture.get(); + } + + } catch (final InterruptedException ignored) { + } catch (final ExecutionException e) { + final Throwable cause = e.getCause(); + if (cause != null) { + throw new RuntimeException(cause); + } + } + } + + private Future getAndroidFetchFuture(final ContentCountry contentCountry, + final Localization localization, + final String videoId) { + return NewPipe.getExecutorService().submit(() -> { try { fetchAndroidMobileJsonPlayer(contentCountry, localization, videoId); } catch (final Exception ignored) { // Ignore exceptions related to ANDROID client fetch or parsing, as it is not // compulsory to play contents } - } + return null; + }); + } - if ((!isAgeRestricted && streamType == StreamType.LIVE_STREAM) - || isIosClientFetchForced) { + private Future getIosFetchFuture(final ContentCountry contentCountry, + final Localization localization, + final String videoId) { + return NewPipe.getExecutorService().submit(() -> { try { fetchIosMobileJsonPlayer(contentCountry, localization, videoId); } catch (final Exception ignored) { // Ignore exceptions related to IOS client fetch or parsing, as it is not // compulsory to play contents } - } + return null; + }); } private void checkPlayabilityStatus(final JsonObject youtubePlayerResponse,