From 9913e14d6959be35eaa4ca9babbaaed5f97f9a5d Mon Sep 17 00:00:00 2001 From: Sp5rky Date: Wed, 9 Apr 2025 15:07:35 -0600 Subject: [PATCH] add services --- README.md | 14 + services/ALL4/__init__.py | 414 +++++++++ services/ALL4/config.yaml | 27 + services/AMZN/__init__.py | 548 ++++++++++++ services/AMZN/config.yaml | 62 ++ services/AUBC/__init__.py | 250 ++++++ services/AUBC/config.yaml | 9 + services/CBC/__init__.py | 318 +++++++ services/CBC/config.yaml | 7 + services/CBS/__init__.py | 241 ++++++ services/CBS/config.yaml | 10 + services/CR/__init__.py | 533 ++++++++++++ services/CR/config.yaml | 9 + services/CRAVE/__init__.py | 289 +++++++ services/CRAVE/config.yaml | 10 + services/CRKL/__init__.py | 138 +++ services/CTV/__init__.py | 374 ++++++++ services/CTV/config.yaml | 6 + services/DROP/__init__.py | 236 +++++ services/DROP/config.yaml | 15 + services/DSCP/__init__.py | 300 +++++++ services/DSCP/config.yaml | 4 + services/DSNP/__init__.py | 630 ++++++++++++++ services/DSNP/config.yaml | 22 + services/DSNP/queries.py | 13 + services/HM/__init__.py | 139 +++ services/HULU/__init__.py | 356 ++++++++ services/HULU/config.yaml | 54 ++ services/Hotstar/__init__.py | 377 ++++++++ services/Hotstar/config.yaml | 18 + services/ITV/__init__.py | 365 ++++++++ services/ITV/config.yaml | 8 + services/MAX/__init__.py | 496 +++++++++++ services/MAX/config.yaml | 5 + services/MY5/__init__.py | 207 +++++ services/MY5/config.yaml | 38 + services/NF/MSL/MSLKeys.py | 10 + services/NF/MSL/MSLObject.py | 6 + services/NF/MSL/__init__.py | 450 ++++++++++ .../NF/MSL/schemes/EntityAuthentication.py | 59 ++ services/NF/MSL/schemes/KeyExchangeRequest.py | 80 ++ services/NF/MSL/schemes/UserAuthentication.py | 59 ++ services/NF/MSL/schemes/__init__.py | 24 + services/NF/__init__.py | 781 +++++++++++++++++ services/NF/config.yaml | 185 ++++ services/NOW/__init__.py | 384 +++++++++ services/NOW/config.yaml | 27 + services/PCOK/__init__.py | 470 ++++++++++ services/PCOK/config.yaml | 27 + services/PLUTO/__init__.py | 292 +++++++ services/PLUTO/config.yaml | 7 + services/PMTP/__init__.py | 292 +++++++ services/PMTP/config.yaml | 19 + services/RBOX/__init__.py | 149 ++++ services/ROKU/__init__.py | 249 ++++++ services/ROKU/config.yaml | 5 + services/RTE/__init__.py | 284 ++++++ services/RTE/config.yaml | 7 + services/SPOT/__init__.py | 256 ++++++ services/SPOT/config.yaml | 9 + services/STV/__init__.py | 230 +++++ services/STV/config.yaml | 20 + services/TFC/__init__.py | 454 ++++++++++ services/TFC/config.yaml | 35 + services/TUBI/__init__.py | 211 +++++ services/TUBI/config.yaml | 5 + services/TVNZ/__init__.py | 304 +++++++ services/TVNZ/config.yaml | 9 + services/UKTV/__init__.py | 193 +++++ services/UKTV/config.yaml | 9 + services/UNEXT/__init__.py | 339 ++++++++ services/UNEXT/config.yaml | 102 +++ services/VIKI/__init__.py | 483 +++++++++++ services/VIKI/config.yaml | 26 + services/VIU/__init__.py | 814 ++++++++++++++++++ services/VIU/config.yaml | 23 + services/WTCH/__init__.py | 237 +++++ services/WTCH/config.yaml | 15 + services/iP/__init__.py | 374 ++++++++ services/iP/config.yaml | 11 + 80 files changed, 14567 insertions(+) create mode 100644 README.md create mode 100644 services/ALL4/__init__.py create mode 100644 services/ALL4/config.yaml create mode 100644 services/AMZN/__init__.py create mode 100644 services/AMZN/config.yaml create mode 100644 services/AUBC/__init__.py create mode 100644 services/AUBC/config.yaml create mode 100644 services/CBC/__init__.py create mode 100644 services/CBC/config.yaml create mode 100644 services/CBS/__init__.py create mode 100644 services/CBS/config.yaml create mode 100644 services/CR/__init__.py create mode 100644 services/CR/config.yaml create mode 100644 services/CRAVE/__init__.py create mode 100644 services/CRAVE/config.yaml create mode 100644 services/CRKL/__init__.py create mode 100644 services/CTV/__init__.py create mode 100644 services/CTV/config.yaml create mode 100644 services/DROP/__init__.py create mode 100644 services/DROP/config.yaml create mode 100644 services/DSCP/__init__.py create mode 100644 services/DSCP/config.yaml create mode 100644 services/DSNP/__init__.py create mode 100644 services/DSNP/config.yaml create mode 100644 services/DSNP/queries.py create mode 100644 services/HM/__init__.py create mode 100644 services/HULU/__init__.py create mode 100644 services/HULU/config.yaml create mode 100644 services/Hotstar/__init__.py create mode 100644 services/Hotstar/config.yaml create mode 100644 services/ITV/__init__.py create mode 100644 services/ITV/config.yaml create mode 100644 services/MAX/__init__.py create mode 100644 services/MAX/config.yaml create mode 100644 services/MY5/__init__.py create mode 100644 services/MY5/config.yaml create mode 100644 services/NF/MSL/MSLKeys.py create mode 100644 services/NF/MSL/MSLObject.py create mode 100644 services/NF/MSL/__init__.py create mode 100644 services/NF/MSL/schemes/EntityAuthentication.py create mode 100644 services/NF/MSL/schemes/KeyExchangeRequest.py create mode 100644 services/NF/MSL/schemes/UserAuthentication.py create mode 100644 services/NF/MSL/schemes/__init__.py create mode 100644 services/NF/__init__.py create mode 100644 services/NF/config.yaml create mode 100644 services/NOW/__init__.py create mode 100644 services/NOW/config.yaml create mode 100644 services/PCOK/__init__.py create mode 100644 services/PCOK/config.yaml create mode 100644 services/PLUTO/__init__.py create mode 100644 services/PLUTO/config.yaml create mode 100644 services/PMTP/__init__.py create mode 100644 services/PMTP/config.yaml create mode 100644 services/RBOX/__init__.py create mode 100644 services/ROKU/__init__.py create mode 100644 services/ROKU/config.yaml create mode 100644 services/RTE/__init__.py create mode 100644 services/RTE/config.yaml create mode 100644 services/SPOT/__init__.py create mode 100644 services/SPOT/config.yaml create mode 100644 services/STV/__init__.py create mode 100644 services/STV/config.yaml create mode 100644 services/TFC/__init__.py create mode 100644 services/TFC/config.yaml create mode 100644 services/TUBI/__init__.py create mode 100644 services/TUBI/config.yaml create mode 100644 services/TVNZ/__init__.py create mode 100644 services/TVNZ/config.yaml create mode 100644 services/UKTV/__init__.py create mode 100644 services/UKTV/config.yaml create mode 100644 services/UNEXT/__init__.py create mode 100644 services/UNEXT/config.yaml create mode 100644 services/VIKI/__init__.py create mode 100644 services/VIKI/config.yaml create mode 100644 services/VIU/__init__.py create mode 100644 services/VIU/config.yaml create mode 100644 services/WTCH/__init__.py create mode 100644 services/WTCH/config.yaml create mode 100644 services/iP/__init__.py create mode 100644 services/iP/config.yaml diff --git a/README.md b/README.md new file mode 100644 index 0000000..1819713 --- /dev/null +++ b/README.md @@ -0,0 +1,14 @@ +## Usage: +Clone repository: + +`git clone https://git.drmlab.io/ImSp4rky/sp4rky-devine-services.git` + +Add folder to `devine.yaml`: + +``` +directories: + services: "path/to/services" +``` +See help text for each service: + +`devine dl SERVICE -?` \ No newline at end of file diff --git a/services/ALL4/__init__.py b/services/ALL4/__init__.py new file mode 100644 index 0000000..6baeb8e --- /dev/null +++ b/services/ALL4/__init__.py @@ -0,0 +1,414 @@ +from __future__ import annotations + +import base64 +import hashlib +import json +import sys +import re +from collections.abc import Generator +from datetime import datetime, timezone +from http.cookiejar import MozillaCookieJar +from typing import Any, Optional, Union + +import click +from click import Context +from Crypto.Util.Padding import unpad +from Cryptodome.Cipher import AES +from pywidevine.cdm import Cdm as WidevineCdm + +from devine.core.credential import Credential +from devine.core.manifests.dash import DASH +from devine.core.search_result import SearchResult +from devine.core.service import Service +from devine.core.titles import Episode, Movie, Movies, Series +from devine.core.tracks import Chapter, Subtitle, Tracks + + +class ALL4(Service): + """ + Service code for Channel 4's All4 streaming service (https://channel4.com). + + \b + Author: stabbedbybrick + Authorization: Credentials + Robustness: + L3: 1080p, AAC2.0 + + \b + Tips: + - Use complete title URL or slug as input: + https://www.channel4.com/programmes/taskmaster OR taskmaster + - Use on demand URL for directly downloading episodes: + https://www.channel4.com/programmes/taskmaster/on-demand/75588-002 + - Both android and web/pc endpoints are checked for quality profiles. + If android is missing 1080p, it automatically falls back to web. + """ + + GEOFENCE = ("gb", "ie") + TITLE_RE = r"^(?:https?://(?:www\.)?channel4\.com/programmes/)?(?P[a-z0-9-]+)(?:/on-demand/(?P[0-9-]+))?" + + @staticmethod + @click.command(name="ALL4", short_help="https://channel4.com", help=__doc__) + @click.argument("title", type=str) + @click.pass_context + def cli(ctx: Context, **kwargs: Any) -> ALL4: + return ALL4(ctx, **kwargs) + + def __init__(self, ctx: Context, title: str): + self.title = title + super().__init__(ctx) + + self.authorization: str + self.asset_id: int + self.license_token: str + self.manifest: str + + self.session.headers.update( + { + "X-C4-Platform-Name": self.config["device"]["platform_name"], + "X-C4-Device-Type": self.config["device"]["device_type"], + "X-C4-Device-Name": self.config["device"]["device_name"], + "X-C4-App-Version": self.config["device"]["app_version"], + "X-C4-Optimizely-Datafile": self.config["device"]["optimizely_datafile"], + } + ) + + def authenticate(self, cookies: Optional[MozillaCookieJar] = None, credential: Optional[Credential] = None) -> None: + super().authenticate(cookies, credential) + if not credential: + raise EnvironmentError("Service requires Credentials for Authentication.") + + cache = self.cache.get(f"tokens_{credential.sha1}") + + if cache and not cache.expired: + # cached + self.log.info(" + Using cached Tokens...") + tokens = cache.data + elif cache and cache.expired: + # expired, refresh + self.log.info("Refreshing cached Tokens") + r = self.session.post( + self.config["endpoints"]["login"], + headers={"authorization": f"Basic {self.config['android']['auth']}"}, + data={ + "grant_type": "refresh_token", + "username": credential.username, + "password": credential.password, + "refresh_token": cache.data["refreshToken"], + }, + ) + try: + res = r.json() + except json.JSONDecodeError: + raise ValueError(f"Failed to refresh tokens: {r.text}") + + if "error" in res: + self.log.error(f"Failed to refresh tokens: {res['errorMessage']}") + sys.exit(1) + + tokens = res + self.log.info(" + Refreshed") + else: + # new + headers = {"authorization": f"Basic {self.config['android']['auth']}"} + data = { + "grant_type": "password", + "username": credential.username, + "password": credential.password, + } + r = self.session.post(self.config["endpoints"]["login"], headers=headers, data=data) + try: + res = r.json() + except json.JSONDecodeError: + raise ValueError(f"Failed to log in: {r.text}") + + if "error" in res: + self.log.error(f"Failed to log in: {res['errorMessage']}") + sys.exit(1) + + tokens = res + self.log.info(" + Acquired tokens...") + + cache.set(tokens, expiration=tokens["expiresIn"]) + + self.authorization = f"Bearer {tokens['accessToken']}" + + def search(self) -> Generator[SearchResult, None, None]: + params = { + "expand": "default", + "q": self.title, + "limit": "100", + "offset": "0", + } + + r = self.session.get(self.config["endpoints"]["search"], params=params) + r.raise_for_status() + + results = r.json() + if isinstance(results["results"], list): + for result in results["results"]: + yield SearchResult( + id_=result["brand"].get("websafeTitle"), + title=result["brand"].get("title"), + description=result["brand"].get("description"), + label=result.get("label"), + url=result["brand"].get("href"), + ) + + def get_titles(self) -> Union[Movies, Series]: + title, on_demand = (re.match(self.TITLE_RE, self.title).group(i) for i in ("id", "vid")) + + r = self.session.get( + self.config["endpoints"]["title"].format(title=title), + params={"client": "android-mod", "deviceGroup": "mobile", "include": "extended-restart"}, + headers={"Authorization": self.authorization}, + ) + if not r.ok: + self.log.error(r.text) + sys.exit(1) + + data = r.json() + + if on_demand is not None: + episodes = [ + Episode( + id_=episode["programmeId"], + service=self.__class__, + title=data["brand"]["title"], + season=episode["seriesNumber"], + number=episode["episodeNumber"], + name=episode["originalTitle"], + language="en", + data=episode["assetInfo"].get("streaming") or episode["assetInfo"].get("download"), + ) + for episode in data["brand"]["episodes"] + if episode.get("assetInfo") and episode["programmeId"] == on_demand + ] + if not episodes: + # Parse HTML of episode page to find title + data = self.get_html(self.title) + episodes = [ + Episode( + id_=data["selectedEpisode"]["programmeId"], + service=self.__class__, + title=data["brand"]["title"], + season=data["selectedEpisode"]["seriesNumber"] or 0, + number=data["selectedEpisode"]["episodeNumber"] or 0, + name=data["selectedEpisode"]["originalTitle"], + language="en", + data=data["selectedEpisode"], + ) + ] + + return Series(episodes) + + elif data["brand"]["programmeType"] == "FM": + return Movies( + [ + Movie( + id_=movie["programmeId"], + service=self.__class__, + name=data["brand"]["title"], + year=int(data["brand"]["summary"].split(" ")[0].strip().strip("()")), + language="en", + data=movie["assetInfo"].get("streaming") or movie["assetInfo"].get("download"), + ) + for movie in data["brand"]["episodes"] + ] + ) + else: + return Series( + [ + Episode( + id_=episode["programmeId"], + service=self.__class__, + title=data["brand"]["title"], + season=episode["seriesNumber"], + number=episode["episodeNumber"], + name=episode["originalTitle"], + language="en", + data=episode["assetInfo"].get("streaming") or episode["assetInfo"].get("download"), + ) + for episode in data["brand"]["episodes"] + if episode.get("assetInfo") + ] + ) + + def get_tracks(self, title: Union[Movie, Episode]) -> Tracks: + android_assets: tuple = self.android_playlist(title.id) + web_assets: tuple = self.web_playlist(title.id) + self.manifest, self.license_token, subtitle, data = self.sort_assets(title, android_assets, web_assets) + self.asset_id = int(title.data["assetId"]) + + tracks = DASH.from_url(self.manifest, self.session).to_tracks(title.language) + tracks.videos[0].data = data + + # manifest subtitles are sometimes empty even if they exist + # so we clear them and add the subtitles manually + tracks.subtitles.clear() + if subtitle is not None: + tracks.add( + Subtitle( + id_=hashlib.md5(subtitle.encode()).hexdigest()[0:6], + url=subtitle, + codec=Subtitle.Codec.from_mime(subtitle[-3:]), + language=title.language, + is_original_lang=True, + forced=False, + sdh=False, + ) + ) + else: + self.log.warning("- Subtitles are either missing or empty") + + return tracks + + def get_chapters(self, title: Union[Movie, Episode]) -> list[Chapter]: + track = title.tracks.videos[0] + + chapters = [ + Chapter( + name=f"Chapter {i + 1:02}", + timestamp=datetime.fromtimestamp((ms / 1000), tz=timezone.utc).strftime("%H:%M:%S.%f")[:-3], + ) + for i, ms in enumerate(x["breakOffset"] for x in track.data["adverts"]["breaks"]) + ] + + if track.data.get("endCredits", {}).get("squeezeIn"): + chapters.append( + Chapter( + name="Credits", + timestamp=datetime.fromtimestamp( + (track.data["endCredits"]["squeezeIn"] / 1000), tz=timezone.utc + ).strftime("%H:%M:%S.%f")[:-3], + ) + ) + + return chapters + + def get_widevine_service_certificate(self, **_: Any) -> str: + return WidevineCdm.common_privacy_cert + + def get_widevine_license(self, challenge: bytes, **_: Any) -> str: + payload = { + "message": base64.b64encode(challenge).decode("utf8"), + "token": self.license_token, + "request_id": self.asset_id, + "video": {"type": "ondemand", "url": self.manifest}, + } + + r = self.session.post(self.config["endpoints"]["license"], json=payload) + if not r.ok: + raise ConnectionError(f"License request failed: {r.json()['status']['type']}") + + return r.json()["license"] + + # Service specific functions + + def sort_assets(self, title: Union[Movie, Episode], android_assets: tuple, web_assets: tuple) -> tuple: + android_heights = None + web_heights = None + + if android_assets is not None: + try: + a_manifest, a_token, a_subtitle, data = android_assets + android_tracks = DASH.from_url(a_manifest, self.session).to_tracks(title.language) + android_heights = sorted([int(track.height) for track in android_tracks.videos], reverse=True) + except Exception: + android_heights = None + + if web_assets is not None: + try: + b_manifest, b_token, b_subtitle, data = web_assets + session = self.session + session.headers.update(self.config["headers"]) + web_tracks = DASH.from_url(b_manifest, session).to_tracks(title.language) + web_heights = sorted([int(track.height) for track in web_tracks.videos], reverse=True) + except Exception: + web_heights = None + + if not android_heights and not web_heights: + self.log.error("Failed to request manifest data. If you're behind a VPN/proxy, you might be blocked") + sys.exit(1) + + if not android_heights or android_heights[0] < 1080: + lic_token = self.decrypt_token(b_token, client="WEB") + return b_manifest, lic_token, b_subtitle, data + else: + lic_token = self.decrypt_token(a_token, client="ANDROID") + return a_manifest, lic_token, a_subtitle, data + + def android_playlist(self, video_id: str) -> tuple: + url = self.config["android"]["vod"].format(video_id=video_id) + headers = {"authorization": self.authorization} + + r = self.session.get(url=url, headers=headers) + if not r.ok: + self.log.warning("Request for Android endpoint returned %s", r) + return None + + data = json.loads(r.content) + manifest = data["videoProfiles"][0]["streams"][0]["uri"] + token = data["videoProfiles"][0]["streams"][0]["token"] + subtitle = next( + (x["url"] for x in data["subtitlesAssets"] if x["url"].endswith(".vtt")), + None, + ) + + return manifest, token, subtitle, data + + def web_playlist(self, video_id: str) -> tuple: + url = self.config["web"]["vod"].format(programmeId=video_id) + r = self.session.get(url, headers=self.config["headers"]) + if not r.ok: + self.log.warning("Request for WEB endpoint returned %s", r) + return None + + data = json.loads(r.content) + + for item in data["videoProfiles"]: + if item["name"] == "dashwv-dyn-stream-1": + token = item["streams"][0]["token"] + manifest = item["streams"][0]["uri"] + + subtitle = next( + (x["url"] for x in data["subtitlesAssets"] if x["url"].endswith(".vtt")), + None, + ) + + return manifest, token, subtitle, data + + def decrypt_token(self, token: str, client: str) -> tuple: + if client == "ANDROID": + key = self.config["android"]["key"] + iv = self.config["android"]["iv"] + + if client == "WEB": + key = self.config["web"]["key"] + iv = self.config["web"]["iv"] + + if isinstance(token, str): + token = base64.b64decode(token) + cipher = AES.new( + key=base64.b64decode(key), + iv=base64.b64decode(iv), + mode=AES.MODE_CBC, + ) + data = unpad(cipher.decrypt(token), AES.block_size) + dec_token = data.decode().split("|")[1] + return dec_token.strip() + + def get_html(self, url: str) -> dict: + r = self.session.get(url=url, headers=self.config["headers"]) + r.raise_for_status() + + init_data = re.search( + "", + "".join(r.content.decode().replace("\u200c", "").replace("\r\n", "").replace("undefined", "null")), + ) + try: + data = json.loads(init_data.group(1)) + return data["initialData"] + except Exception: + self.log.error(f"Failed to get episode for {url}") + sys.exit(1) diff --git a/services/ALL4/config.yaml b/services/ALL4/config.yaml new file mode 100644 index 0000000..1468e01 --- /dev/null +++ b/services/ALL4/config.yaml @@ -0,0 +1,27 @@ +headers: + Accept-Language: en-US,en;q=0.8 + User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.75 Safari/537.36 + +endpoints: + login: https://api.channel4.com/online/v2/auth/token + title: https://api.channel4.com/online/v1/views/content-hubs/{title}.json + license: https://c4.eme.lp.aws.redbeemedia.com/wvlicenceproxy-service/widevine/acquire + search: https://all4nav.channel4.com/v1/api/search + +android: + key: QVlESUQ4U0RGQlA0TThESA==" + iv: MURDRDAzODNES0RGU0w4Mg==" + auth: MzZVVUN0OThWTVF2QkFnUTI3QXU4ekdIbDMxTjlMUTE6Sllzd3lIdkdlNjJWbGlrVw== + vod: https://api.channel4.com/online/v1/vod/stream/{video_id}?client=android-mod + +web: + key: bjljTGllWWtxd3pOQ3F2aQ== + iv: b2R6Y1UzV2RVaVhMdWNWZA== + vod: https://www.channel4.com/vod/stream/{programmeId} + +device: + platform_name: android + device_type: mobile + device_name: "Sony C6903 (C6903)" + app_version: "android_app:9.4.2" + optimizely_datafile: "2908" diff --git a/services/AMZN/__init__.py b/services/AMZN/__init__.py new file mode 100644 index 0000000..90272bf --- /dev/null +++ b/services/AMZN/__init__.py @@ -0,0 +1,548 @@ +import base64 +import hashlib +import json +import os +import re +import sys +from collections import defaultdict +from http.cookiejar import CookieJar +from typing import Any, Optional +from urllib.parse import urlparse, urlunparse + +import click +from langcodes import Language +from pywidevine.device import DeviceTypes + +from devine.core.credential import Credential +from devine.core.manifests import DASH +from devine.core.service import Service +from devine.core.titles import Episode, Movie, Movies, Series, Title_T, Titles_T +from devine.core.tracks import Chapter, Chapters, Subtitle, Tracks, Track, Video +from devine.core.utilities import is_close_match +from devine.core.utils.collections import as_list + + +class AMZN(Service): + """ + \b + Service code for Amazon Prime Video (https://primevideo.com). + Based on original code for VT, credit to original author. + + \b + Authorization: Cookies + Robustness: + Widevine: + L1: 2160p + L3 Chrome: 720p, 1080p + L3 Android: 540p + PlayReady: + SL3: 2160p + SL2: 1080p + + \b + Tips: + - Input should be asin only grabbed from URL or other means, e.g.: + B0B8KZPQBX OR amzn1.dv.gti.7aa9f19e-9c00-40e3-98e7-b365678492dd + - Use the --lang LANG_RANGE option to request non-english tracks + - Use --bitrate CBR to request Constant Bitrate: + devine dl -w s01e01 AMZN -b CBR B0B8KZPQBX + - Use --quality SD to request SD tracks (default is HD): + devine dl -w s01e01 AMZN -q SD B0B8KZPQBX + + \b + Notes: + - Written specifically for use with ChromeCDM, with Android L3 as fallback. + - Region is chosen automatically based on domain extension found in cookies. + - Loading tracks could take a few seconds if the title has many audio tracks. + """ + + # GEOFENCE = ("",) + ALIASES = ("amazon", "prime") + TITLE_RE = r"^(?:https?://(?:www\.)?(?Pamazon\.(?Pcom|co\.uk|de|co\.jp)|primevideo\.com)(?:/.+)?/)?(?P[A-Z0-9]{10,}|amzn1\.dv\.gti\.[a-f0-9-]+)" # noqa: E501 + + AUDIO_CODEC_MAP = {"AAC": "mp4a", "EC3": "ec-3"} + + @staticmethod + @click.command(name="AMZN", short_help="https://primevideo.com") + @click.argument("title", type=str, required=False) + @click.option( + "-b", + "--bitrate", + default="VBR+CBR", + type=click.Choice(["VBR", "CBR", "VBR+CBR"], case_sensitive=False), + help="Video Bitrate Mode to download in. VBR=Variable Bitrate, CBR=Constant Bitrate.", + ) + # UHD, HD, SD. UHD only returns HEVC, ever, even for <=HD only content + @click.option( + "-q", + "--quality", + default="HD", + type=click.Choice(["SD", "HD", "UHD"], case_sensitive=False), + help="Manifest quality to request.", + ) + @click.option( + "-am", + "--audio-manifest", + default=None, + type=click.Choice(["VBR", "CBR", "H265"], case_sensitive=False), + help="Manifest to use for audio. Defaults to H265 if the video manifest is missing 640k audio.", + ) + @click.option( + "-aq", + "--audio-quality", + default="SD", + type=click.Choice(["SD", "HD", "UHD"], case_sensitive=False), + help="Manifest quality to request for audio. Defaults to the same as --quality.", + ) + @click.pass_context + def cli(ctx, **kwargs): + return AMZN(ctx, **kwargs) + + def __init__(self, ctx, title, bitrate, quality, audio_manifest, audio_quality): + m = self.parse_title(ctx, title) + self.domain = m.get("domain") + self.domain_region = m.get("region") + super().__init__(ctx) + self.bitrate = bitrate + self.quality = quality + self.audio_manifest = audio_manifest + self.audio_quality = audio_quality + + self.vcodec = "H265" if ctx.parent.params.get("vcodec") == Video.Codec.HEVC else "H264" + self.acodec = ctx.parent.params.get("acodec") + + self.cdm = ctx.obj.cdm + self.region = {} + self.endpoints = {} + self.device = {} + + self.pv = self.domain == "primevideo.com" + self.device_token = None + self.device_id = None + self.customer_id = None + self.client_id = "f22dbddb-ef2c-48c5-8876-bed0d47594fd" + + if self.cdm.device_type != DeviceTypes.CHROME: + self.log.info("Setting manifest quality to SD for Android L3 (use -q HD to override)") + self.quality = "SD" + + # Abstracted functions + + def authenticate(self, cookies: Optional[CookieJar] = None, credential: Optional[Credential] = None) -> None: + super().authenticate(cookies, credential) + if not cookies: + raise EnvironmentError("Service requires Cookies for Authentication.") + + self.session.cookies.update(cookies) + self.configure() + + def get_titles(self) -> Titles_T: + res = self.session.get( + url=self.endpoints["detail"], + params={"titleID": self.title, "isElcano": "1", "sections": "Atf"}, + headers={"Accept": "application/json"}, + ).json()["widgets"] + + entity = res["header"]["detail"].get("entityType") + if not entity: + self.log.error(" - Failed to get entity type") + sys.exit(1) + + if entity == "Movie": + metadata = res["header"]["detail"] + return Movies( + [ + Movie( + id_=metadata.get("catalogId"), + year=metadata.get("releaseYear"), + name=metadata.get("title"), + service=self.__class__, + data=metadata, + ) + ] + ) + elif entity == "TV Show": + seasons = [x.get("titleID") for x in res["seasonSelector"]] + + episodes = [] + for season in seasons: + res = self.session.get( + url=self.endpoints["detail"], + params={"titleID": season, "isElcano": "1", "sections": "Btf"}, + headers={"Accept": "application/json"}, + ).json()["widgets"] + + # cards = [x["detail"] for x in as_list(res["titleContent"][0]["cards"])] + cards = [ + {**x["detail"], "sequenceNumber": x["self"]["sequenceNumber"]} + for x in res["episodeList"]["episodes"] + ] + + product_details = res["productDetails"]["detail"] + + episodes.extend( + Episode( + id_=title.get("titleId") or title["catalogId"], + title=product_details.get("parentTitle") or product_details["title"], + year=title.get("releaseYear") or product_details.get("releaseYear"), + season=product_details.get("seasonNumber"), + number=title.get("sequenceNumber"), + name=title.get("title"), + service=self.__class__, + data=title, + ) + for title in cards + if title["entityType"] == "TV Show" + ) + + return Series(episodes) + + def get_tracks(self, title: Title_T) -> Tracks: + manifest = self.get_manifest(title, video_codec=self.vcodec, bitrate_mode=self.bitrate, quality=self.quality) + + if "rightsException" in manifest["returnedTitleRendition"]["selectedEntitlement"]: + self.log.error(" - The profile used does not have the rights to this title.") + sys.exit(1) + + self.customer_id = manifest["returnedTitleRendition"]["selectedEntitlement"]["grantedByCustomerId"] + + chosen_manifest = self.choose_manifest(manifest) + mpd_url = self.clean_mpd_url(chosen_manifest["avUrlInfoList"][0]["url"]) + + tracks = DASH.from_url(url=mpd_url, session=self.session).to_tracks( + language=re.sub(r"_dialog.*$", "", manifest["playbackUrls"]["defaultAudioTrackId"]) + ) + tracks.videos[0].data["timecodes"] = manifest.get("transitionTimecodes") + + audios = defaultdict(list) + for audio in tracks.audio: + audios[audio.language].append(audio) + + need_separate_audio = False + for lang in audios: + if not any((x.bitrate or 0) >= 640000 for x in audios[lang]): + need_separate_audio = True + break + + if need_separate_audio: + manifest_type = self.audio_manifest or "H265" + self.log.info(f"Getting audio from {manifest_type} manifest for potential higher bitrate or better codec") + audio_manifest = self.get_manifest( + title, + "H265" if manifest_type == "H265" else "H264", + "VBR" if manifest_type != "CBR" else "CBR", + self.audio_quality or self.quality, + ) + audio_mpd_url = self.clean_mpd_url(self.choose_manifest(audio_manifest)["avUrlInfoList"][0]["url"]) + self.log.debug(audio_mpd_url) + + try: + audio_mpd = DASH.from_url(url=audio_mpd_url, session=self.session).to_tracks(language="en") + except KeyError: + self.log.warning(f" - Title has no {self.audio_manifest} stream, cannot get higher quality audio") + else: + tracks.audio = audio_mpd.audio + + for audio in tracks.audio: + # Amazon @lang is just the lang code, no dialect, @audioTrackId has it. + audio_track_id = audio.data["dash"]["adaptation_set"].get("audioTrackId") + sub_type = audio.data["dash"]["adaptation_set"].get("audioTrackSubtype") + if audio_track_id is not None: + audio.language = Language.get(audio_track_id.split("_")[0]) # e.g. es-419_ec3_blabla + if sub_type is not None and "descriptive" in sub_type.lower(): + audio.descriptive = True + + for track in tracks: + rep_base = track.data["dash"]["representation"].find("BaseURL") + if rep_base is not None: + base_url = os.path.dirname(track.url) + track_base = rep_base.text + track.url = f"{base_url}/{track_base}" + track.descriptor = Track.Descriptor.URL + track.data["dash"].clear() + + # filter out boosted, descriptive, and lowest bitrate audio tracks + # tracks.audio = [ + # audio + # for audio in tracks.audio + # if audio.data["dash"]["adaptation_set"].get("audioTrackSubtype", "").lower() == "dialog" + # and int(audio.data["dash"]["adaptation_set"].get("maxBandwidth", 0)) >= 192000 + # ] + + for sub in manifest.get("subtitleUrls", []) + manifest.get("forcedNarratives", []): + tracks.add( + Subtitle( + id_=sub.get( + "timedTextTrackId", f"{sub['languageCode']}_{sub['type']}_{sub['subtype']}_{sub['index']}" + ), + url=os.path.splitext(sub["url"])[0] + ".srt", # DFXP -> SRT forcefully seems to work fine + codec=Subtitle.Codec.from_codecs("srt"), # sub["format"].lower(), + language=sub["languageCode"], + forced="forced" in sub["displayName"], + sdh=sub["type"].lower() == "sdh", # TODO: what other sub types? cc? forced? + ), + warn_only=True, + ) # expecting possible dupes, ignore + + return tracks + + def get_chapters(self, title: Title_T) -> Chapters: + timecodes = title.tracks.videos[0].data.get("timecodes") + if not timecodes: + return Chapters() + + elements = [x for x in timecodes.get("skipElements", [])] + + chapters = [ + Chapter( + name=x.get("elementType", "Chapter"), + timestamp=x.get("startTimecodeMs"), + ) + for x in elements + ] + + if timecodes.get("endCreditsStart"): + chapters.append( + Chapter( + name="CREDITS", + timestamp=timecodes.get("endCreditsStart"), + ) + ) + + return chapters + + def get_widevine_service_certificate(self, **_: Any) -> str: + return self.config["certificate"] + + def get_widevine_license(self, *, challenge: bytes, title: Title_T, track) -> None: + response = self.session.post( + url=self.endpoints["license"], + params={ + "asin": title.id, + "consumptionType": "Streaming", + "desiredResources": "Widevine2License", + "deviceTypeID": self.device["device_type"], + "deviceID": self.device_id, + "firmware": 1, + "gascEnabled": str(self.pv).lower(), + "marketplaceID": self.region["marketplace_id"], + "resourceUsage": "ImmediateConsumption", + "videoMaterialType": "Feature", + "operatingSystemName": "Linux" if self.quality == "SD" else "Windows", + "operatingSystemVersion": "unknown" if self.quality == "SD" else "10.0", + "customerID": self.customer_id, + "deviceDrmOverride": "CENC", + "deviceStreamingTechnologyOverride": "DASH", + "deviceVideoQualityOverride": "HD", + "deviceHdrFormatsOverride": "None", + }, + headers={ + "Accept": "application/json", + "Content-Type": "application/x-www-form-urlencoded", + "Authorization": f"Bearer {self.device_token}", + }, + data={ + "widevine2Challenge": base64.b64encode(challenge).decode(), + "includeHdcpTestKeyInLicense": "false", + }, + ).json() + if "errorsByResource" in response: + error_code = response["errorsByResource"]["Widevine2License"] + if "errorCode" in error_code: + error_code = error_code["errorCode"] + elif "type" in error_code: + error_code = error_code["type"] + + if error_code in ["PRS.NoRights.AnonymizerIP", "PRS.NoRights.NotOwned"]: + self.log.error("Proxy detected, Unable to License") + elif error_code == "PRS.Dependency.DRM.Widevine.UnsupportedCdmVersion": + self.log.error("Cdm version not supported") + else: + self.log.error(f" x Error from Amazon's License Server: [{error_code}]") + sys.exit(1) + + return response["widevine2License"]["license"] + + # Service specific functions + + def configure(self): + if len(self.title) > 10 and not (self.domain or "").startswith("amazon."): + self.pv = True + + self.log.info("Getting account region") + self.region = self.get_region() + if not self.region: + self.log.error(" - Failed to get Amazon account region") + sys.exit(1) + # self.GEOFENCE.append(self.region["code"]) + self.log.info(f" + Region: {self.region['code'].upper()}") + + # endpoints must be prepared AFTER region data is retrieved + self.endpoints = self.prepare_endpoints(self.config["endpoints"], self.region) + + self.session.headers.update({"Origin": f"https://{self.region['base']}"}) + + self.device_id = hashlib.sha224(("CustomerID" + self.session.headers["User-Agent"]).encode("utf-8")).hexdigest() + self.device = {"device_type": self.config["device_types"]["browser"]} + + def get_region(self): + domain_region = self.get_domain_region() + if not domain_region: + return {} + + region = self.config["regions"].get(domain_region) + if not region: + raise self.log.exit(f" - There's no region configuration data for the region: {domain_region}") + + region["code"] = domain_region + + if self.pv: + res = self.session.get("https://www.primevideo.com").text + match = re.search(r'ue_furl *= *([\'"])fls-(na|eu|fe)\.amazon\.[a-z.]+\1', res) + if match: + pv_region = match.group(2).lower() + else: + raise self.log.exit(" - Failed to get PrimeVideo region") + pv_region = {"na": "atv-ps"}.get(pv_region, f"atv-ps-{pv_region}") + region["base_manifest"] = f"{pv_region}.primevideo.com" + region["base"] = "www.primevideo.com" + + return region + + def get_domain_region(self): + """Get the region of the cookies from the domain.""" + tld = (self.domain_region or "").split(".")[-1] + if not tld: + domains = [x.domain for x in self.session.cookies if x.domain_specified] + tld = next((x.split(".")[-1] for x in domains if x.startswith((".amazon.", ".primevideo."))), None) + return {"com": "us", "uk": "gb"}.get(tld, tld) + + def prepare_endpoint(self, name, uri, region): + if name in ("browse", "playback", "license", "xray"): + return f"https://{(region['base_manifest'])}{uri}" + if name in ("detail", "ontv", "devicelink"): + return f"https://{region['base']}{uri}" + if name in ("codepair", "register", "token"): + return f"https://{self.config['regions']['us']['base_api']}{uri}" + raise ValueError(f"Unknown endpoint: {name}") + + def prepare_endpoints(self, endpoints, region): + return {k: self.prepare_endpoint(k, v, region) for k, v in endpoints.items()} + + def choose_manifest(self, manifest): + """Get manifest URL for the title based on CDN weight (or specified CDN).""" + manifest = sorted(manifest["audioVideoUrls"]["avCdnUrlSets"], key=lambda x: int(x["cdnWeightsRank"]))[0] + return manifest + + def get_manifest(self, title, video_codec, bitrate_mode, quality, hdr=None): + r = self.session.get( + url=self.endpoints["playback"], + params={ + "asin": title.id, + "consumptionType": "Streaming", + "desiredResources": ",".join( + [ + "PlaybackUrls", + "AudioVideoUrls", + "CatalogMetadata", + "ForcedNarratives", + "SubtitlePresets", + "SubtitleUrls", + "TransitionTimecodes", + "TrickplayUrls", + "CuepointPlaylist", + "XRayMetadata", + "PlaybackSettings", + ] + ), + "deviceID": self.device_id, + "deviceTypeID": self.device["device_type"], + "firmware": 1, + "gascEnabled": str(self.pv).lower(), + "marketplaceID": self.region["marketplace_id"], + "resourceUsage": "CacheResources", + "videoMaterialType": "Feature", + "playerType": "html5", + "clientId": self.client_id, + "operatingSystemName": "Linux" if quality == "SD" else "Windows", + "operatingSystemVersion": "unknown" if quality == "SD" else "10.0", + "deviceDrmOverride": "CENC", + "deviceStreamingTechnologyOverride": "DASH", + "deviceProtocolOverride": "Https", + "deviceVideoCodecOverride": video_codec, + "deviceBitrateAdaptationsOverride": bitrate_mode.replace("VBR", "CVBR").replace("+", ","), + "deviceVideoQualityOverride": "HD", + "deviceHdrFormatsOverride": "None", + "supportedDRMKeyScheme": "DUAL_KEY", + "liveManifestType": "live,accumulating", + "titleDecorationScheme": "primary-content", + "subtitleFormat": "TTMLv2", + "languageFeature": "MLFv2", + "uxLocale": "en_US", + "xrayDeviceClass": "normal", + "xrayPlaybackMode": "playback", + "xrayToken": "XRAY_WEB_2020_V1", + "playbackSettingsFormatVersion": "1.0.0", + "playerAttributes": json.dumps({"frameRate": "HFR"}), + "audioTrackId": "all", + }, + headers={"Authorization": f"Bearer {self.device_token}"}, + ) + try: + manifest = r.json() + except json.JSONDecodeError: + self.log.debug(r.text) + self.log.error(" - Amazon didn't return JSON data when obtaining the playback manifest.") + sys.exit(1) + if "error" in manifest: + self.log.error(" - Amazon reported an error when obtaining the playback manifest.") + self.log.error(manifest["error"]) + sys.exit(1) + return manifest + + @staticmethod + def get_original_language(manifest): + """Get a title's original language from manifest data.""" + try: + return next( + x["language"].replace("_", "-") + for x in manifest["catalogMetadata"]["playback"]["audioTracks"] + if x["isOriginalLanguage"] + ) + except (KeyError, StopIteration): + pass + + if "defaultAudioTrackId" in manifest.get("playbackUrls", {}): + try: + return manifest["playbackUrls"]["defaultAudioTrackId"].split("_")[0] + except IndexError: + pass + + return None + + @staticmethod + def clean_mpd_url(mpd_url): + """Clean up an Amazon MPD manifest url.""" + try: + parsed_url = urlparse(mpd_url) + new_path = "/".join( + segment for segment in parsed_url.path.split("/") if not any(sub in segment for sub in ["$", "dm"]) + ) + return urlunparse(parsed_url._replace(path=new_path)) + except Exception as e: + raise ValueError(f"Unable to parse MPD URL: {e}") + + def parse_title(self, ctx, title): + title = title or ctx.parent.params.get("title") + if not title: + self.log.exit(" - No title ID specified") + if not getattr(self, "TITLE_RE"): + self.title = title + return {} + for regex in as_list(self.TITLE_RE): + m = re.search(regex, title) + if m: + self.title = m.group("id") + return m.groupdict() + self.log.warning(f" - Unable to parse title ID {title!r}, using as-is") + self.title = title diff --git a/services/AMZN/config.yaml b/services/AMZN/config.yaml new file mode 100644 index 0000000..7beab9e --- /dev/null +++ b/services/AMZN/config.yaml @@ -0,0 +1,62 @@ +certificate: | + CAUSwgUKvAIIAxIQCuQRtZRasVgFt7DIvVtVHBi17OSpBSKOAjCCAQoCggEBAKU2UrYVOSDlcXajWhpEgGhqGraJtFdUPgu6plJGy9ViaRn5mhyXON5PXm + w1krQdi0SLxf00FfIgnYFLpDfvNeItGn9rcx0RNPwP39PW7aW0Fbqi6VCaKWlR24kRpd7NQ4woyMXr7xlBWPwPNxK4xmR/6UuvKyYWEkroyeIjWHAqgCjC + mpfIpVcPsyrnMuPFGl82MMVnAhTweTKnEPOqJpxQ1bdQvVNCvkba5gjOTbEnJ7aXegwhmCdRQzXjTeEV2dO8oo5YfxW6pRBovzF6wYBMQYpSCJIA24ptAP + /2TkneyJuqm4hJNFvtF8fsBgTQQ4TIhnX4bZ9imuhivYLa6HsCAwEAAToPYW1hem9uLmNvbS1wcm9kEoADETQD6R0H/h9fyg0Hw7mj0M7T4s0bcBf4fMhA + Rpwk2X4HpvB49bJ5Yvc4t41mAnXGe/wiXbzsddKMiMffkSE1QWK1CFPBgziU23y1PjQToGiIv/sJIFRKRJ4qMBxIl95xlvSEzKdt68n7wqGa442+uAgk7C + XU3uTfVofYY76CrPBnEKQfad/CVqTh48geNTb4qRH1TX30NzCsB9NWlcdvg10pCnWSm8cSHu1d9yH+2yQgsGe52QoHHCqHNzG/wAxMYWTevXQW7EPTBeFy + SPY0xUN+2F2FhCf5/A7uFUHywd0zNTswh0QJc93LBTh46clRLO+d4RKBiBSj3rah6Y5iXMw9N9o58tCRc9gFHrjfMNubopWHjDOO3ATUgqXrTp+fKVCmsG + uGl1ComHxXV9i1AqHwzzY2JY2vFqo73jR3IElr6oChPIwcNokmNc0D4TXtjE0BoYkbWKJfHvJJihzMOvDicWUsemVHvua9/FBtpbHgpbgwijFPjtQF9Ldb + 8Swf + +device_types: + browser: 'AOAGZA014O5RE' + +endpoints: + detail: /gp/video/api/getDetailPage + browse: /cdp/catalog/Browse + playback: /cdp/catalog/GetPlaybackResources + license: /cdp/catalog/GetPlaybackResources + xray: /swift/page/xray + ontv: /gp/video/ontv/code + devicelink: /gp/video/api/codeBasedLinking + codepair: /auth/create/codepair + register: /auth/register + token: /auth/token + +regions: + us: + base: 'www.amazon.com' + base_api: 'api.amazon.com' + base_manifest: 'atv-ps.amazon.com' + marketplace_id: 'ATVPDKIKX0DER' + + gb: + base: 'www.amazon.co.uk' + base_api: 'api.amazon.co.uk' + base_manifest: 'atv-ps-eu.amazon.co.uk' + marketplace_id: 'A2IR4J4NTCP2M5' # A1F83G8C2ARO7P is also another marketplace_id + + it: + base: 'www.amazon.it' + base_api: 'api.amazon.it' + base_manifest: 'atv-ps-eu.primevideo.com' + marketplace_id: 'A3K6Y4MI8GDYMT' + + de: + base: 'www.amazon.de' + base_api: 'api.amazon.de' + base_manifest: 'atv-ps-eu.amazon.de' + marketplace_id: 'A1PA6795UKMFR9' + + au: + base: 'www.amazon.com.au' + base_api: 'api.amazon.com.au' + base_manifest: 'atv-ps-fe.amazon.com.au' + marketplace_id: 'A3K6Y4MI8GDYMT' + + jp: + base: 'www.amazon.co.jp' + base_api: 'api.amazon.co.jp' + base_manifest: 'atv-ps-fe.amazon.co.jp' + marketplace_id: 'A1VC38T7YXB528' diff --git a/services/AUBC/__init__.py b/services/AUBC/__init__.py new file mode 100644 index 0000000..bc733b0 --- /dev/null +++ b/services/AUBC/__init__.py @@ -0,0 +1,250 @@ +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Generator +from typing import Any, Optional, Union +from urllib.parse import urljoin + +import click +from click import Context +from devine.core.constants import AnyTrack +from devine.core.manifests.dash import DASH +from devine.core.search_result import SearchResult +from devine.core.service import Service +from devine.core.titles import Episode, Movie, Movies, Series +from devine.core.tracks import Chapter, Chapters, Subtitle, Tracks +from requests import Request + + +class AUBC(Service): + """ + \b + Service code for ABC iView streaming service (https://iview.abc.net.au/). + + \b + Author: stabbedbybrick + Authorization: None + Robustness: + L3: 1080p, AAC2.0 + + \b + Tips: + - Input should be complete URL: + SHOW: https://iview.abc.net.au/show/return-to-paradise + EPISODE: https://iview.abc.net.au/video/DR2314H001S00 + MOVIE: https://iview.abc.net.au/show/way-back / https://iview.abc.net.au/show/way-back/video/ZW3981A001S00 + + """ + + GEOFENCE = ("au",) + ALIASES = ("iview", "abciview", "iv",) + + @staticmethod + @click.command(name="AUBC", short_help="https://iview.abc.net.au/", help=__doc__) + @click.argument("title", type=str) + @click.pass_context + def cli(ctx: Context, **kwargs: Any) -> AUBC: + return AUBC(ctx, **kwargs) + + def __init__(self, ctx: Context, title: str): + self.title = title + super().__init__(ctx) + + self.session.headers.update(self.config["headers"]) + + def search(self) -> Generator[SearchResult, None, None]: + url = ( + "https://y63q32nvdl-1.algolianet.com/1/indexes/*/queries?x-algolia-agent=Algolia" + "%20for%20JavaScript%20(4.9.1)%3B%20Browser%20(lite)%3B%20react%20(17.0.2)%3B%20" + "react-instantsearch%20(6.30.2)%3B%20JS%20Helper%20(3.10.0)&x-" + "algolia-api-key=bcdf11ba901b780dc3c0a3ca677fbefc&x-algolia-application-id=Y63Q32NVDL" + ) + payload = { + "requests": [ + { + "indexName": "ABC_production_iview_web", + "params": f"query={self.title}&tagFilters=&userToken=anonymous-74be3cf1-1dc7-4fa1-9cff-19592162db1c", + } + ], + } + + results = self._request("POST", url, payload=payload)["results"] + hits = [x for x in results[0]["hits"] if x["docType"] == "Program"] + + for result in hits: + yield SearchResult( + id_="https://iview.abc.net.au/show/{}".format(result.get("slug")), + title=result.get("title"), + description=result.get("synopsis"), + label=result.get("subType"), + url="https://iview.abc.net.au/show/{}".format(result.get("slug")), + ) + + def get_titles(self) -> Union[Movies, Series]: + title_re = r"^(?:https?://(?:www.)?iview.abc.net.au/(?Pshow|video)/)?(?P[a-zA-Z0-9_-]+)" + try: + kind, title_id = (re.match(title_re, self.title).group(i) for i in ("type", "id")) + except Exception: + raise ValueError("- Could not parse ID from title") + + if kind == "show": + data = self._request("GET", "/v3/show/{}".format(title_id)) + label = data.get("type") + + if label.lower() in ("series", "program"): + episodes = self._series(title_id) + return Series(episodes) + + elif label.lower() in ("feature", "movie"): + movie = self._movie(data) + return Movies(movie) + + elif kind == "video": + episode = self._episode(title_id) + return Series([episode]) + + def get_tracks(self, title: Union[Movie, Episode]) -> Tracks: + video = self._request("GET", "/v3/video/{}".format(title.id)) + if not video.get("playable"): + raise ConnectionError(video.get("unavailableMessage")) + + playlist = video.get("_embedded", {}).get("playlist", {}) + if not playlist: + raise ConnectionError("Could not find a playlist for this title") + + streams = next(x["streams"]["mpegdash"] for x in playlist if x["type"] == "program") + captions = next((x.get("captions") for x in playlist if x["type"] == "program"), None) + manifest = streams["720"].replace("720", "1080") if streams.get("720") else streams["sd"] + title.data["protected"] = True if streams.get("protected") else False + + tracks = DASH.from_url(manifest, self.session).to_tracks(title.language) + + for track in tracks.audio: + role = track.data["dash"]["adaptation_set"].find("Role") + if role is not None and role.get("value") in ["description", "alternative", "alternate"]: + track.descriptive = True + + if captions: + subtitles = captions.get("src-vtt") + tracks.add( + Subtitle( + id_=hashlib.md5(subtitles.encode()).hexdigest()[0:6], + url=subtitles, + codec=Subtitle.Codec.from_mime(subtitles[-3:]), + language=title.language, + forced=False, + ) + ) + + return tracks + + def get_chapters(self, title: Union[Movie, Episode]) -> Chapters: + if not title.data.get("cuePoints"): + return Chapters() + + credits = next((x.get("start") for x in title.data["cuePoints"] if x["type"] == "end-credits"), None) + if credits: + return Chapters([Chapter(name="Credits", timestamp=credits * 1000)]) + + return Chapters() + + def get_widevine_service_certificate(self, **_: Any) -> str: + return None + + def get_widevine_license(self, *, challenge: bytes, title: Union[Movies, Series], track: AnyTrack) -> Optional[Union[bytes, str]]: + if not title.data.get("protected"): + return None + + customdata = self._license(title.id) + headers = {"customdata": customdata} + + r = self.session.post(self.config["endpoints"]["license"], headers=headers, data=challenge) + r.raise_for_status() + return r.content + + # Service specific + + def _series(self, title: str) -> Episode: + data = self._request("GET", "/v3/series/{}".format(title)) + + episodes = [ + self.create_episode(episode) + for season in data + for episode in reversed(season["_embedded"]["videoEpisodes"]["items"]) + if season.get("episodeCount") + ] + return Series(episodes) + + def _movie(self, data: dict) -> Movie: + return [ + Movie( + id_=data["_embedded"]["highlightVideo"]["id"], + service=self.__class__, + name=data.get("title"), + year=data.get("productionYear"), + data=data, + language=data.get("analytics", {}).get("dataLayer", {}).get("d_language", "en"), + ) + ] + + def _episode(self, video_id: str) -> Episode: + data = self._request("GET", "/v3/video/{}".format(video_id)) + return self.create_episode(data) + + def _license(self, video_id: str): + token = self._request("POST", "/v3/token/jwt", data={"clientId": self.config["client"]})["token"] + response = self._request("GET", "/v3/token/drm/{}".format(video_id), headers={"bearer": token}) + + return response["license"] + + def create_episode(self, episode: dict) -> Episode: + title = episode["showTitle"] + season = re.search(r"Series (\d+)", episode.get("title")) + number = re.search(r"Episode (\d+)", episode.get("title")) + names_a = re.search(r"Series \d+ Episode \d+ (.+)", episode.get("title")) + names_b = re.search(r"Series \d+ (.+)", episode.get("title")) + name = names_a.group(1) if names_a else names_b.group(1) if names_b else episode.get("displaySubtitle") + + language = episode.get("analytics", {}).get("dataLayer", {}).get("d_language", "en") + + return Episode( + id_=episode["id"], + service=self.__class__, + title=title, + season=int(season.group(1)) if season else 0, + number=int(number.group(1)) if number else 0, + name=name, + data=episode, + language=language, + ) + + def _request( + self, + method: str, + api: str, + params: dict = None, + headers: dict = None, + data: dict = None, + payload: dict = None, + ) -> Any[dict | str]: + url = urljoin(self.config["endpoints"]["base_url"], api) + + if params: + self.session.params.update(params) + if headers: + self.session.headers.update(headers) + + prep = self.session.prepare_request(Request(method, url, data=data, json=payload)) + + response = self.session.send(prep) + if response.status_code != 200: + raise ConnectionError(f"{response.text}") + + try: + return json.loads(response.content) + + except json.JSONDecodeError as e: + raise ValueError(f"Failed to parse JSON: {response.text}") from e + diff --git a/services/AUBC/config.yaml b/services/AUBC/config.yaml new file mode 100644 index 0000000..604b739 --- /dev/null +++ b/services/AUBC/config.yaml @@ -0,0 +1,9 @@ +headers: + User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:126.0) Gecko/20100101 Firefox/126.0 + accept-language: en-US,en;q=0.8 + +endpoints: + base_url: https://api.iview.abc.net.au + license: https://wv-keyos.licensekeyserver.com/ + +client: "1d4b5cba-42d2-403e-80e7-34565cdf772d" \ No newline at end of file diff --git a/services/CBC/__init__.py b/services/CBC/__init__.py new file mode 100644 index 0000000..4ee55ed --- /dev/null +++ b/services/CBC/__init__.py @@ -0,0 +1,318 @@ +from __future__ import annotations + +import json +import re +import sys +from collections.abc import Generator +from http.cookiejar import CookieJar +from typing import Any, Optional, Union +from urllib.parse import urljoin + +import click +from click import Context +from devine.core.constants import AnyTrack +from devine.core.credential import Credential +from devine.core.manifests import DASH, HLS +from devine.core.search_result import SearchResult +from devine.core.service import Service +from devine.core.titles import Episode, Movie, Movies, Series +from devine.core.tracks import Chapter, Chapters, Tracks +from requests import Request + + +class CBC(Service): + """ + \b + Service code for CBC Gem streaming service (https://gem.cbc.ca/). + + \b + Author: stabbedbybrick + Authorization: Credentials + Robustness: + AES-128: 1080p, DDP5.1 + Widevine L3: 720p, DDP5.1 + + \b + Tips: + - Input can be complete title URL or just the slug: + SHOW: https://gem.cbc.ca/murdoch-mysteries OR murdoch-mysteries + MOVIE: https://gem.cbc.ca/the-babadook OR the-babadook + + \b + Notes: + - DRM encrypted titles max out at 720p. + - CCExtrator v0.94 will likely fail to extract subtitles. It's recommended to downgrade to v0.93. + - Some audio tracks contain invalid data, causing warning messages from mkvmerge during muxing + These can be ignored. + + """ + + GEOFENCE = ("ca",) + ALIASES = ("gem", "cbcgem",) + + @staticmethod + @click.command(name="CBC", short_help="https://gem.cbc.ca/", help=__doc__) + @click.argument("title", type=str) + @click.pass_context + def cli(ctx: Context, **kwargs: Any) -> CBC: + return CBC(ctx, **kwargs) + + def __init__(self, ctx: Context, title: str): + self.title: str = title + super().__init__(ctx) + + self.base_url: str = self.config["endpoints"]["base_url"] + + def search(self) -> Generator[SearchResult, None, None]: + params = { + "device": "web", + "pageNumber": "1", + "pageSize": "20", + "term": self.title, + } + response: dict = self._request("GET", "/ott/catalog/v1/gem/search", params=params) + + for result in response.get("result", []): + yield SearchResult( + id_="https://gem.cbc.ca/{}".format(result.get("url")), + title=result.get("title"), + description=result.get("synopsis"), + label=result.get("type"), + url="https://gem.cbc.ca/{}".format(result.get("url")), + ) + + def authenticate(self, cookies: Optional[CookieJar] = None, credential: Optional[Credential] = None) -> None: + super().authenticate(cookies, credential) + if not credential: + raise EnvironmentError("Service requires Credentials for Authentication.") + + tokens: Optional[Any] = self.cache.get(f"tokens_{credential.sha1}") + + """ + All grant types for future reference: + PASSWORD("password"), + ACCESS_TOKEN("access_token"), + REFRESH_TOKEN("refresh_token"), + CLIENT_CREDENTIALS("client_credentials"), + AUTHORIZATION_CODE("authorization_code"), + CODE("code"); + """ + + if tokens and not tokens.expired: + # cached + self.log.info(" + Using cached tokens") + auth_token: str = tokens.data["access_token"] + + elif tokens and tokens.expired: + # expired, refresh + self.log.info("Refreshing cached tokens...") + auth_url, scopes = self.settings() + params = { + "client_id": self.config["client"]["id"], + "grant_type": "refresh_token", + "refresh_token": tokens.data["refresh_token"], + "scope": scopes, + } + + access: dict = self._request("POST", auth_url, params=params) + + # Shorten expiration by one hour to account for clock skew + tokens.set(access, expiration=int(access["expires_in"]) - 3600) + auth_token: str = access["access_token"] + + else: + # new + self.log.info("Requesting new tokens...") + auth_url, scopes = self.settings() + params = { + "client_id": self.config["client"]["id"], + "grant_type": "password", + "username": credential.username, + "password": credential.password, + "scope": scopes, + } + + access: dict = self._request("POST", auth_url, params=params) + + # Shorten expiration by one hour to account for clock skew + tokens.set(access, expiration=int(access["expires_in"]) - 3600) + auth_token: str = access["access_token"] + + claims_token: str = self.claims_token(auth_token) + self.session.headers.update({"x-claims-token": claims_token}) + + def get_titles(self) -> Union[Movies, Series]: + title_re: str = r"^(?:https?://(?:www.)?gem.cbc.ca/)?(?P[a-zA-Z0-9_-]+)" + try: + title_id: str = re.match(title_re, self.title).group("id") + except Exception: + raise ValueError("- Could not parse ID from title") + + params = {"device": "web"} + data: dict = self._request("GET", "/ott/catalog/v2/gem/show/{}".format(title_id), params=params) + label: str = data.get("contentType", "").lower() + + if label in ("film", "movie", "standalone"): + movies: list[Movie] = self._movie(data) + return Movies(movies) + + else: + episodes: list[Episode] = self._show(data) + return Series(episodes) + + def get_tracks(self, title: Union[Movie, Episode]) -> Tracks: + index: dict = self._request( + "GET", "/media/meta/v1/index.ashx", params={"appCode": "gem", "idMedia": title.id, "output": "jsonObject"} + ) + + title.data["extra"] = { + "chapters": index["Metas"].get("Chapitres"), + "credits": index["Metas"].get("CreditStartTime"), + } + + self.drm: bool = index["Metas"].get("isDrmActive") == "true" + if self.drm: + tech: str = next(tech["name"] for tech in index["availableTechs"] if "widevine" in tech["drm"]) + else: + tech: str = next(tech["name"] for tech in index["availableTechs"] if not tech["drm"]) + + response: dict = self._request( + "GET", self.config["endpoints"]["validation"].format("android", title.id, "smart-tv", tech) + ) + + manifest = response.get("url") + self.license = next((x["value"] for x in response["params"] if "widevineLicenseUrl" in x["name"]), None) + self.token = next((x["value"] for x in response["params"] if "widevineAuthToken" in x["name"]), None) + + stream_type: Union[HLS, DASH] = HLS if tech == "hls" else DASH + tracks: Tracks = stream_type.from_url(manifest, self.session).to_tracks(language=title.language) + + if stream_type == DASH: + for track in tracks.audio: + label = track.data["dash"]["adaptation_set"].find("Label") + if label is not None and "descriptive" in label.text.lower(): + track.descriptive = True + + for track in tracks: + track.language = title.language + + return tracks + + def get_chapters(self, title: Union[Movie, Episode]) -> Chapters: + extra: dict = title.data["extra"] + + chapters = [] + if extra.get("chapters"): + chapters = [Chapter(timestamp=x) for x in set(extra["chapters"].split(","))] + + if extra.get("credits"): + chapters.append(Chapter(name="Credits", timestamp=float(extra["credits"]))) + + return Chapters(chapters) + + def get_widevine_service_certificate(self, **_: Any) -> str: + return None + + def get_widevine_license( + self, *, challenge: bytes, title: Union[Movies, Series], track: AnyTrack + ) -> Optional[Union[bytes, str]]: + if not self.license or not self.token: + return None + + headers = {"x-dt-auth-token": self.token} + r = self.session.post(self.license, headers=headers, data=challenge) + r.raise_for_status() + return r.content + + # Service specific + + def _show(self, data: dict) -> list[Episode]: + lineups: list = next((x["lineups"] for x in data["content"] if x.get("title", "").lower() == "episodes"), None) + if not lineups: + self.log.warning("No episodes found for: {}".format(data.get("title"))) + return + + titles = [] + for season in lineups: + for episode in season["items"]: + if episode.get("mediaType", "").lower() == "episode": + parts = episode.get("title", "").split(".", 1) + episode_name = parts[1].strip() if len(parts) > 1 else parts[0].strip() + titles.append( + Episode( + id_=episode["idMedia"], + service=self.__class__, + title=data.get("title"), + season=int(season.get("seasonNumber", 0)), + number=int(episode.get("episodeNumber", 0)), + name=episode_name, + year=episode.get("metadata", {}).get("productionYear"), + language=data["structuredMetadata"].get("inLanguage", "en-CA"), + data=episode, + ) + ) + + return titles + + def _movie(self, data: dict) -> list[Movie]: + unwanted: tuple = ("episodes", "trailers", "extras") + lineups: list = next((x["lineups"] for x in data["content"] if x.get("title", "").lower() not in unwanted), None) + if not lineups: + self.log.warning("No movies found for: {}".format(data.get("title"))) + return + + titles = [] + for season in lineups: + for movie in season["items"]: + if movie.get("mediaType", "").lower() == "episode": + parts = movie.get("title", "").split(".", 1) + movie_name = parts[1].strip() if len(parts) > 1 else parts[0].strip() + titles.append( + Movie( + id_=movie.get("idMedia"), + service=self.__class__, + name=movie_name, + year=movie.get("metadata", {}).get("productionYear"), + language=data["structuredMetadata"].get("inLanguage", "en-CA"), + data=movie, + ) + ) + + return titles + + def settings(self) -> tuple: + settings = self._request("GET", "/ott/catalog/v1/gem/settings", params={"device": "web"}) + auth_url: str = settings["identityManagement"]["ropc"]["url"] + scopes: str = settings["identityManagement"]["ropc"]["scopes"] + return auth_url, scopes + + def claims_token(self, token: str) -> str: + headers = { + "Authorization": "Bearer " + token, + } + params = {"device": "web"} + response: dict = self._request( + "GET", "/ott/subscription/v2/gem/Subscriber/profile", headers=headers, params=params + ) + return response["claimsToken"] + + def _request(self, method: str, api: str, **kwargs: Any) -> Any[dict | str]: + url: str = urljoin(self.base_url, api) + + prep: Request = self.session.prepare_request(Request(method, url, **kwargs)) + response = self.session.send(prep) + if response.status_code not in (200, 426): + raise ConnectionError(f"{response.status_code} - {response.text}") + + try: + data = json.loads(response.content) + error_keys = ["errorMessage", "ErrorMessage", "ErrorCode", "errorCode", "error"] + error_message = next((data.get(key) for key in error_keys if key in data), None) + if error_message: + self.log.error(f"\n - Error: {error_message}\n") + sys.exit(1) + + return data + + except json.JSONDecodeError: + raise ConnectionError("Request for {} failed: {}".format(response.url, response.text)) diff --git a/services/CBC/config.yaml b/services/CBC/config.yaml new file mode 100644 index 0000000..31bd19a --- /dev/null +++ b/services/CBC/config.yaml @@ -0,0 +1,7 @@ +endpoints: + base_url: "https://services.radio-canada.ca" + validation: "/media/validation/v2?appCode=gem&&deviceType={}&idMedia={}&manifestType={}&output=json&tech={}" + api_key: "3f4beddd-2061-49b0-ae80-6f1f2ed65b37" + +client: + id: "fc05b0ee-3865-4400-a3cc-3da82c330c23" \ No newline at end of file diff --git a/services/CBS/__init__.py b/services/CBS/__init__.py new file mode 100644 index 0000000..e35e4d8 --- /dev/null +++ b/services/CBS/__init__.py @@ -0,0 +1,241 @@ +from __future__ import annotations + +import json +import re +import sys +from collections.abc import Generator +from typing import Any, Optional, Union +from urllib.parse import urljoin + +import click +from devine.core.constants import AnyTrack +from devine.core.manifests import DASH +from devine.core.search_result import SearchResult +from devine.core.service import Service +from devine.core.titles import Episode, Series, Title_T, Titles_T +from devine.core.tracks import Chapter, Chapters, Tracks +from devine.core.utils.sslciphers import SSLCiphers +from devine.core.utils.xml import load_xml +from requests import Request + + +class CBS(Service): + """ + \b + Service code for CBS.com streaming service (https://cbs.com). + Credit to @srpen6 for the tip on anonymous session + + \b + Author: stabbedbybrick + Authorization: None + Robustness: + Widevine: + L3: 2160p, DDP5.1 + + \b + Tips: + - Input should be complete URLs: + SERIES: https://www.cbs.com/shows/tracker/ + EPISODE: https://www.cbs.com/shows/video/E0wG_ovVMkLlHOzv7KDpUV9bjeKFFG2v/ + + \b + Common VPN/proxy errors: + - SSLError(SSLEOFError(8, '[SSL: UNEXPECTED_EOF_WHILE_READING]')) + - ConnectionError: 406 Not Acceptable, 403 Forbidden + + """ + + GEOFENCE = ("us",) + + @staticmethod + @click.command(name="CBS", short_help="https://cbs.com", help=__doc__) + @click.argument("title", type=str, required=False) + @click.pass_context + def cli(ctx, **kwargs) -> CBS: + return CBS(ctx, **kwargs) + + def __init__(self, ctx, title): + self.title = title + super().__init__(ctx) + + def search(self) -> Generator[SearchResult, None, None]: + params = { + "term": self.title, + "termCount": 50, + "showCanVids": "true", + } + results = self._request("GET", "/apps-api/v3.1/androidphone/contentsearch/search.json", params=params)["terms"] + + for result in results: + yield SearchResult( + id_=result.get("path"), + title=result.get("title"), + description=None, + label=result.get("term_type"), + url=result.get("path"), + ) + + def get_titles(self) -> Titles_T: + title_re = r"https://www\.cbs\.com/shows/(?P