from __future__ import annotations import base64 import json import os import random import re import sys import time from datetime import timedelta from http.cookiejar import CookieJar from typing import Any, Optional, Union from uuid import UUID import click import requests from langcodes import Language from pymp4.parser import Box from pywidevine.cdm import Cdm from pywidevine.device import DeviceTypes from pywidevine.pssh import PSSH from devine.core.credential import Credential from devine.core.service import Service from devine.core.drm import Widevine from devine.core.titles import Episode, Movie, Movies, Series, Title_T from devine.core.tracks import Audio, Chapter, Chapters, Subtitle, Track, Tracks, Video from devine.core.utils.collections import as_list, flatten from .MSL import MSL from .MSL.schemes import KeyExchangeSchemes from .MSL.schemes.UserAuthentication import UserAuthentication class NF(Service): """ Service code for the Netflix streaming service (https://netflix.com). \b Authorization: Cookies Robustness: Widevine: L1: 2160p L3 Chrome: 720p, 1080p L3 Android: 540p PlayReady: SL3: 2160p SL2: 1080p *MPL: FHD with Android L3, sporadically available with ChromeCDM HPL: 1080p with ChromeCDM, 720p/1080p with other L3 (varies per title) \b Tips: - Input can be either just title ID or URL: devine dl -w s01e01 NF 80057281 devine dl -w s01e01 NF https://www.netflix.com/title/80057281 \b Notes: - Android CDM is currently not supported as the MSL Widevine KeyExchange is broken. - The library of contents as well as regional availability is available at https://unogs.com However, Do note that Netflix locked everyone out of being able to automate the available data meaning the reliability and amount of information may be reduced. - You could combine the information from https://unogs.com with https://justwatch.com for further data TODO: Fix Widevine KeyExchange scheme """ ALIASES = ("netflix",) TITLE_RE = [ r"^(?:https?://(?:www\.)?netflix\.com(?:/[a-z0-9]{2})?/(?:title/|watch/|.+jbv=))?(?P\d+)", r"^https?://(?:www\.)?unogs\.com/title/(?P\d+)", ] NF_LANG_MAP = { "es": "es-419", "pt": "pt-PT", } @staticmethod @click.command(name="NF", short_help="https://netflix.com") @click.argument("title", type=str, required=False) @click.option( "-p", "--profile", type=click.Choice(["MPL", "HPL", "MPL+HPL"], case_sensitive=False), default="MPL+HPL", help="H.264 profile to use. Default is best available.", ) @click.option("--meta-lang", type=str, help="Language to use for metadata") @click.pass_context def cli(ctx, **kwargs): return NF(ctx, **kwargs) def __init__(self, ctx, title, profile, meta_lang): super().__init__(ctx) self.parse_title(ctx, title) self.profile = profile self.meta_lang = meta_lang if ctx.parent.params["proxy"] and len("".join(i for i in ctx.parent.params["proxy"] if not i.isdigit())) == 2: self.GEOFENCE.append(ctx.parent.params["proxy"]) vcodec = ctx.parent.params.get("vcodec") self.vcodec = "H265" if vcodec and vcodec == Video.Codec.HEVC else "H264" self.acodec = ctx.parent.params["acodec"] self.range = ctx.parent.params["range_"][0].name self.quality = ctx.parent.params["quality"] self.audio_only = ctx.parent.params["audio_only"] self.subs_only = ctx.parent.params["subs_only"] self.chapters_only = ctx.parent.params["chapters_only"] self.profiles = [] self.cdm = ctx.obj.cdm if self.cdm.device_type == DeviceTypes.ANDROID: self.log.error( " - Android CDMs are currently not supported as the Widevine KeyExchange scheme is broken.") sys.exit(1) self.user_profile = ctx.parent.params.get("profile") if not self.user_profile: self.user_profile = "default" def authenticate( self, cookies: Optional[CookieJar] = None, credential: Optional[Credential] = None, ) -> None: super().authenticate(cookies, credential) if not cookies: raise EnvironmentError("Service requires Cookies for Authentication.") self.session.cookies.update(cookies) self.log.info(f" + User profile: '{self.user_profile}'") self.configure() def get_titles(self) -> Union[Movies, Series]: metadata = self.get_metadata(self.title)["video"] if metadata["type"] == "movie": movie = [ Movie( id_=self.title, name=metadata["title"], year=metadata["year"], service=self.__class__, data=metadata ) ] return Movies(movie) else: episodes = [ episode for season in [ [dict(x, **{"season": season["seq"]}) for x in season["episodes"]] for season in metadata["seasons"] ] for episode in season ] titles = [ Episode( id_=self.title, title=metadata["title"], year=metadata["seasons"][0].get("year"), season=episode.get("season"), number=episode.get("seq"), name=episode.get("title"), service=self.__class__, data=episode, ) for episode in episodes ] return Series(titles) # TODO: Get original language without making an extra manifest request # manifest = self.get_manifest(titles[0], self.profiles) # original_language = self.get_original_language(manifest) # for title in titles: # title.original_lang = original_language def get_tracks(self, title: Union[Movie, Episode]) -> Tracks: if self.vcodec == "H264": # If H.264, get both MPL and HPL tracks as they alternate in terms of bitrate tracks = Tracks() self.config["profiles"]["video"]["H264"]["MPL+HPL+QC"] = ( self.config["profiles"]["video"]["H264"]["MPL"] + self.config["profiles"]["video"]["H264"]["HPL"] + self.config["profiles"]["video"]["H264"]["QC"] ) if self.audio_only or self.subs_only or self.chapters_only: profiles = ["MPL+HPL+QC"] else: profiles = self.profile.split("+") for profile in profiles: try: manifest = self.get_manifest(title, self.config["profiles"]["video"]["H264"][profile]) except Exception: manifest = self.get_manifest( title, self.config["profiles"]["video"]["H264"]["MPL"] + self.config["profiles"]["video"]["H264"]["HPL"] ) manifest_tracks = self.manifest_as_tracks(manifest) license_url = manifest["links"]["license"]["href"] if self.cdm.security_level == 3 and self.cdm.device_type == DeviceTypes.ANDROID: max_quality = max(x.height for x in manifest_tracks.videos) if profile == "MPL" and max_quality >= 720: manifest_sd = self.get_manifest(title, self.config["profiles"]["video"]["H264"]["BPL"]) license_url_sd = manifest_sd["links"]["license"]["href"] if "SD_LADDER" in manifest_sd["video_tracks"][0]["streams"][0]["tags"]: # SD manifest is new encode encrypted with different keys that won't work for HD continue license_url = license_url_sd if profile == "HPL" and max_quality >= 1080: if "SEGMENT_MAP_2KEY" in manifest["video_tracks"][0]["streams"][0]["tags"]: # 1080p license restricted from Android L3, 720p license will work for 1080p manifest_720 = self.get_manifest( title, [x for x in self.config["profiles"]["video"]["H264"]["HPL"] if "l40" not in x] ) license_url = manifest_720["links"]["license"]["href"] else: # Older encode, can't use 720p keys for 1080p continue for track in manifest_tracks: if track.drm: track.data["license_url"] = license_url tracks.add(manifest_tracks, warn_only=True) return tracks else: manifest = self.get_manifest(title, self.profiles) manifest_tracks = self.manifest_as_tracks(manifest) license_url = manifest["links"]["license"]["href"] for track in manifest_tracks: if track.drm: track.data["license_url"] = license_url # if isinstance(track, Video): # # TODO: Needs something better than this # track.hdr10 = track.codec.split("-")[1] == "hdr" # hevc-hdr, vp9-hdr # track.dv = track.codec.startswith("hevc-dv") return manifest_tracks def get_chapters(self, title: Union[Movie, Episode]) -> Chapters: def _convert(total_seconds): hours = total_seconds // 3600 minutes = (total_seconds % 3600) // 60 seconds = total_seconds % 60 # milliseconds = (total_seconds % 1) * 1000 return f"{int(hours):02}:{int(minutes):02}:{int(seconds):02}" metadata = self.get_metadata(title.id)["video"] if metadata["type"] == "movie": episode = metadata else: season = next(x for x in metadata["seasons"] if x["seq"] == title.season) episode = next(x for x in season["episodes"] if x["seq"] == title.number) if not (episode.get("skipMarkers") and episode.get("creditsOffset")): return [] chapters = {} for item in episode["skipMarkers"]: chapters[item] = {"start": 0, "end": 0} if not episode["skipMarkers"][item]: continue if episode["skipMarkers"][item]["start"] is None: chapters[item]["start"] = 0 else: chapters[item]["start"] = episode["skipMarkers"][item]["start"] / 1000 if episode["skipMarkers"][item]["end"] is None: chapters[item]["end"] = 0 else: chapters[item]["end"] = episode["skipMarkers"][item]["end"] / 1000 cc, intro = 1, 0 chaps = [Chapter(timestamp="00:00:00.000")] for item in chapters: if chapters[item]["start"] != 0: if intro == 0: cc += 1 chaps.append(Chapter(name="Intro", timestamp=_convert(chapters[item]["start"]))) cc += 1 chaps.append(Chapter(timestamp=_convert(chapters[item]["end"]))) else: cc += 1 chaps.append(Chapter(timestamp=_convert(chapters[item]["start"]))) cc += 1 chaps.append(Chapter(timestamp=_convert(chapters[item]["end"]))) cc += 1 if cc == 1: chaps.append(Chapter(name="Credits", timestamp=_convert(episode["creditsOffset"]))) else: chaps.append(Chapter(name="Credits", timestamp=_convert(episode["creditsOffset"]))) return chaps def get_widevine_service_certificate(self, challenge: bytes, **_: Any) -> str: return self.config["certificate"] def get_widevine_license(self, *, challenge: bytes, session_id: bytes, title: Title_T, track) -> None: if not self.msl: self.log.error(" - Cannot get license, MSL client has not been created yet.") sys.exit(1) header, payload_data = self.msl.send_message( endpoint=self.config["endpoints"]["licence"], params={}, application_data={ "version": 2, "url": track.data["license_url"], "id": int(time.time() * 10000), "esn": self.esn, "languages": ["en-US"], "uiVersion": self.react_context["serverDefs"]["data"]["uiVersion"], "clientVersion": "6.0026.291.011", "params": [ { "sessionId": base64.b64encode(session_id).decode("utf-8"), "clientTime": int(time.time()), "challengeBase64": base64.b64encode(challenge).decode("utf-8"), "xid": str(int((int(time.time()) + 0.1612) * 1000)), } ], "echo": "sessionId", }, userauthdata=self.userauthdata, ) if not payload_data: self.log.error(f" - Failed to get license: {header['message']} [{header['code']}]") sys.exit(1) if "error" in payload_data[0]: error = payload_data[0]["error"] error_display = error.get("display") error_detail = re.sub(r" \(E3-[^)]+\)", "", error.get("detail", "")) if error_display: self.log.critical(f" - {error_display}") if error_detail: self.log.critical(f" - {error_detail}") if not (error_display or error_detail): self.log.critical(f" - {error}") sys.exit(1) return payload_data[0]["licenseResponseBase64"] # Service specific functions def configure(self): self.session.headers.update({"Origin": "https://netflix.com"}) self.profiles = self.get_profiles() self.esn = None self.msl = None self.userauthdata = None self.log.info("Initializing a Netflix MSL client") if self.cdm.device_type == DeviceTypes.CHROME: self.esn = self.chrome_esn_generator() else: esn_map = self.config.get("esn_map", {}) self.esn = esn_map.get(self.cdm.system_id) or esn_map.get(str(self.cdm.system_id)) if not self.esn: self.log.error(" - No ESN specified") sys.exit(1) self.log.info(f" + ESN: {self.esn}") scheme = { DeviceTypes.CHROME: KeyExchangeSchemes.AsymmetricWrapped, DeviceTypes.ANDROID: KeyExchangeSchemes.Widevine, }[self.cdm.device_type] self.log.info(f" + Scheme: {scheme}") self.msl = MSL.handshake( scheme=scheme, session=self.session, endpoint=self.config["endpoints"]["manifest"], sender=self.esn, cdm=self.cdm, msl_keys_path=self.cache.get( "msl_{id}_{esn}_{scheme}_{profile}".format( id=self.cdm.system_id, esn=self.esn, scheme=scheme, profile=self.user_profile ) ), ) if not self.session.cookies: self.log.error(" - No cookies provided, cannot log in.") sys.exit(1) if self.cdm.device_type == DeviceTypes.CHROME: self.userauthdata = UserAuthentication.NetflixIDCookies( netflixid=self.session.cookies.get_dict()["NetflixId"], securenetflixid=self.session.cookies.get_dict()["SecureNetflixId"], ) else: if not self.credentials: self.log.error(" - Credentials are required for Android CDMs, and none were provided.") sys.exit(1) # need to get cookies via an android-like way # outdated # self.android_login(credentials.username, credentials.password) # need to use EmailPassword for userauthdata, it specifically checks for this self.userauthdata = UserAuthentication.EmailPassword( email=self.credentials.username, password=self.credentials.password ) self.react_context = self.get_react_context() def get_profiles(self): if self.range in ("HDR10", "DV") and self.vcodec not in ("H265", "VP9"): self.vcodec = "H265" profiles = self.config["profiles"]["video"][self.vcodec] if self.range and self.range in profiles: return profiles[self.range] return profiles def get_react_context(self): """Netflix uses a "BUILD_IDENTIFIER" value on some API's, e.g. the Shakti (metadata) API. This value isn't given to the user through normal means so REGEX is needed. It's obtained by grabbing the body of a logged-in netflix homepage. The value changes often but doesn't often matter if it's only a bit out of date. It also uses a Client Version for various MPL calls. :returns: reactContext parsed json-loaded dictionary """ cached_context = self.cache.get(f"data_{self.user_profile}") if not cached_context: src = self.session.get("https://www.netflix.com/browse").text match = re.search(r"netflix\.reactContext = ({.+});